[Java] Package native dependencies into jar (#4367)

2026-06-28 17:18:45 +08:00 · 2019-03-15 12:38:40 +08:00
parent 6b93ec3034
commit f8d12b0418
14 changed files with 126 additions and 102 deletions
@@ -2,8 +2,13 @@ package org.ray.runtime;

 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
+import java.io.File;
+import java.io.InputStream;
 import java.lang.reflect.Field;
 import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -72,11 +77,22 @@ public final class RayNativeRuntime extends AbstractRayRuntime {

  @Override
  public void start() throws Exception {
-    // Load native libraries.
    try {
+      // Reset library path at runtime.
      resetLibraryPath();
-      System.loadLibrary("raylet_library_java");
-      System.loadLibrary("plasma_java");
+
+      // Load native libraries.
+      String[] libraries = new String[]{"raylet_library_java", "plasma_java"};
+      for (String library : libraries) {
+        String fileName = System.mapLibraryName(library);
+        // Copy the file from resources to a temp dir, and load the native library.
+        File file = File.createTempFile(fileName, "");
+        file.deleteOnExit();
+        InputStream in = RayNativeRuntime.class.getResourceAsStream("/" + fileName);
+        Preconditions.checkNotNull(in, "{} doesn't exist.", fileName);
+        Files.copy(in, Paths.get(file.getAbsolutePath()), StandardCopyOption.REPLACE_EXISTING);
+        System.load(file.getAbsolutePath());
+      }
    } catch (Exception e) {
      LOGGER.error("Failed to load native libraries.", e);
      throw e;
@@ -30,7 +30,6 @@ public class RayConfig {
  public static final String DEFAULT_CONFIG_FILE = "ray.default.conf";
  public static final String CUSTOM_CONFIG_FILE = "ray.conf";

-  public final String rayHome;
  public final String nodeIp;
  public final WorkerMode workerMode;
  public final RunMode runMode;
@@ -56,10 +55,6 @@ public class RayConfig {
  public final String rayletSocketName;
  public final List<String> rayletConfigParameters;

-  public final String redisServerExecutablePath;
-  public final String redisModulePath;
-  public final String plasmaStoreExecutablePath;
-  public final String rayletExecutablePath;
  public final String driverResourcePath;
  public final String pythonWorkerCommand;

@@ -72,9 +67,6 @@ public class RayConfig {
    if (workerMode == WorkerMode.WORKER) {
      Preconditions.checkArgument(redisAddress != null,
          "Redis address must be set in worker mode.");
-    } else {
-      Preconditions.checkArgument(!rayHome.isEmpty(),
-          "'ray.home' must be set in driver mode");
    }
  }

@@ -87,32 +79,24 @@ public class RayConfig {
  }

  public RayConfig(Config config) {
-    // worker mode
+    // Worker mode.
    WorkerMode localWorkerMode;
    try {
      localWorkerMode = config.getEnum(WorkerMode.class, "ray.worker.mode");
    } catch (ConfigException.Missing e) {
      localWorkerMode = WorkerMode.DRIVER;
    }
-
    workerMode = localWorkerMode;
    boolean isDriver = workerMode == WorkerMode.DRIVER;
-    // run mode
+    // Run mode.
    runMode = config.getEnum(RunMode.class, "ray.run-mode");
-    // ray home
-    String localRayHome = config.getString("ray.home");
-    if (!localRayHome.startsWith("/")) {
-      // If ray.home isn't an absolute path, prepend it with current work dir.
-      localRayHome = System.getProperty("user.dir") + "/" + localRayHome;
-    }
-    rayHome = removeTrailingSlash(localRayHome);
-    // node ip
+    // Node ip.
    String nodeIp = config.getString("ray.node-ip");
    if (nodeIp.isEmpty()) {
      nodeIp = NetworkUtil.getIpAddress(null);
    }
    this.nodeIp = nodeIp;
-    // resources
+    // Resources.
    resources = ResourceUtil.getResourcesMapFromString(
        config.getString("ray.resources"));
    if (isDriver) {
@@ -127,22 +111,22 @@ public class RayConfig {
        resources.put("GPU", 0.0);
      }
    }
-    // driver id
+    // Driver id.
    String driverId = config.getString("ray.driver.id");
    if (!driverId.isEmpty()) {
      this.driverId = UniqueId.fromHexString(driverId);
    } else {
      this.driverId = UniqueId.randomId();
    }
-    // log dir
+    // Log dir.
    logDir = removeTrailingSlash(config.getString("ray.log-dir"));
-    // redirect output
+    // Redirect output.
    redirectOutput = config.getBoolean("ray.redirect-output");
-    // custom library path
-    List<String> customLibraryPath = config.getStringList("ray.library.path");
-    // custom classpath
+    // Library path.
+    libraryPath = config.getStringList("ray.library.path");
+    // Custom classpath.
    classpath = config.getStringList("ray.classpath");
-    // custom worker jvm parameters
+    // Custom worker jvm parameters.
    if (config.hasPath("ray.worker.jvm-parameters")) {
      jvmParameters = config.getStringList("ray.worker.jvm-parameters");
    } else {
@@ -155,7 +139,7 @@ public class RayConfig {
      pythonWorkerCommand = null;
    }

-    // redis configurations
+    // Redis configurations.
    String redisAddress = config.getString("ray.redis.address");
    if (!redisAddress.isEmpty()) {
      setRedisAddress(redisAddress);
@@ -167,34 +151,22 @@ public class RayConfig {
    headRedisPassword = config.getString("ray.redis.head-password");
    redisPassword = config.getString("ray.redis.password");

-    // object store configurations
+    // Object store configurations.
    objectStoreSocketName = config.getString("ray.object-store.socket-name");
    objectStoreSize = config.getBytes("ray.object-store.size");

-    // raylet socket name
+    // Raylet socket name.
    rayletSocketName = config.getString("ray.raylet.socket-name");

-    // raylet parameters
-    rayletConfigParameters = new ArrayList<String>();
+    // Raylet parameters.
+    rayletConfigParameters = new ArrayList<>();
    Config rayletConfig = config.getConfig("ray.raylet.config");
    for (Map.Entry<String,ConfigValue> entry : rayletConfig.entrySet()) {
-      String parameter = entry.getKey() + "," + String.valueOf(entry.getValue().unwrapped());
+      String parameter = entry.getKey() + "," + entry.getValue().unwrapped();
      rayletConfigParameters.add(parameter);
    }

-    // library path
-    this.libraryPath = new ImmutableList.Builder<String>().add(
-        rayHome + "/build/src/plasma",
-        rayHome + "/build/src/ray/raylet"
-    ).addAll(customLibraryPath).build();
-
-    redisServerExecutablePath = rayHome +
-        "/build/src/ray/thirdparty/redis/src/redis-server";
-    redisModulePath = rayHome + "/build/src/ray/gcs/redis_module/libray_redis_module.so";
-    plasmaStoreExecutablePath = rayHome + "/build/src/plasma/plasma_store_server";
-    rayletExecutablePath = rayHome + "/build/src/ray/raylet/raylet";
-
-    // driver resource path
+    // Driver resource path.
    if (config.hasPath("ray.driver.resource-path")) {
      driverResourcePath = config.getString("ray.driver.resource-path");
    } else {
@@ -204,7 +176,7 @@ public class RayConfig {
    // Number of threads that execute tasks.
    numberExecThreadsForDevRuntime = config.getInt("ray.dev-runtime.execution-parallelism");

-    // validate config
+    // Validate config.
    validate();
    LOGGER.debug("Created config: {}", this);
  }
@@ -235,7 +207,6 @@ public class RayConfig {
  @Override
  public String toString() {
    return "RayConfig{"
-        + "rayHome='" + rayHome + '\''
        + ", nodeIp='" + nodeIp + '\''
        + ", workerMode=" + workerMode
        + ", runMode=" + runMode
@@ -255,10 +226,6 @@ public class RayConfig {
        + ", objectStoreSize=" + objectStoreSize
        + ", rayletSocketName='" + rayletSocketName + '\''
        + ", rayletConfigParameters=" + rayletConfigParameters
-        + ", redisServerExecutablePath='" + redisServerExecutablePath + '\''
-        + ", redisModulePath='" + redisModulePath + '\''
-        + ", plasmaStoreExecutablePath='" + plasmaStoreExecutablePath + '\''
-        + ", rayletExecutablePath='" + rayletExecutablePath + '\''
        + ", driverResourcePath='" + driverResourcePath + '\''
        + ", pythonWorkerCommand='" + pythonWorkerCommand + '\''
        + '}';
@@ -5,9 +5,14 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import java.io.File;
 import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
@@ -42,10 +47,13 @@ public class RunManager {

  private static final int KILL_PROCESS_WAIT_TIMEOUT_SECONDS = 1;

+  private final Map<String, File> tempFiles;
+
  public RunManager(RayConfig rayConfig) {
    this.rayConfig = rayConfig;
    processes = new ArrayList<>();
    random = new Random();
+    tempFiles = new HashMap<>();
  }

  public void cleanup() {
@@ -61,7 +69,7 @@ public class RunManager {
        p.waitFor(KILL_PROCESS_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
      } catch (InterruptedException e) {
        LOGGER.warn("Got InterruptedException while waiting for process {}" +
-            " to be terminated.",  processes.get(i));
+            " to be terminated.", processes.get(i));
      }

      if (p.isAlive()) {
@@ -76,8 +84,30 @@ public class RunManager {
    FileUtil.mkDir(new File(rayConfig.objectStoreSocketName).getParentFile());
  }

+  /**
+   * Copy a file from resources to a temp dir, and return the file object.
+   */
+  private File getTempFile(String fileName) {
+    File file = tempFiles.get(fileName);
+    if (file == null) {
+      try {
+        file = File.createTempFile(fileName, "");
+        file.deleteOnExit();
+        try (InputStream in = RunManager.class.getResourceAsStream(fileName)) {
+          Files.copy(in, Paths.get(file.getCanonicalPath()), StandardCopyOption.REPLACE_EXISTING);
+        }
+        file.setExecutable(true);
+      } catch (IOException e) {
+        throw new RuntimeException("Couldn't get temp file " + fileName, e);
+      }
+      tempFiles.put(fileName, file);
+    }
+    return file;
+  }
+
  /**
   * Start a process.
+   *
   * @param command The command to start the process with.
   * @param env Environment variables.
   * @param name Process name.
@@ -126,6 +156,7 @@ public class RunManager {

  /**
   * Start all Ray processes on this node.
+   *
   * @param isHead Whether this node is the head node. If true, redis server will be started.
   */
  public void startRayProcesses(boolean isHead) {
@@ -171,7 +202,8 @@ public class RunManager {

  private String startRedisInstance(String ip, int port, String password, Integer shard) {
    List<String> command = Lists.newArrayList(
-        rayConfig.redisServerExecutablePath,
+        // The redis-server executable file.
+        getTempFile("/redis-server").getAbsolutePath(),
        "--protected-mode",
        "no",
        "--port",
@@ -179,7 +211,8 @@ public class RunManager {
        "--loglevel",
        "warning",
        "--loadmodule",
-        rayConfig.redisModulePath
+        // The redis module file.
+        getTempFile("/libray_redis_module.so").getAbsolutePath()
    );

    if (!StringUtil.isNullOrEmpty(password)) {
@@ -216,7 +249,8 @@ public class RunManager {

    // See `src/ray/raylet/main.cc` for the meaning of each parameter.
    List<String> command = ImmutableList.of(
-        rayConfig.rayletExecutablePath,
+        // The raylet executable file.
+        getTempFile("/raylet").getAbsolutePath(),
        rayConfig.rayletSocketName,
        rayConfig.objectStoreSocketName,
        "0",  // The object manager port.
@@ -291,7 +325,8 @@ public class RunManager {

  private void startObjectStore() {
    List<String> command = ImmutableList.of(
-        rayConfig.plasmaStoreExecutablePath,
+        // The plasma store executable file.
+        getTempFile("/plasma_store_server").getAbsolutePath(),
        "-s",
        rayConfig.objectStoreSocketName,
        "-m",
@@ -7,11 +7,6 @@ ray {
  // Basic configurations
  // ----------------------

-  // This is the path to the directory where Ray is installed, e.g.,
-  // something like /home/ubmutu/ray. This can be an absolute path or
-  // a relative path from the current working directory.
-  home: ""
-
  // IP of this node. if not provided, IP will be automatically detected.
  node-ip: ""