[Java] Automatically clean up temp files. (#5507)

* Remove unused code

* Auto clean up temp files

* refine

* fix

* print warning

* fix

java/runtime/src/main/java/org/ray/runtime/AbstractRayRuntime.java

@@ -1,6 +1,7 @@
 package org.ray.runtime;
 
 import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import java.util.List;
 import org.ray.api.RayActor;
@@ -31,7 +32,6 @@ import org.ray.runtime.task.ArgumentsBuilder;
 import org.ray.runtime.task.FunctionArg;
 import org.ray.runtime.task.TaskExecutor;
 import org.ray.runtime.task.TaskSubmitter;
-import org.ray.runtime.util.StringUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -201,7 +201,7 @@ public abstract class AbstractRayRuntime implements RayRuntime {
     List<FunctionArg> functionArgs = ArgumentsBuilder
         .wrap(args, functionDescriptor.getLanguage() != Language.JAVA);
     if (functionDescriptor.getLanguage() != Language.JAVA && options != null) {
-      Preconditions.checkState(StringUtil.isNullOrEmpty(options.jvmOptions));
+      Preconditions.checkState(Strings.isNullOrEmpty(options.jvmOptions));
     }
     RayActor actor = taskSubmitter
         .createActor(functionDescriptor, functionArgs,

java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java

@@ -2,13 +2,7 @@ package org.ray.runtime;
 
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
 import java.lang.reflect.Field;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
 import java.util.HashMap;
 import java.util.Map;
 import org.ray.api.id.JobId;
@@ -23,6 +17,7 @@ import org.ray.runtime.raylet.NativeRayletClient;
 import org.ray.runtime.runner.RunManager;
 import org.ray.runtime.task.NativeTaskSubmitter;
 import org.ray.runtime.task.TaskExecutor;
+import org.ray.runtime.util.FileUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -41,23 +36,15 @@ public final class RayNativeRuntime extends AbstractRayRuntime {
   private long nativeCoreWorkerPointer;
 
   static {
-    try {
-      LOGGER.debug("Loading native libraries.");
-      // Load native libraries.
-      String[] libraries = new String[]{"core_worker_library_java"};
-      for (String library : libraries) {
-        String fileName = System.mapLibraryName(library);
-        // Copy the file from resources to a temp dir, and load the native library.
-        File file = File.createTempFile(fileName, "");
-        file.deleteOnExit();
-        InputStream in = AbstractRayRuntime.class.getResourceAsStream("/" + fileName);
-        Preconditions.checkNotNull(in, "{} doesn't exist.", fileName);
-        Files.copy(in, Paths.get(file.getAbsolutePath()), StandardCopyOption.REPLACE_EXISTING);
-        System.load(file.getAbsolutePath());
+    LOGGER.debug("Loading native libraries.");
+    // Load native libraries.
+    String[] libraries = new String[]{"core_worker_library_java"};
+    for (String library : libraries) {
+      String fileName = System.mapLibraryName(library);
+      try (FileUtil.TempFile libFile = FileUtil.getTempFileFromResource(fileName)) {
+        System.load(libFile.getFile().getAbsolutePath());
       }
       LOGGER.debug("Native libraries loaded.");
-    } catch (IOException e) {
-      throw new RuntimeException("Couldn't load native libraries.", e);
     }
     nativeSetup(RayConfig.create().logDir);
     Runtime.getRuntime().addShutdownHook(new Thread(RayNativeRuntime::nativeShutdownHook));

java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java

@@ -1,6 +1,7 @@
 package org.ray.runtime.config;
 
 import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.typesafe.config.Config;
 import com.typesafe.config.ConfigException;
@@ -14,7 +15,6 @@ import org.ray.api.id.JobId;
 import org.ray.runtime.generated.Common.WorkerType;
 import org.ray.runtime.util.NetworkUtil;
 import org.ray.runtime.util.ResourceUtil;
-import org.ray.runtime.util.StringUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -244,7 +244,7 @@ public class RayConfig {
     ConfigFactory.invalidateCaches();
     Config config = ConfigFactory.systemProperties();
     String configPath = System.getProperty("ray.config");
-    if (StringUtil.isNullOrEmpty(configPath)) {
+    if (Strings.isNullOrEmpty(configPath)) {
       LOGGER.info("Loading config from \"ray.conf\" file in classpath.");
       config = config.withFallback(ConfigFactory.load(CUSTOM_CONFIG_FILE));
     } else {

java/runtime/src/main/java/org/ray/runtime/gcs/RedisClient.java

@@ -1,9 +1,8 @@
 package org.ray.runtime.gcs;
 
+import com.google.common.base.Strings;
 import java.util.List;
 import java.util.Map;
-
-import org.ray.runtime.util.StringUtil;
 import redis.clients.jedis.Jedis;
 import redis.clients.jedis.JedisPool;
 import redis.clients.jedis.JedisPoolConfig;
@@ -31,7 +30,7 @@ public class RedisClient {
     JedisPoolConfig jedisPoolConfig = new JedisPoolConfig();
     jedisPoolConfig.setMaxTotal(JEDIS_POOL_SIZE);
 
-    if (StringUtil.isNullOrEmpty(password)) {
+    if (Strings.isNullOrEmpty(password)) {
       jedisPool = new JedisPool(jedisPoolConfig,
           ipAndPort[0], Integer.parseInt(ipAndPort[1]), 30000);
     } else {

java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java

@@ -1,30 +1,25 @@
 package org.ray.runtime.runner;
 
 import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import java.io.File;
 import java.io.IOException;
-import java.io.InputStream;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
 import java.time.LocalDateTime;
 import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.tuple.Pair;
 import org.ray.runtime.config.RayConfig;
 import org.ray.runtime.util.FileUtil;
 import org.ray.runtime.util.ResourceUtil;
-import org.ray.runtime.util.StringUtil;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import redis.clients.jedis.Jedis;
@@ -49,8 +44,6 @@ public class RunManager {
 
   private static final int KILL_PROCESS_WAIT_TIMEOUT_SECONDS = 1;
 
-  private static final Map<String, File> tempFiles = new HashMap<>();
-
   public RunManager(RayConfig rayConfig) {
     this.rayConfig = rayConfig;
     processes = new ArrayList<>();
@@ -89,31 +82,14 @@ public class RunManager {
   }
 
   private void createTempDirs() {
-    FileUtil.mkDir(new File(rayConfig.logDir));
-    FileUtil.mkDir(new File(rayConfig.rayletSocketName).getParentFile());
-    FileUtil.mkDir(new File(rayConfig.objectStoreSocketName).getParentFile());
-  }
-
-  /**
-   * Copy a file from resources to a temp dir, and return the file object.
-   */
-  private File getTempFile(String fileName) {
-    File file = tempFiles.get(fileName);
-    if (file == null) {
-      try {
-        file = File.createTempFile(fileName, "");
-        file.deleteOnExit();
-        try (InputStream in = RunManager.class.getResourceAsStream(fileName)) {
-          Preconditions.checkNotNull(in, "{} doesn't exist.", fileName);
-          Files.copy(in, Paths.get(file.getCanonicalPath()), StandardCopyOption.REPLACE_EXISTING);
-        }
-        file.setExecutable(true);
-      } catch (IOException e) {
-        throw new RuntimeException("Couldn't get temp file " + fileName, e);
-      }
-      tempFiles.put(fileName, file);
+    try {
+      FileUtils.forceMkdir(new File(rayConfig.logDir));
+      FileUtils.forceMkdir(new File(rayConfig.rayletSocketName).getParentFile());
+      FileUtils.forceMkdir(new File(rayConfig.objectStoreSocketName).getParentFile());
+    } catch (IOException e) {
+      LOGGER.error("Couldn't create temp directories.", e);
+      throw new RuntimeException(e);
     }
-    return file;
   }
 
   /**
@@ -125,18 +101,20 @@ public class RunManager {
    */
   private void startProcess(List<String> command, Map<String, String> env, String name) {
     if (LOGGER.isDebugEnabled()) {
-      LOGGER.debug("Starting process {} with command: {}", name, command,
+      LOGGER.debug("Starting process {} with command: {}", name,
           Joiner.on(" ").join(command));
     }
 
     ProcessBuilder builder = new ProcessBuilder(command);
 
+    String stdout = "";
+    String stderr = "";
     if (rayConfig.redirectOutput) {
       // Set stdout and stderr paths.
       int logId = random.nextInt(10000);
       String date = DATE_TIME_FORMATTER.format(LocalDateTime.now());
-      String stdout = String.format("%s/%s-%s-%05d.out", rayConfig.logDir, name, date, logId);
-      String stderr = String.format("%s/%s-%s-%05d.err", rayConfig.logDir, name, date, logId);
+      stdout = String.format("%s/%s-%s-%05d.out", rayConfig.logDir, name, date, logId);
+      stderr = String.format("%s/%s-%s-%05d.err", rayConfig.logDir, name, date, logId);
       builder.redirectOutput(new File(stdout));
       builder.redirectError(new File(stderr));
     }
@@ -162,7 +140,13 @@ public class RunManager {
       throw new RuntimeException("Failed to start " + name);
     }
     processes.add(Pair.of(name, p));
-    LOGGER.info("{} process started", name);
+    if (LOGGER.isInfoEnabled()) {
+      String message = String.format("%s process started.", name);
+      if (rayConfig.redirectOutput) {
+        message += String.format(" Logs are redirected to %s and %s.", stdout, stderr);
+      }
+      LOGGER.info(message);
+    }
   }
 
   /**
@@ -194,7 +178,7 @@ public class RunManager {
         rayConfig.headRedisPort, rayConfig.headRedisPassword, null);
     rayConfig.setRedisAddress(primary);
     try (Jedis client = new Jedis("127.0.0.1", rayConfig.headRedisPort)) {
-      if (!StringUtil.isNullOrEmpty(rayConfig.headRedisPassword)) {
+      if (!Strings.isNullOrEmpty(rayConfig.headRedisPassword)) {
         client.auth(rayConfig.headRedisPassword);
       }
       client.set("UseRaylet", "1");
@@ -214,30 +198,36 @@ public class RunManager {
   }
 
   private String startRedisInstance(String ip, int port, String password, Integer shard) {
-    List<String> command = Lists.newArrayList(
-        // The redis-server executable file.
-        getTempFile("/redis-server").getAbsolutePath(),
-        "--protected-mode",
-        "no",
-        "--port",
-        String.valueOf(port),
-        "--loglevel",
-        "warning",
-        "--loadmodule",
-        // The redis module file.
-        getTempFile("/libray_redis_module.so").getAbsolutePath()
-    );
+    try (FileUtil.TempFile redisServerFile = FileUtil.getTempFileFromResource("redis-server")) {
+      try (FileUtil.TempFile redisModuleFile = FileUtil.getTempFileFromResource(
+          "libray_redis_module.so")) {
+        redisServerFile.getFile().setExecutable(true);
+        List<String> command = Lists.newArrayList(
+            // The redis-server executable file.
+            redisServerFile.getFile().getAbsolutePath(),
+            "--protected-mode",
+            "no",
+            "--port",
+            String.valueOf(port),
+            "--loglevel",
+            "warning",
+            "--loadmodule",
+            // The redis module file.
+            redisModuleFile.getFile().getAbsolutePath()
+        );
 
-    if (!StringUtil.isNullOrEmpty(password)) {
-      command.add("--requirepass ");
-      command.add(password);
+        if (!Strings.isNullOrEmpty(password)) {
+          command.add("--requirepass ");
+          command.add(password);
+        }
+
+        String name = shard == null ? "redis" : "redis-" + shard;
+        startProcess(command, null, name);
+      }
     }
 
-    String name = shard == null ? "redis" : "redis-" + shard;
-    startProcess(command, null, name);
-
     try (Jedis client = new Jedis("127.0.0.1", port)) {
-      if (!StringUtil.isNullOrEmpty(password)) {
+      if (!Strings.isNullOrEmpty(password)) {
         client.auth(password);
       }
 
@@ -256,32 +246,34 @@ public class RunManager {
         Math.min(rayConfig.resources.getOrDefault("CPU", 0.0).intValue(), hardwareConcurrency));
 
     String redisPasswordOption = "";
-    if (!StringUtil.isNullOrEmpty(rayConfig.headRedisPassword)) {
+    if (!Strings.isNullOrEmpty(rayConfig.headRedisPassword)) {
       redisPasswordOption = rayConfig.headRedisPassword;
     }
 
     // See `src/ray/raylet/main.cc` for the meaning of each parameter.
-    List<String> command = ImmutableList.of(
-        // The raylet executable file.
-        getTempFile("/raylet").getAbsolutePath(),
-        String.format("--raylet_socket_name=%s", rayConfig.rayletSocketName),
-        String.format("--store_socket_name=%s", rayConfig.objectStoreSocketName),
-        String.format("--object_manager_port=%d", 0), // The object manager port.
-        String.format("--node_manager_port=%d", 0),  // The node manager port.
-        String.format("--node_ip_address=%s", rayConfig.nodeIp),
-        String.format("--redis_address=%s", rayConfig.getRedisIp()),
-        String.format("--redis_port=%d", rayConfig.getRedisPort()),
-        String.format("--num_initial_workers=%d", 0),  // number of initial workers
-        String.format("--maximum_startup_concurrency=%d", maximumStartupConcurrency),
-        String.format("--static_resource_list=%s",
-            ResourceUtil.getResourcesStringFromMap(rayConfig.resources)),
-        String.format("--config_list=%s", String.join(",", rayConfig.rayletConfigParameters)),
-        String.format("--python_worker_command=%s", buildPythonWorkerCommand()),
-        String.format("--java_worker_command=%s", buildWorkerCommandRaylet()),
-        String.format("--redis_password=%s", redisPasswordOption)
-    );
+    try (FileUtil.TempFile rayletFile = FileUtil.getTempFileFromResource("raylet")) {
+      rayletFile.getFile().setExecutable(true);
+      List<String> command = ImmutableList.of(
+          rayletFile.getFile().getAbsolutePath(),
+          String.format("--raylet_socket_name=%s", rayConfig.rayletSocketName),
+          String.format("--store_socket_name=%s", rayConfig.objectStoreSocketName),
+          String.format("--object_manager_port=%d", 0), // The object manager port.
+          String.format("--node_manager_port=%d", 0),  // The node manager port.
+          String.format("--node_ip_address=%s", rayConfig.nodeIp),
+          String.format("--redis_address=%s", rayConfig.getRedisIp()),
+          String.format("--redis_port=%d", rayConfig.getRedisPort()),
+          String.format("--num_initial_workers=%d", 0),  // number of initial workers
+          String.format("--maximum_startup_concurrency=%d", maximumStartupConcurrency),
+          String.format("--static_resource_list=%s",
+              ResourceUtil.getResourcesStringFromMap(rayConfig.resources)),
+          String.format("--config_list=%s", String.join(",", rayConfig.rayletConfigParameters)),
+          String.format("--python_worker_command=%s", buildPythonWorkerCommand()),
+          String.format("--java_worker_command=%s", buildWorkerCommandRaylet()),
+          String.format("--redis_password=%s", redisPasswordOption)
+      );
 
-    startProcess(command, null, "raylet");
+      startProcess(command, null, "raylet");
+    }
   }
 
   private String concatPath(Stream<String> stream) {
@@ -324,7 +316,7 @@ public class RunManager {
     cmd.add("-Dray.redis.address=" + rayConfig.getRedisAddress());
 
     // redis password
-    if (!StringUtil.isNullOrEmpty(rayConfig.headRedisPassword)) {
+    if (!Strings.isNullOrEmpty(rayConfig.headRedisPassword)) {
       cmd.add("-Dray.redis.password=" + rayConfig.headRedisPassword);
     }
 
@@ -341,15 +333,19 @@ public class RunManager {
   }
 
   private void startObjectStore() {
-    List<String> command = ImmutableList.of(
-        // The plasma store executable file.
-        getTempFile("/plasma_store_server").getAbsolutePath(),
-        "-s",
-        rayConfig.objectStoreSocketName,
-        "-m",
-        rayConfig.objectStoreSize.toString()
-    );
-    startProcess(command, null, "plasma_store");
+    try (FileUtil.TempFile plasmaStoreFile = FileUtil
+        .getTempFileFromResource("plasma_store_server")) {
+      plasmaStoreFile.getFile().setExecutable(true);
+      List<String> command = ImmutableList.of(
+          // The plasma store executable file.
+          plasmaStoreFile.getFile().getAbsolutePath(),
+          "-s",
+          rayConfig.objectStoreSocketName,
+          "-m",
+          rayConfig.objectStoreSize.toString()
+      );
+      startProcess(command, null, "plasma_store");
+    }
   }
 
   private String buildPythonWorkerCommand() {

java/runtime/src/main/java/org/ray/runtime/util/FileUtil.java

@@ -1,134 +1,68 @@
 package org.ray.runtime.util;
 
+import com.google.common.base.Preconditions;
 import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileWriter;
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.Files;
-import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.Scanner;
+import java.nio.file.StandardCopyOption;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class FileUtil {
 
-  public static String getFilename(String logPath) {
-    if (logPath != null && !logPath.isEmpty()) {
-      int lastPos = logPath.lastIndexOf('/');
-      if (lastPos != -1) {
-        return logPath.substring(lastPos + 1);
-      } else {
-        return logPath;
+  private static final Logger LOGGER = LoggerFactory.getLogger(FileUtil.class);
+
+  /**
+   * Represents a temp file.
+   *
+   * This class implements the `AutoCloseable` interface. It can be used in a `try-with-resource`
+   * block. When exiting the block, the temp file will be automatically removed.
+   */
+  public static class TempFile implements AutoCloseable {
+
+    File file;
+
+    TempFile(File file) {
+      this.file = file;
+    }
+
+    public File getFile() {
+      return file;
+    }
+
+    @Override
+    public void close() {
+      if (!file.delete()) {
+        LOGGER.warn("Couldn't delete temp file {}", file.getAbsolutePath());
       }
     }
-
-    return null;
-  }
-
-  public static boolean deleteFile(String filePath) {
-    File file = new File(filePath);
-    if (!file.exists()) {
-      return true;
-    } else {
-      if (file.isFile()) {
-        return file.delete();
-      } else {
-        for (File f : file.listFiles()) {
-          deleteFile(f.getAbsolutePath());
-        }
-        return file.delete();
-      }
-    }
-  }
-
-  public static void mkDir(File dir) {
-    if (dir.exists()) {
-      return;
-    }
-    if (dir.getParentFile().exists()) {
-      dir.mkdir();
-    } else {
-      mkDir(dir.getParentFile());
-      dir.mkdir();
-    }
-  }
-
-  public static void mkDirAndFile(File file) throws IOException {
-    if (file.exists()) {
-      return;
-    }
-    if (!file.getParentFile().exists()) {
-      mkDir(file.getParentFile());
-    }
-    file.createNewFile();
-  }
-
-  public static String readResourceFile(String fileName) throws FileNotFoundException {
-    ClassLoader classLoader = FileUtil.class.getClassLoader();
-    File file = new File(classLoader.getResource(fileName).getFile());
-    StringBuilder result = new StringBuilder();
-    try (Scanner scanner = new Scanner(file)) {
-
-      //Get file from resources folder
-
-      while (scanner.hasNextLine()) {
-        String line = scanner.nextLine();
-        result.append(line).append("\n");
-      }
-      return result.toString();
-    }
-
-  }
-
-  public static void overrideFile(String file, String str) throws IOException {
-    try (FileWriter fw = new FileWriter(file)) {
-      fw.write(str);
-    }
-  }
-
-  public static boolean createDir(String dirName, boolean failIfExist) {
-    File wdir = new File(dirName);
-    if (wdir.isFile()) {
-      return false;
-    }
-
-    if (!wdir.exists()) {
-      wdir.mkdirs();
-    } else if (failIfExist) {
-      return false;
-    }
-
-    return true;
-  }
-
-  public static void bytesToFile(byte[] bytes, String name) throws IOException {
-    Path path = Paths.get(name);
-    Files.write(path, bytes);
-  }
-
-  public static byte[] fileToBytes(String name) throws IOException {
-    Path path = Paths.get(name);
-    return Files.readAllBytes(path);
   }
 
   /**
-   * If the given string is the empty string, then the result is the current directory.
+   * Get a temp file from resource.
    *
-   * @param rawDir a path in any legal form, such as a relative path
-   * @return the absolute and unique path in String
+   * @param resourceFileName File name.
+   * @return A `TempFile` object.
    */
-  public static String getCanonicalDirectory(final String rawDir) throws IOException {
-    String dir = rawDir.length() == 0 ? "." : rawDir;
-
-    // create working dir if necessary
-    File dd = new File(dir);
-    if (!dd.exists()) {
-      dd.mkdirs();
+  public static TempFile getTempFileFromResource(String resourceFileName) {
+    File file;
+    try {
+      file = File.createTempFile(resourceFileName, "");
+    } catch (IOException e) {
+      throw new RuntimeException("Couldn't create temp file " + resourceFileName, e);
     }
 
-    if (!dir.startsWith("/")) {
-      dir = dd.getCanonicalPath();
+    try (InputStream in = FileUtil.class.getResourceAsStream("/" + resourceFileName)) {
+      Preconditions.checkNotNull(in, "{} doesn't exist.", resourceFileName);
+      Files.copy(in, Paths.get(file.getCanonicalPath()), StandardCopyOption.REPLACE_EXISTING);
+
+    } catch (IOException e) {
+      throw new RuntimeException("Couldn't get temp file from resource " + resourceFileName, e);
     }
 
-    return dir;
+    return new TempFile(file);
   }
 }
+

java/runtime/src/main/java/org/ray/runtime/util/NetworkUtil.java

@@ -1,5 +1,6 @@
 package org.ray.runtime.util;
 
+import com.google.common.base.Strings;
 import java.io.IOException;
 import java.net.DatagramSocket;
 import java.net.Inet6Address;
@@ -22,7 +23,7 @@ public class NetworkUtil {
         if (!current.isUp() || current.isLoopback() || current.isVirtual()) {
           continue;
         }
-        if (!StringUtil.isNullOrEmpty(interfaceName) && !interfaceName
+        if (!Strings.isNullOrEmpty(interfaceName) && !interfaceName
             .equals(current.getDisplayName())) {
           continue;
         }

java/runtime/src/main/java/org/ray/runtime/util/Sha1Digestor.java

@@ -1,43 +0,0 @@
-package org.ray.runtime.util;
-
-import java.nio.ByteBuffer;
-import java.security.MessageDigest;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public class Sha1Digestor {
-
-  private static final Logger LOGGER = LoggerFactory.getLogger(Sha1Digestor.class);
-  private static final ThreadLocal<MessageDigest> md = ThreadLocal.withInitial(() -> {
-    try {
-      return MessageDigest.getInstance("SHA1");
-    } catch (Exception e) {
-      LOGGER.error("Cannot get SHA1 MessageDigest", e);
-      throw new RuntimeException("Cannot get SHA1 digest", e);
-    }
-  });
-
-
-  private static final ThreadLocal<ByteBuffer> longBuffer = ThreadLocal
-      .withInitial(() -> ByteBuffer.allocate(Long.SIZE / Byte.SIZE));
-
-  public static byte[] digest(byte[] src, long addIndex) {
-    MessageDigest dg = md.get();
-    longBuffer.get().clear();
-    dg.reset();
-
-    dg.update(src);
-    dg.update(longBuffer.get().putLong(addIndex).array());
-    return dg.digest();
-  }
-
-  public static byte[] digest(String str) {
-    return digest(str.getBytes(StringUtil.UTF8));
-  }
-
-  public static byte[] digest(byte[] src) {
-    MessageDigest dg = md.get();
-    dg.reset();
-    return dg.digest(src);
-  }
-}

java/runtime/src/main/java/org/ray/runtime/util/StringUtil.java

@@ -1,150 +0,0 @@
-package org.ray.runtime.util;
-
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Vector;
-
-public class StringUtil {
-
-  public static final Charset UTF8 = Charset.forName("UTF-8");
-
-  private static final char[] HEX_CHARS = "0123456789abcdef".toCharArray();
-
-  /**
-   * split.
-   * @param s         input string
-   * @param splitters common splitters
-   * @param open      open braces
-   * @param close     close braces
-   * @return output array list
-   */
-  public static Vector<String> split(String s, String splitters, String open, String close) {
-    // The splits.
-    Vector<String> split = new Vector<>();
-    // The stack.
-    ArrayList<Start> stack = new ArrayList<>();
-
-    int lastPos = 0;
-
-    // Walk the string.
-    for (int i = 0; i < s.length(); i++) {
-      // Get the char there.
-      char ch = s.charAt(i);
-
-      // Is it an open brace?
-      int o = open.indexOf(ch);
-
-      // Is it a close brace?
-      int c = close.indexOf(ch);
-
-      // Is it a splitter?
-      int sp = splitters.indexOf(ch);
-
-      if (stack.size() == 0 && sp >= 0) {
-        if (i == lastPos) {
-          ++lastPos;
-          continue;
-        }
-
-        split.add(s.substring(lastPos, i));
-        lastPos = i + 1;
-      } else if (o >= 0 && (c < 0 || stack.size() == 0)) {
-        // Its an open! Push it.
-        stack.add(new Start(o, i));
-      } else if (c >= 0 && stack.size() > 0) {
-        // Pop (if matches).
-        int tosPos = stack.size() - 1;
-        Start tos = stack.get(tosPos);
-        // Does the brace match?
-        if (tos.brace == c) {
-          // Done with that one.
-          stack.remove(tosPos);
-        }
-      }
-    }
-
-    if (lastPos < s.length()) {
-      split.add(s.substring(lastPos, s.length()));
-    }
-
-    // build removal filter set
-    HashSet<Character> removals = new HashSet<>();
-    for (int i = 0; i < splitters.length(); i++) {
-      removals.add(splitters.charAt(i));
-    }
-
-    for (int i = 0; i < open.length(); i++) {
-      removals.add(open.charAt(i));
-    }
-
-    for (int i = 0; i < close.length(); i++) {
-      removals.add(close.charAt(i));
-    }
-
-    // apply removal filter set
-    for (int i = 0; i < split.size(); i++) {
-      String cs = split.get(i);
-
-      // remove heading chars
-      int j;
-      for (j = 0; j < cs.length(); j++) {
-        if (!removals.contains(cs.charAt(j))) {
-          break;
-        }
-      }
-      cs = cs.substring(j);
-
-      // remove tail chars
-      for (j = cs.length() - 1; j >= 0; j--) {
-        if (!removals.contains(cs.charAt(j))) {
-          break;
-        }
-      }
-      cs = cs.substring(0, j + 1);
-
-      // reset cs
-      split.set(i, cs);
-    }
-
-    return split;
-  }
-
-  public static boolean isNullOrEmpty(String s) {
-    return s == null || s.length() == 0;
-  }
-
-  public static <T> String mergeArray(T[] objs, String concatenator) {
-    StringBuilder sb = new StringBuilder();
-    for (T obj : objs) {
-      sb.append(obj).append(concatenator);
-    }
-    return objs.length == 0 ? "" : sb.substring(0, sb.length() - concatenator.length());
-  }
-
-  public static String toHexHashString(byte[] id) {
-    StringBuilder sb = new StringBuilder(20);
-    assert (id.length == 20);
-    for (int i = 0; i < 20; i++) {
-      int val = id[i] & 0xff;
-      sb.append(HEX_CHARS[val >> 4]);
-      sb.append(HEX_CHARS[val & 0xf]);
-    }
-    return sb.toString();
-  }
-
-  // Holds the start of an element and which brace started it.
-  private static class Start {
-
-    // The brace number from the braces string in use.
-    final int brace;
-    // The position in the string it was seen.
-    final int pos;
-
-    // Constructor.
-    public Start(int brace, int pos) {
-      this.brace = brace;
-      this.pos = pos;
-    }
-  }
-}

java/runtime/src/main/java/org/ray/runtime/util/generator/RayCallGenerator.java

@@ -1,9 +1,11 @@
 package org.ray.runtime.util.generator;
 
+import java.io.File;
 import java.io.IOException;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.List;
-import org.ray.runtime.util.FileUtil;
+import org.apache.commons.io.FileUtils;
 
 /**
  * A util class that generates `RayCall.java`, which provides type-safe interfaces for `Ray.call`
@@ -267,7 +269,7 @@ public class RayCallGenerator extends BaseGenerator {
   public static void main(String[] args) throws IOException {
     String path = System.getProperty("user.dir")
         + "/api/src/main/java/org/ray/api/RayCall.java";
-    FileUtil.overrideFile(path, new RayCallGenerator().build());
+    FileUtils.write(new File(path), new RayCallGenerator().build(), Charset.defaultCharset());
   }
 }
 

java/runtime/src/main/java/org/ray/runtime/util/generator/RayFuncGenerator.java

@@ -1,7 +1,9 @@
 package org.ray.runtime.util.generator;
 
+import java.io.File;
 import java.io.IOException;
-import org.ray.runtime.util.FileUtil;
+import java.nio.charset.Charset;
+import org.apache.commons.io.FileUtils;
 
 /**
  * A util class that generates all the RayFuncX classes under org.ray.api.function package.
@@ -58,10 +60,12 @@ public class RayFuncGenerator extends BaseGenerator {
     for (int i = 0; i <= MAX_PARAMETERS; i++) {
       // Functions that have return.
       String content = generator.generate(i, true);
-      FileUtil.overrideFile(root + "RayFunc" + i + ".java", content);
+      FileUtils.write(new File(root + "RayFunc" + i + ".java"), content,
+          Charset.defaultCharset());
       // Functions that don't have return.
       content = generator.generate(i, false);
-      FileUtil.overrideFile(root + "RayFuncVoid" + i + ".java", content);
+      FileUtils.write(new File(root + "RayFuncVoid" + i + ".java"), content,
+          Charset.defaultCharset());
     }
   }