From fab5ae64c4d0cdc5bcacb45f39989aaac8799e20 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Sat, 24 Aug 2019 17:20:27 +0800 Subject: [PATCH] [Java] Automatically clean up temp files. (#5507) * Remove unused code * Auto clean up temp files * refine * fix * print warning * fix --- .../org/ray/runtime/AbstractRayRuntime.java | 4 +- .../org/ray/runtime/RayNativeRuntime.java | 29 +-- .../org/ray/runtime/config/RayConfig.java | 4 +- .../java/org/ray/runtime/gcs/RedisClient.java | 5 +- .../org/ray/runtime/runner/RunManager.java | 174 +++++++++--------- .../java/org/ray/runtime/util/FileUtil.java | 158 +++++----------- .../org/ray/runtime/util/NetworkUtil.java | 3 +- .../org/ray/runtime/util/Sha1Digestor.java | 43 ----- .../java/org/ray/runtime/util/StringUtil.java | 150 --------------- .../util/generator/RayCallGenerator.java | 6 +- .../util/generator/RayFuncGenerator.java | 10 +- .../java/org/ray/api/test/WordCountTest.java | 74 -------- 12 files changed, 158 insertions(+), 502 deletions(-) delete mode 100644 java/runtime/src/main/java/org/ray/runtime/util/Sha1Digestor.java delete mode 100644 java/runtime/src/main/java/org/ray/runtime/util/StringUtil.java delete mode 100644 java/test/src/main/java/org/ray/api/test/WordCountTest.java diff --git a/java/runtime/src/main/java/org/ray/runtime/AbstractRayRuntime.java b/java/runtime/src/main/java/org/ray/runtime/AbstractRayRuntime.java index 36143f0d1..e3132aa29 100644 --- a/java/runtime/src/main/java/org/ray/runtime/AbstractRayRuntime.java +++ b/java/runtime/src/main/java/org/ray/runtime/AbstractRayRuntime.java @@ -1,6 +1,7 @@ package org.ray.runtime; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import java.util.List; import org.ray.api.RayActor; @@ -31,7 +32,6 @@ import org.ray.runtime.task.ArgumentsBuilder; import org.ray.runtime.task.FunctionArg; import org.ray.runtime.task.TaskExecutor; import org.ray.runtime.task.TaskSubmitter; -import org.ray.runtime.util.StringUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -201,7 +201,7 @@ public abstract class AbstractRayRuntime implements RayRuntime { List functionArgs = ArgumentsBuilder .wrap(args, functionDescriptor.getLanguage() != Language.JAVA); if (functionDescriptor.getLanguage() != Language.JAVA && options != null) { - Preconditions.checkState(StringUtil.isNullOrEmpty(options.jvmOptions)); + Preconditions.checkState(Strings.isNullOrEmpty(options.jvmOptions)); } RayActor actor = taskSubmitter .createActor(functionDescriptor, functionArgs, diff --git a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java index c15a87033..28a0d0828 100644 --- a/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java +++ b/java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java @@ -2,13 +2,7 @@ package org.ray.runtime; import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; import java.lang.reflect.Field; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.util.HashMap; import java.util.Map; import org.ray.api.id.JobId; @@ -23,6 +17,7 @@ import org.ray.runtime.raylet.NativeRayletClient; import org.ray.runtime.runner.RunManager; import org.ray.runtime.task.NativeTaskSubmitter; import org.ray.runtime.task.TaskExecutor; +import org.ray.runtime.util.FileUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,23 +36,15 @@ public final class RayNativeRuntime extends AbstractRayRuntime { private long nativeCoreWorkerPointer; static { - try { - LOGGER.debug("Loading native libraries."); - // Load native libraries. - String[] libraries = new String[]{"core_worker_library_java"}; - for (String library : libraries) { - String fileName = System.mapLibraryName(library); - // Copy the file from resources to a temp dir, and load the native library. - File file = File.createTempFile(fileName, ""); - file.deleteOnExit(); - InputStream in = AbstractRayRuntime.class.getResourceAsStream("/" + fileName); - Preconditions.checkNotNull(in, "{} doesn't exist.", fileName); - Files.copy(in, Paths.get(file.getAbsolutePath()), StandardCopyOption.REPLACE_EXISTING); - System.load(file.getAbsolutePath()); + LOGGER.debug("Loading native libraries."); + // Load native libraries. + String[] libraries = new String[]{"core_worker_library_java"}; + for (String library : libraries) { + String fileName = System.mapLibraryName(library); + try (FileUtil.TempFile libFile = FileUtil.getTempFileFromResource(fileName)) { + System.load(libFile.getFile().getAbsolutePath()); } LOGGER.debug("Native libraries loaded."); - } catch (IOException e) { - throw new RuntimeException("Couldn't load native libraries.", e); } nativeSetup(RayConfig.create().logDir); Runtime.getRuntime().addShutdownHook(new Thread(RayNativeRuntime::nativeShutdownHook)); diff --git a/java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java b/java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java index 1e90d68f4..05e18ed27 100644 --- a/java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java +++ b/java/runtime/src/main/java/org/ray/runtime/config/RayConfig.java @@ -1,6 +1,7 @@ package org.ray.runtime.config; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.typesafe.config.Config; import com.typesafe.config.ConfigException; @@ -14,7 +15,6 @@ import org.ray.api.id.JobId; import org.ray.runtime.generated.Common.WorkerType; import org.ray.runtime.util.NetworkUtil; import org.ray.runtime.util.ResourceUtil; -import org.ray.runtime.util.StringUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -244,7 +244,7 @@ public class RayConfig { ConfigFactory.invalidateCaches(); Config config = ConfigFactory.systemProperties(); String configPath = System.getProperty("ray.config"); - if (StringUtil.isNullOrEmpty(configPath)) { + if (Strings.isNullOrEmpty(configPath)) { LOGGER.info("Loading config from \"ray.conf\" file in classpath."); config = config.withFallback(ConfigFactory.load(CUSTOM_CONFIG_FILE)); } else { diff --git a/java/runtime/src/main/java/org/ray/runtime/gcs/RedisClient.java b/java/runtime/src/main/java/org/ray/runtime/gcs/RedisClient.java index 5e0a7b978..e2261509b 100644 --- a/java/runtime/src/main/java/org/ray/runtime/gcs/RedisClient.java +++ b/java/runtime/src/main/java/org/ray/runtime/gcs/RedisClient.java @@ -1,9 +1,8 @@ package org.ray.runtime.gcs; +import com.google.common.base.Strings; import java.util.List; import java.util.Map; - -import org.ray.runtime.util.StringUtil; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; @@ -31,7 +30,7 @@ public class RedisClient { JedisPoolConfig jedisPoolConfig = new JedisPoolConfig(); jedisPoolConfig.setMaxTotal(JEDIS_POOL_SIZE); - if (StringUtil.isNullOrEmpty(password)) { + if (Strings.isNullOrEmpty(password)) { jedisPool = new JedisPool(jedisPoolConfig, ipAndPort[0], Integer.parseInt(ipAndPort[1]), 30000); } else { diff --git a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java index 5e197da89..97f6dd135 100644 --- a/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java +++ b/java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java @@ -1,30 +1,25 @@ package org.ray.runtime.runner; import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; +import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.tuple.Pair; import org.ray.runtime.config.RayConfig; import org.ray.runtime.util.FileUtil; import org.ray.runtime.util.ResourceUtil; -import org.ray.runtime.util.StringUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import redis.clients.jedis.Jedis; @@ -49,8 +44,6 @@ public class RunManager { private static final int KILL_PROCESS_WAIT_TIMEOUT_SECONDS = 1; - private static final Map tempFiles = new HashMap<>(); - public RunManager(RayConfig rayConfig) { this.rayConfig = rayConfig; processes = new ArrayList<>(); @@ -89,31 +82,14 @@ public class RunManager { } private void createTempDirs() { - FileUtil.mkDir(new File(rayConfig.logDir)); - FileUtil.mkDir(new File(rayConfig.rayletSocketName).getParentFile()); - FileUtil.mkDir(new File(rayConfig.objectStoreSocketName).getParentFile()); - } - - /** - * Copy a file from resources to a temp dir, and return the file object. - */ - private File getTempFile(String fileName) { - File file = tempFiles.get(fileName); - if (file == null) { - try { - file = File.createTempFile(fileName, ""); - file.deleteOnExit(); - try (InputStream in = RunManager.class.getResourceAsStream(fileName)) { - Preconditions.checkNotNull(in, "{} doesn't exist.", fileName); - Files.copy(in, Paths.get(file.getCanonicalPath()), StandardCopyOption.REPLACE_EXISTING); - } - file.setExecutable(true); - } catch (IOException e) { - throw new RuntimeException("Couldn't get temp file " + fileName, e); - } - tempFiles.put(fileName, file); + try { + FileUtils.forceMkdir(new File(rayConfig.logDir)); + FileUtils.forceMkdir(new File(rayConfig.rayletSocketName).getParentFile()); + FileUtils.forceMkdir(new File(rayConfig.objectStoreSocketName).getParentFile()); + } catch (IOException e) { + LOGGER.error("Couldn't create temp directories.", e); + throw new RuntimeException(e); } - return file; } /** @@ -125,18 +101,20 @@ public class RunManager { */ private void startProcess(List command, Map env, String name) { if (LOGGER.isDebugEnabled()) { - LOGGER.debug("Starting process {} with command: {}", name, command, + LOGGER.debug("Starting process {} with command: {}", name, Joiner.on(" ").join(command)); } ProcessBuilder builder = new ProcessBuilder(command); + String stdout = ""; + String stderr = ""; if (rayConfig.redirectOutput) { // Set stdout and stderr paths. int logId = random.nextInt(10000); String date = DATE_TIME_FORMATTER.format(LocalDateTime.now()); - String stdout = String.format("%s/%s-%s-%05d.out", rayConfig.logDir, name, date, logId); - String stderr = String.format("%s/%s-%s-%05d.err", rayConfig.logDir, name, date, logId); + stdout = String.format("%s/%s-%s-%05d.out", rayConfig.logDir, name, date, logId); + stderr = String.format("%s/%s-%s-%05d.err", rayConfig.logDir, name, date, logId); builder.redirectOutput(new File(stdout)); builder.redirectError(new File(stderr)); } @@ -162,7 +140,13 @@ public class RunManager { throw new RuntimeException("Failed to start " + name); } processes.add(Pair.of(name, p)); - LOGGER.info("{} process started", name); + if (LOGGER.isInfoEnabled()) { + String message = String.format("%s process started.", name); + if (rayConfig.redirectOutput) { + message += String.format(" Logs are redirected to %s and %s.", stdout, stderr); + } + LOGGER.info(message); + } } /** @@ -194,7 +178,7 @@ public class RunManager { rayConfig.headRedisPort, rayConfig.headRedisPassword, null); rayConfig.setRedisAddress(primary); try (Jedis client = new Jedis("127.0.0.1", rayConfig.headRedisPort)) { - if (!StringUtil.isNullOrEmpty(rayConfig.headRedisPassword)) { + if (!Strings.isNullOrEmpty(rayConfig.headRedisPassword)) { client.auth(rayConfig.headRedisPassword); } client.set("UseRaylet", "1"); @@ -214,30 +198,36 @@ public class RunManager { } private String startRedisInstance(String ip, int port, String password, Integer shard) { - List command = Lists.newArrayList( - // The redis-server executable file. - getTempFile("/redis-server").getAbsolutePath(), - "--protected-mode", - "no", - "--port", - String.valueOf(port), - "--loglevel", - "warning", - "--loadmodule", - // The redis module file. - getTempFile("/libray_redis_module.so").getAbsolutePath() - ); + try (FileUtil.TempFile redisServerFile = FileUtil.getTempFileFromResource("redis-server")) { + try (FileUtil.TempFile redisModuleFile = FileUtil.getTempFileFromResource( + "libray_redis_module.so")) { + redisServerFile.getFile().setExecutable(true); + List command = Lists.newArrayList( + // The redis-server executable file. + redisServerFile.getFile().getAbsolutePath(), + "--protected-mode", + "no", + "--port", + String.valueOf(port), + "--loglevel", + "warning", + "--loadmodule", + // The redis module file. + redisModuleFile.getFile().getAbsolutePath() + ); - if (!StringUtil.isNullOrEmpty(password)) { - command.add("--requirepass "); - command.add(password); + if (!Strings.isNullOrEmpty(password)) { + command.add("--requirepass "); + command.add(password); + } + + String name = shard == null ? "redis" : "redis-" + shard; + startProcess(command, null, name); + } } - String name = shard == null ? "redis" : "redis-" + shard; - startProcess(command, null, name); - try (Jedis client = new Jedis("127.0.0.1", port)) { - if (!StringUtil.isNullOrEmpty(password)) { + if (!Strings.isNullOrEmpty(password)) { client.auth(password); } @@ -256,32 +246,34 @@ public class RunManager { Math.min(rayConfig.resources.getOrDefault("CPU", 0.0).intValue(), hardwareConcurrency)); String redisPasswordOption = ""; - if (!StringUtil.isNullOrEmpty(rayConfig.headRedisPassword)) { + if (!Strings.isNullOrEmpty(rayConfig.headRedisPassword)) { redisPasswordOption = rayConfig.headRedisPassword; } // See `src/ray/raylet/main.cc` for the meaning of each parameter. - List command = ImmutableList.of( - // The raylet executable file. - getTempFile("/raylet").getAbsolutePath(), - String.format("--raylet_socket_name=%s", rayConfig.rayletSocketName), - String.format("--store_socket_name=%s", rayConfig.objectStoreSocketName), - String.format("--object_manager_port=%d", 0), // The object manager port. - String.format("--node_manager_port=%d", 0), // The node manager port. - String.format("--node_ip_address=%s", rayConfig.nodeIp), - String.format("--redis_address=%s", rayConfig.getRedisIp()), - String.format("--redis_port=%d", rayConfig.getRedisPort()), - String.format("--num_initial_workers=%d", 0), // number of initial workers - String.format("--maximum_startup_concurrency=%d", maximumStartupConcurrency), - String.format("--static_resource_list=%s", - ResourceUtil.getResourcesStringFromMap(rayConfig.resources)), - String.format("--config_list=%s", String.join(",", rayConfig.rayletConfigParameters)), - String.format("--python_worker_command=%s", buildPythonWorkerCommand()), - String.format("--java_worker_command=%s", buildWorkerCommandRaylet()), - String.format("--redis_password=%s", redisPasswordOption) - ); + try (FileUtil.TempFile rayletFile = FileUtil.getTempFileFromResource("raylet")) { + rayletFile.getFile().setExecutable(true); + List command = ImmutableList.of( + rayletFile.getFile().getAbsolutePath(), + String.format("--raylet_socket_name=%s", rayConfig.rayletSocketName), + String.format("--store_socket_name=%s", rayConfig.objectStoreSocketName), + String.format("--object_manager_port=%d", 0), // The object manager port. + String.format("--node_manager_port=%d", 0), // The node manager port. + String.format("--node_ip_address=%s", rayConfig.nodeIp), + String.format("--redis_address=%s", rayConfig.getRedisIp()), + String.format("--redis_port=%d", rayConfig.getRedisPort()), + String.format("--num_initial_workers=%d", 0), // number of initial workers + String.format("--maximum_startup_concurrency=%d", maximumStartupConcurrency), + String.format("--static_resource_list=%s", + ResourceUtil.getResourcesStringFromMap(rayConfig.resources)), + String.format("--config_list=%s", String.join(",", rayConfig.rayletConfigParameters)), + String.format("--python_worker_command=%s", buildPythonWorkerCommand()), + String.format("--java_worker_command=%s", buildWorkerCommandRaylet()), + String.format("--redis_password=%s", redisPasswordOption) + ); - startProcess(command, null, "raylet"); + startProcess(command, null, "raylet"); + } } private String concatPath(Stream stream) { @@ -324,7 +316,7 @@ public class RunManager { cmd.add("-Dray.redis.address=" + rayConfig.getRedisAddress()); // redis password - if (!StringUtil.isNullOrEmpty(rayConfig.headRedisPassword)) { + if (!Strings.isNullOrEmpty(rayConfig.headRedisPassword)) { cmd.add("-Dray.redis.password=" + rayConfig.headRedisPassword); } @@ -341,15 +333,19 @@ public class RunManager { } private void startObjectStore() { - List command = ImmutableList.of( - // The plasma store executable file. - getTempFile("/plasma_store_server").getAbsolutePath(), - "-s", - rayConfig.objectStoreSocketName, - "-m", - rayConfig.objectStoreSize.toString() - ); - startProcess(command, null, "plasma_store"); + try (FileUtil.TempFile plasmaStoreFile = FileUtil + .getTempFileFromResource("plasma_store_server")) { + plasmaStoreFile.getFile().setExecutable(true); + List command = ImmutableList.of( + // The plasma store executable file. + plasmaStoreFile.getFile().getAbsolutePath(), + "-s", + rayConfig.objectStoreSocketName, + "-m", + rayConfig.objectStoreSize.toString() + ); + startProcess(command, null, "plasma_store"); + } } private String buildPythonWorkerCommand() { diff --git a/java/runtime/src/main/java/org/ray/runtime/util/FileUtil.java b/java/runtime/src/main/java/org/ray/runtime/util/FileUtil.java index 16f7237c5..c65976330 100644 --- a/java/runtime/src/main/java/org/ray/runtime/util/FileUtil.java +++ b/java/runtime/src/main/java/org/ray/runtime/util/FileUtil.java @@ -1,134 +1,68 @@ package org.ray.runtime.util; +import com.google.common.base.Preconditions; import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileWriter; import java.io.IOException; +import java.io.InputStream; import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Scanner; +import java.nio.file.StandardCopyOption; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class FileUtil { - public static String getFilename(String logPath) { - if (logPath != null && !logPath.isEmpty()) { - int lastPos = logPath.lastIndexOf('/'); - if (lastPos != -1) { - return logPath.substring(lastPos + 1); - } else { - return logPath; + private static final Logger LOGGER = LoggerFactory.getLogger(FileUtil.class); + + /** + * Represents a temp file. + * + * This class implements the `AutoCloseable` interface. It can be used in a `try-with-resource` + * block. When exiting the block, the temp file will be automatically removed. + */ + public static class TempFile implements AutoCloseable { + + File file; + + TempFile(File file) { + this.file = file; + } + + public File getFile() { + return file; + } + + @Override + public void close() { + if (!file.delete()) { + LOGGER.warn("Couldn't delete temp file {}", file.getAbsolutePath()); } } - - return null; - } - - public static boolean deleteFile(String filePath) { - File file = new File(filePath); - if (!file.exists()) { - return true; - } else { - if (file.isFile()) { - return file.delete(); - } else { - for (File f : file.listFiles()) { - deleteFile(f.getAbsolutePath()); - } - return file.delete(); - } - } - } - - public static void mkDir(File dir) { - if (dir.exists()) { - return; - } - if (dir.getParentFile().exists()) { - dir.mkdir(); - } else { - mkDir(dir.getParentFile()); - dir.mkdir(); - } - } - - public static void mkDirAndFile(File file) throws IOException { - if (file.exists()) { - return; - } - if (!file.getParentFile().exists()) { - mkDir(file.getParentFile()); - } - file.createNewFile(); - } - - public static String readResourceFile(String fileName) throws FileNotFoundException { - ClassLoader classLoader = FileUtil.class.getClassLoader(); - File file = new File(classLoader.getResource(fileName).getFile()); - StringBuilder result = new StringBuilder(); - try (Scanner scanner = new Scanner(file)) { - - //Get file from resources folder - - while (scanner.hasNextLine()) { - String line = scanner.nextLine(); - result.append(line).append("\n"); - } - return result.toString(); - } - - } - - public static void overrideFile(String file, String str) throws IOException { - try (FileWriter fw = new FileWriter(file)) { - fw.write(str); - } - } - - public static boolean createDir(String dirName, boolean failIfExist) { - File wdir = new File(dirName); - if (wdir.isFile()) { - return false; - } - - if (!wdir.exists()) { - wdir.mkdirs(); - } else if (failIfExist) { - return false; - } - - return true; - } - - public static void bytesToFile(byte[] bytes, String name) throws IOException { - Path path = Paths.get(name); - Files.write(path, bytes); - } - - public static byte[] fileToBytes(String name) throws IOException { - Path path = Paths.get(name); - return Files.readAllBytes(path); } /** - * If the given string is the empty string, then the result is the current directory. + * Get a temp file from resource. * - * @param rawDir a path in any legal form, such as a relative path - * @return the absolute and unique path in String + * @param resourceFileName File name. + * @return A `TempFile` object. */ - public static String getCanonicalDirectory(final String rawDir) throws IOException { - String dir = rawDir.length() == 0 ? "." : rawDir; - - // create working dir if necessary - File dd = new File(dir); - if (!dd.exists()) { - dd.mkdirs(); + public static TempFile getTempFileFromResource(String resourceFileName) { + File file; + try { + file = File.createTempFile(resourceFileName, ""); + } catch (IOException e) { + throw new RuntimeException("Couldn't create temp file " + resourceFileName, e); } - if (!dir.startsWith("/")) { - dir = dd.getCanonicalPath(); + try (InputStream in = FileUtil.class.getResourceAsStream("/" + resourceFileName)) { + Preconditions.checkNotNull(in, "{} doesn't exist.", resourceFileName); + Files.copy(in, Paths.get(file.getCanonicalPath()), StandardCopyOption.REPLACE_EXISTING); + + } catch (IOException e) { + throw new RuntimeException("Couldn't get temp file from resource " + resourceFileName, e); } - return dir; + return new TempFile(file); } } + diff --git a/java/runtime/src/main/java/org/ray/runtime/util/NetworkUtil.java b/java/runtime/src/main/java/org/ray/runtime/util/NetworkUtil.java index 40d0860a8..23875aba1 100644 --- a/java/runtime/src/main/java/org/ray/runtime/util/NetworkUtil.java +++ b/java/runtime/src/main/java/org/ray/runtime/util/NetworkUtil.java @@ -1,5 +1,6 @@ package org.ray.runtime.util; +import com.google.common.base.Strings; import java.io.IOException; import java.net.DatagramSocket; import java.net.Inet6Address; @@ -22,7 +23,7 @@ public class NetworkUtil { if (!current.isUp() || current.isLoopback() || current.isVirtual()) { continue; } - if (!StringUtil.isNullOrEmpty(interfaceName) && !interfaceName + if (!Strings.isNullOrEmpty(interfaceName) && !interfaceName .equals(current.getDisplayName())) { continue; } diff --git a/java/runtime/src/main/java/org/ray/runtime/util/Sha1Digestor.java b/java/runtime/src/main/java/org/ray/runtime/util/Sha1Digestor.java deleted file mode 100644 index 761e7192f..000000000 --- a/java/runtime/src/main/java/org/ray/runtime/util/Sha1Digestor.java +++ /dev/null @@ -1,43 +0,0 @@ -package org.ray.runtime.util; - -import java.nio.ByteBuffer; -import java.security.MessageDigest; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class Sha1Digestor { - - private static final Logger LOGGER = LoggerFactory.getLogger(Sha1Digestor.class); - private static final ThreadLocal md = ThreadLocal.withInitial(() -> { - try { - return MessageDigest.getInstance("SHA1"); - } catch (Exception e) { - LOGGER.error("Cannot get SHA1 MessageDigest", e); - throw new RuntimeException("Cannot get SHA1 digest", e); - } - }); - - - private static final ThreadLocal longBuffer = ThreadLocal - .withInitial(() -> ByteBuffer.allocate(Long.SIZE / Byte.SIZE)); - - public static byte[] digest(byte[] src, long addIndex) { - MessageDigest dg = md.get(); - longBuffer.get().clear(); - dg.reset(); - - dg.update(src); - dg.update(longBuffer.get().putLong(addIndex).array()); - return dg.digest(); - } - - public static byte[] digest(String str) { - return digest(str.getBytes(StringUtil.UTF8)); - } - - public static byte[] digest(byte[] src) { - MessageDigest dg = md.get(); - dg.reset(); - return dg.digest(src); - } -} \ No newline at end of file diff --git a/java/runtime/src/main/java/org/ray/runtime/util/StringUtil.java b/java/runtime/src/main/java/org/ray/runtime/util/StringUtil.java deleted file mode 100644 index 42046c1f1..000000000 --- a/java/runtime/src/main/java/org/ray/runtime/util/StringUtil.java +++ /dev/null @@ -1,150 +0,0 @@ -package org.ray.runtime.util; - -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Vector; - -public class StringUtil { - - public static final Charset UTF8 = Charset.forName("UTF-8"); - - private static final char[] HEX_CHARS = "0123456789abcdef".toCharArray(); - - /** - * split. - * @param s input string - * @param splitters common splitters - * @param open open braces - * @param close close braces - * @return output array list - */ - public static Vector split(String s, String splitters, String open, String close) { - // The splits. - Vector split = new Vector<>(); - // The stack. - ArrayList stack = new ArrayList<>(); - - int lastPos = 0; - - // Walk the string. - for (int i = 0; i < s.length(); i++) { - // Get the char there. - char ch = s.charAt(i); - - // Is it an open brace? - int o = open.indexOf(ch); - - // Is it a close brace? - int c = close.indexOf(ch); - - // Is it a splitter? - int sp = splitters.indexOf(ch); - - if (stack.size() == 0 && sp >= 0) { - if (i == lastPos) { - ++lastPos; - continue; - } - - split.add(s.substring(lastPos, i)); - lastPos = i + 1; - } else if (o >= 0 && (c < 0 || stack.size() == 0)) { - // Its an open! Push it. - stack.add(new Start(o, i)); - } else if (c >= 0 && stack.size() > 0) { - // Pop (if matches). - int tosPos = stack.size() - 1; - Start tos = stack.get(tosPos); - // Does the brace match? - if (tos.brace == c) { - // Done with that one. - stack.remove(tosPos); - } - } - } - - if (lastPos < s.length()) { - split.add(s.substring(lastPos, s.length())); - } - - // build removal filter set - HashSet removals = new HashSet<>(); - for (int i = 0; i < splitters.length(); i++) { - removals.add(splitters.charAt(i)); - } - - for (int i = 0; i < open.length(); i++) { - removals.add(open.charAt(i)); - } - - for (int i = 0; i < close.length(); i++) { - removals.add(close.charAt(i)); - } - - // apply removal filter set - for (int i = 0; i < split.size(); i++) { - String cs = split.get(i); - - // remove heading chars - int j; - for (j = 0; j < cs.length(); j++) { - if (!removals.contains(cs.charAt(j))) { - break; - } - } - cs = cs.substring(j); - - // remove tail chars - for (j = cs.length() - 1; j >= 0; j--) { - if (!removals.contains(cs.charAt(j))) { - break; - } - } - cs = cs.substring(0, j + 1); - - // reset cs - split.set(i, cs); - } - - return split; - } - - public static boolean isNullOrEmpty(String s) { - return s == null || s.length() == 0; - } - - public static String mergeArray(T[] objs, String concatenator) { - StringBuilder sb = new StringBuilder(); - for (T obj : objs) { - sb.append(obj).append(concatenator); - } - return objs.length == 0 ? "" : sb.substring(0, sb.length() - concatenator.length()); - } - - public static String toHexHashString(byte[] id) { - StringBuilder sb = new StringBuilder(20); - assert (id.length == 20); - for (int i = 0; i < 20; i++) { - int val = id[i] & 0xff; - sb.append(HEX_CHARS[val >> 4]); - sb.append(HEX_CHARS[val & 0xf]); - } - return sb.toString(); - } - - // Holds the start of an element and which brace started it. - private static class Start { - - // The brace number from the braces string in use. - final int brace; - // The position in the string it was seen. - final int pos; - - // Constructor. - public Start(int brace, int pos) { - this.brace = brace; - this.pos = pos; - } - } -} diff --git a/java/runtime/src/main/java/org/ray/runtime/util/generator/RayCallGenerator.java b/java/runtime/src/main/java/org/ray/runtime/util/generator/RayCallGenerator.java index 95e649929..dc916fbf9 100644 --- a/java/runtime/src/main/java/org/ray/runtime/util/generator/RayCallGenerator.java +++ b/java/runtime/src/main/java/org/ray/runtime/util/generator/RayCallGenerator.java @@ -1,9 +1,11 @@ package org.ray.runtime.util.generator; +import java.io.File; import java.io.IOException; +import java.nio.charset.Charset; import java.util.ArrayList; import java.util.List; -import org.ray.runtime.util.FileUtil; +import org.apache.commons.io.FileUtils; /** * A util class that generates `RayCall.java`, which provides type-safe interfaces for `Ray.call` @@ -267,7 +269,7 @@ public class RayCallGenerator extends BaseGenerator { public static void main(String[] args) throws IOException { String path = System.getProperty("user.dir") + "/api/src/main/java/org/ray/api/RayCall.java"; - FileUtil.overrideFile(path, new RayCallGenerator().build()); + FileUtils.write(new File(path), new RayCallGenerator().build(), Charset.defaultCharset()); } } diff --git a/java/runtime/src/main/java/org/ray/runtime/util/generator/RayFuncGenerator.java b/java/runtime/src/main/java/org/ray/runtime/util/generator/RayFuncGenerator.java index 1098ab950..c272de04f 100644 --- a/java/runtime/src/main/java/org/ray/runtime/util/generator/RayFuncGenerator.java +++ b/java/runtime/src/main/java/org/ray/runtime/util/generator/RayFuncGenerator.java @@ -1,7 +1,9 @@ package org.ray.runtime.util.generator; +import java.io.File; import java.io.IOException; -import org.ray.runtime.util.FileUtil; +import java.nio.charset.Charset; +import org.apache.commons.io.FileUtils; /** * A util class that generates all the RayFuncX classes under org.ray.api.function package. @@ -58,10 +60,12 @@ public class RayFuncGenerator extends BaseGenerator { for (int i = 0; i <= MAX_PARAMETERS; i++) { // Functions that have return. String content = generator.generate(i, true); - FileUtil.overrideFile(root + "RayFunc" + i + ".java", content); + FileUtils.write(new File(root + "RayFunc" + i + ".java"), content, + Charset.defaultCharset()); // Functions that don't have return. content = generator.generate(i, false); - FileUtil.overrideFile(root + "RayFuncVoid" + i + ".java", content); + FileUtils.write(new File(root + "RayFuncVoid" + i + ".java"), content, + Charset.defaultCharset()); } } diff --git a/java/test/src/main/java/org/ray/api/test/WordCountTest.java b/java/test/src/main/java/org/ray/api/test/WordCountTest.java deleted file mode 100644 index 6efedf8e2..000000000 --- a/java/test/src/main/java/org/ray/api/test/WordCountTest.java +++ /dev/null @@ -1,74 +0,0 @@ -package org.ray.api.test; - -import java.io.FileNotFoundException; -import java.util.Arrays; -import java.util.List; -import org.ray.api.Ray; -import org.ray.api.RayObject; -import org.ray.api.annotation.RayRemote; -import org.ray.runtime.util.FileUtil; -import org.testng.Assert; - -/** - * given a directory of document files on each "machine", we would like to count the appearance of - * some word. - */ -public class WordCountTest { - - @RayRemote - public static List getMachineList() { - return Arrays.asList("A", "B", "C"); - } - - @RayRemote - public static Integer countWord(String machine, String word) { - - String log; - try { - log = FileUtil.readResourceFile("mapreduce/" + machine + ".log"); - } catch (FileNotFoundException e) { - e.printStackTrace(); - log = ""; - } - log = log.toLowerCase(); - int start = 0; - int count = 0; - while (true) { - if (start >= log.length()) { - break; - } - int index = log.indexOf(word, start); - if (index == -1) { - break; - } - start = index + word.length(); - count++; - } - return count; - } - - @RayRemote - public static Integer sum(Integer a, Integer/*TODO modify int to Integer in ASM hook*/ b) { - return a + b; - } - - //@Test - public void test() { - int sum = mapReduce(); - Assert.assertEquals(sum, 143); - } - - public int mapReduce() { - RayObject> machines = Ray.call(WordCountTest::getMachineList); - RayObject total = null; - for (String machine : machines.get()) { - RayObject wordcount = Ray.call(WordCountTest::countWord, machine, "ray"); - if (total == null) { - total = wordcount; - } else { - total = Ray.call(WordCountTest::sum, total, wordcount); - } - } - return total.get(); - } -}