[Java] Build Java code with Bazel (#4284)

java/runtime/src/main/java/org/ray/runtime/RayNativeRuntime.java

@@ -3,6 +3,7 @@ package org.ray.runtime;
 import com.google.common.base.Preconditions;
 import com.google.common.base.Strings;
 import java.io.File;
+import java.io.IOException;
 import java.io.InputStream;
 import java.lang.reflect.Field;
 import java.nio.ByteBuffer;
@@ -30,7 +31,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * native runtime for local box and cluster run.
+ * Native runtime for cluster mode.
  */
 public final class RayNativeRuntime extends AbstractRayRuntime {
 
@@ -46,41 +47,9 @@ public final class RayNativeRuntime extends AbstractRayRuntime {
   private List<RedisClient> redisClients;
   private RunManager manager = null;
 
-  public RayNativeRuntime(RayConfig rayConfig) {
-    super(rayConfig);
-  }
-
-  private void resetLibraryPath() {
-    String path = System.getProperty("java.library.path");
-    if (Strings.isNullOrEmpty(path)) {
-      path = "";
-    } else {
-      path += ":";
-    }
-
-    path += String.join(":", rayConfig.libraryPath);
-
-    // This is a hack to reset library path at runtime,
-    // see https://stackoverflow.com/questions/15409223/.
-    System.setProperty("java.library.path", path);
-    //set sys_paths to null so that java.library.path will be re-evalueted next time it is needed
-    final Field sysPathsField;
+  static {
     try {
-      sysPathsField = ClassLoader.class.getDeclaredField("sys_paths");
-      sysPathsField.setAccessible(true);
-      sysPathsField.set(null, null);
-    } catch (NoSuchFieldException | IllegalAccessException e) {
-      e.printStackTrace();
-      LOGGER.error("Failed to set library path.", e);
-    }
-  }
-
-  @Override
-  public void start() throws Exception {
-    try {
-      // Reset library path at runtime.
-      resetLibraryPath();
-
+      LOGGER.debug("Loading native libraries.");
       // Load native libraries.
       String[] libraries = new String[]{"raylet_library_java", "plasma_java"};
       for (String library : libraries) {
@@ -93,10 +62,47 @@ public final class RayNativeRuntime extends AbstractRayRuntime {
         Files.copy(in, Paths.get(file.getAbsolutePath()), StandardCopyOption.REPLACE_EXISTING);
         System.load(file.getAbsolutePath());
       }
-    } catch (Exception e) {
-      LOGGER.error("Failed to load native libraries.", e);
-      throw e;
+      LOGGER.debug("Native libraries loaded.");
+    } catch (IOException e) {
+      throw new RuntimeException("Couldn't load native libraries.", e);
     }
+  }
+
+  public RayNativeRuntime(RayConfig rayConfig) {
+    super(rayConfig);
+  }
+
+  private void resetLibraryPath() {
+    if (rayConfig.libraryPath.isEmpty()) {
+      return;
+    }
+
+    String path = System.getProperty("java.library.path");
+    if (Strings.isNullOrEmpty(path)) {
+      path = "";
+    } else {
+      path += ":";
+    }
+    path += String.join(":", rayConfig.libraryPath);
+
+    // This is a hack to reset library path at runtime,
+    // see https://stackoverflow.com/questions/15409223/.
+    System.setProperty("java.library.path", path);
+    // Set sys_paths to null so that java.library.path will be re-evaluated next time it is needed.
+    final Field sysPathsField;
+    try {
+      sysPathsField = ClassLoader.class.getDeclaredField("sys_paths");
+      sysPathsField.setAccessible(true);
+      sysPathsField.set(null, null);
+    } catch (NoSuchFieldException | IllegalAccessException e) {
+      LOGGER.error("Failed to set library path.", e);
+    }
+  }
+
+  @Override
+  public void start() {
+    // Reset library path at runtime.
+    resetLibraryPath();
 
     if (rayConfig.getRedisAddress() == null) {
       manager = new RunManager(rayConfig);

java/runtime/src/main/java/org/ray/runtime/functionmanager/FunctionManager.java

@@ -1,9 +1,14 @@
 package org.ray.runtime.functionmanager;
 
+import com.google.common.base.Strings;
+import java.io.File;
 import java.lang.invoke.SerializedLambda;
 import java.lang.reflect.Constructor;
 import java.lang.reflect.Executable;
 import java.lang.reflect.Method;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -15,7 +20,6 @@ import org.apache.commons.lang3.tuple.Pair;
 import org.objectweb.asm.Type;
 import org.ray.api.function.RayFunc;
 import org.ray.api.id.UniqueId;
-import org.ray.runtime.util.JarLoader;
 import org.ray.runtime.util.LambdaUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -49,8 +53,8 @@ public class FunctionManager {
   /**
    * Construct a FunctionManager with the specified driver resource path.
    *
-   * @param driverResourcePath The specified driver resource that
-   *     can store the driver's resources.
+   * @param driverResourcePath The specified driver resource that can store the driver's
+   *     resources.
    */
   public FunctionManager(String driverResourcePath) {
     this.driverResourcePath = driverResourcePath;
@@ -71,7 +75,7 @@ public class FunctionManager {
       final String methodName = serializedLambda.getImplMethodName();
       final String typeDescriptor = serializedLambda.getImplMethodSignature();
       functionDescriptor = new JavaFunctionDescriptor(className, methodName, typeDescriptor);
-      RAY_FUNC_CACHE.get().put(func.getClass(),functionDescriptor);
+      RAY_FUNC_CACHE.get().put(func.getClass(), functionDescriptor);
     }
     return getFunction(driverId, functionDescriptor);
   }
@@ -86,15 +90,18 @@ public class FunctionManager {
   public RayFunction getFunction(UniqueId driverId, JavaFunctionDescriptor functionDescriptor) {
     DriverFunctionTable driverFunctionTable = driverFunctionTables.get(driverId);
     if (driverFunctionTable == null) {
-      String resourcePath = driverResourcePath + "/" + driverId.toString() + "/";
       ClassLoader classLoader;
-
-      if (driverResourcePath != null && !driverResourcePath.isEmpty()) {
-        classLoader = JarLoader.loadJars(resourcePath, false);
-        LOGGER.info("Succeeded to load driver({}) resource. Resource path is {}",
-            driverId, resourcePath);
-      } else {
+      if (Strings.isNullOrEmpty(driverResourcePath)) {
         classLoader = getClass().getClassLoader();
+      } else {
+        File resourceDir = new File(driverResourcePath + "/" + driverId.toString() + "/");
+        try {
+          classLoader = new URLClassLoader(new URL[]{resourceDir.toURI().toURL()});
+        } catch (MalformedURLException e) {
+          throw new RuntimeException(e);
+        }
+        LOGGER.debug("Resource loaded for driver {} from path {}.", driverId,
+            resourceDir.getAbsolutePath());
       }
 
       driverFunctionTable = new DriverFunctionTable(classLoader);

java/runtime/src/main/java/org/ray/runtime/runner/RunManager.java

@@ -1,6 +1,7 @@
 package org.ray.runtime.runner;
 
 import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Lists;
 import java.io.File;
@@ -94,6 +95,7 @@ public class RunManager {
         file = File.createTempFile(fileName, "");
         file.deleteOnExit();
         try (InputStream in = RunManager.class.getResourceAsStream(fileName)) {
+          Preconditions.checkNotNull(in, "{} doesn't exist.", fileName);
           Files.copy(in, Paths.get(file.getCanonicalPath()), StandardCopyOption.REPLACE_EXISTING);
         }
         file.setExecutable(true);

java/runtime/src/main/java/org/ray/runtime/util/JarLoader.java

@@ -1,92 +0,0 @@
-package org.ray.runtime.util;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Enumeration;
-import java.util.List;
-import java.util.jar.JarEntry;
-import java.util.jar.JarFile;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.io.filefilter.DirectoryFileFilter;
-import org.apache.commons.io.filefilter.RegexFileFilter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * load and unload jars from a dir.
- */
-public class JarLoader {
-
-  private static final Logger LOGGER = LoggerFactory.getLogger(JarLoader.class);
-
-  public static URLClassLoader loadJars(String dir, boolean explicitLoad) {
-    // get all jars
-    Collection<File> jars = FileUtils.listFiles(
-        new File(dir),
-        new RegexFileFilter(".*\\.jar"),
-        DirectoryFileFilter.DIRECTORY
-    );
-    return loadJar(jars, explicitLoad);
-  }
-
-  public static void unloadJars(ClassLoader loader) {
-    // now do nothing, if no ref to the loader and loader's class.
-    // they would be gc.
-  }
-
-  private static URLClassLoader loadJar(Collection<File> appJars, boolean explicitLoad) {
-    List<JarFile> jars = new ArrayList<>();
-    List<URL> urls = new ArrayList<>();
-
-    for (File appJar : appJars) {
-      try {
-        LOGGER.info("succeeded to load jar {}.", appJar.getAbsolutePath());
-        JarFile jar = new JarFile(appJar.getAbsolutePath());
-        jars.add(jar);
-        urls.add(appJar.toURI().toURL());
-      } catch (IOException e) {
-        throw new RuntimeException(
-            "invalid app jar path: " + appJar.getAbsolutePath() + ", load failed with exception",
-            e);
-      }
-    }
-
-    URLClassLoader cl = URLClassLoader.newInstance(urls.toArray(new URL[urls.size()]));
-
-    if (!explicitLoad) {
-      return cl;
-    }
-    for (JarFile jar : jars) {
-      try {
-        Enumeration<JarEntry> e = jar.entries();
-        while (e.hasMoreElements()) {
-          JarEntry je = e.nextElement();
-          if (je.isDirectory() || !je.getName().endsWith(".class")) {
-            continue;
-          }
-
-          String className = classNameOfJarEntry(je);
-          className = className.replace('/', '.');
-          try {
-            Class.forName(className, true, cl);
-          } catch (ClassNotFoundException e1) {
-            e1.printStackTrace();
-          }
-        }
-      } finally {
-        IOUtils.closeQuietly(jar);
-      }
-    }
-    return cl;
-  }
-
-  private static String classNameOfJarEntry(JarEntry je) {
-    return je.getName().substring(0, je.getName().length() - ".class".length());
-  }
-
-}

java/runtime/src/test/java/org/ray/runtime/functionmanager/FunctionManagerTest.java

@@ -3,8 +3,10 @@ package org.ray.runtime.functionmanager;
 import java.io.File;
 import java.nio.file.Files;
 import java.nio.file.Paths;
-import java.nio.file.StandardCopyOption;
 import java.util.Map;
+import javax.tools.JavaCompiler;
+import javax.tools.ToolProvider;
+import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.lang3.tuple.Pair;
 import org.ray.api.annotation.RayRemote;
@@ -115,27 +117,40 @@ public class FunctionManagerTest {
         ImmutablePair.of(barConstructorDescriptor.name, barConstructorDescriptor.typeDescriptor)));
   }
 
-  //TODO(qwang): This is an integration test case, and we should move it to test folder in the future.
   @Test
-  public void testGetFunctionFromLocalResource() throws Exception{
-    UniqueId driverId = UniqueId.fromHexString("0123456789012345678901234567890123456789");
+  public void testGetFunctionFromLocalResource() throws Exception {
+    UniqueId driverId = UniqueId.randomId();
+    final String resourcePath = FileUtils.getTempDirectoryPath() + "/ray_test_resources";
+    final String driverResourcePath = resourcePath + "/" + driverId.toString();
+    File driverResourceDir = new File(driverResourcePath);
+    FileUtils.deleteQuietly(driverResourceDir);
+    driverResourceDir.mkdirs();
+    driverResourceDir.deleteOnExit();
 
-    //TODO(qwang): We should use a independent app demo instead of `tutorial`.
-    final String resourcePath = "/tmp/ray/test/resource";
-    final String srcJarPath = System.getProperty("user.dir") +
-                                  "/../tutorial/target/ray-tutorial-0.1-SNAPSHOT.jar";
-    final String destJarPath = resourcePath + "/" + driverId.toString() +
-                                   "/ray-tutorial-0.1-SNAPSHOT.jar";
+    String demoJavaFile = "";
+    demoJavaFile += "public class DemoApp {\n";
+    demoJavaFile += "  public static String hello() {\n";
+    demoJavaFile += "    return \"hello\";\n";
+    demoJavaFile += "  }\n";
+    demoJavaFile += "}";
 
-    File file = new File(resourcePath + "/" + driverId.toString());
-    file.mkdirs();
-    Files.copy(Paths.get(srcJarPath), Paths.get(destJarPath), StandardCopyOption.REPLACE_EXISTING);
+    // Write the demo java file to the driver resource path.
+    String javaFilePath = driverResourcePath + "/DemoApp.java";
+    Files.write(Paths.get(javaFilePath), demoJavaFile.getBytes());
 
+    // Compile the java file.
+    JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
+    int result = compiler.run(null, null, null, "-d", driverResourcePath, javaFilePath);
+    if (result != 0) {
+      throw new RuntimeException("Couldn't compile Demo.java.");
+    }
+
+    // Test loading the function.
+    JavaFunctionDescriptor descriptor = new JavaFunctionDescriptor(
+        "DemoApp", "hello", "()Ljava/lang/String;");
     final FunctionManager functionManager = new FunctionManager(resourcePath);
-    JavaFunctionDescriptor sayHelloDescriptor = new JavaFunctionDescriptor("org.ray.exercise.Exercise02",
-        "sayHello", "()Ljava/lang/String;");
-    RayFunction func = functionManager.getFunction(driverId, sayHelloDescriptor);
-    Assert.assertEquals(func.getFunctionDescriptor(), sayHelloDescriptor);
+    RayFunction func = functionManager.getFunction(driverId, descriptor);
+    Assert.assertEquals(func.getFunctionDescriptor(), descriptor);
   }
 
 }