[JavaWorker] Enable java worker support (#2094)

* Enable java worker support
--------------------------
This commit includes a tailored version of the Java worker implementation from Ant Financial.
The changes for build system, python module, src module and arrow are in other commits, this commit consists of the following modules:
 - java/api: Ray API definition
 - java/common: utilities
 - java/hook: binary rewrite of the Java byte-code for remote execution
 - java/runtime-common: common implementation of the runtime in worker
 - java/runtime-dev: a pure-java mock implementation of the runtime for fast development
 - java/runtime-native: a native implementation of the runtime
 - java/test: various tests

Contributors for this work:
 Guyang Song, Peng Cao, Senlin Zhu,Xiaoying Chu, Yiming Yu, Yujie Liu, Zhenyu Guo

* change the format of java help document from markdown to RST

* update the vesion of Arrow for java worker

* adapt the new version of plasma java client from arrow which use byte[] instead of custom type

* add java worker test to ci

* add the example module for better usage guide

java/runtime-dev/pom.xml

@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <parent>
+    <groupId>org.ray.parent</groupId>
+    <artifactId>ray-superpom</artifactId>
+    <version>1.0</version>
+  </parent>
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>org.ray</groupId>
+  <artifactId>ray-runtime-dev</artifactId>
+
+  <name>runtime-dev</name>
+  <description>runtime for app development</description>
+  <url></url>
+
+  <packaging>jar</packaging>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.ray</groupId>
+      <artifactId>ray-api</artifactId>
+      <version>1.0</version>
+    </dependency>
+    <dependency>
+      <groupId>org.ray</groupId>
+      <artifactId>ray-runtime-common</artifactId>
+      <version>1.0</version>
+    </dependency>
+    <dependency>
+      <groupId>de.ruedigermoeller</groupId>
+      <artifactId>fst</artifactId>
+      <version>2.47</version>
+    </dependency>
+
+    <!-- https://mvnrepository.com/artifact/com.github.davidmoten/flatbuffers-java -->
+    <dependency>
+      <groupId>com.github.davidmoten</groupId>
+      <artifactId>flatbuffers-java</artifactId>
+      <version>1.7.0.1</version>
+    </dependency>
+
+    <!-- https://mvnrepository.com/artifact/redis.clients/jedis -->
+    <dependency>
+      <groupId>redis.clients</groupId>
+      <artifactId>jedis</artifactId>
+      <version>2.8.0</version>
+    </dependency>
+  </dependencies>
+
+
+</project>

java/runtime-dev/src/main/java/org/ray/core/impl/RayDevRuntime.java

@@ -0,0 +1,85 @@
+package org.ray.core.impl;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.concurrent.ConcurrentHashMap;
+import org.ray.api.Ray;
+import org.ray.api.RayActor;
+import org.ray.api.RayRemote;
+import org.ray.api.UniqueID;
+import org.ray.core.RayRuntime;
+import org.ray.core.UniqueIdHelper;
+import org.ray.core.WorkerContext;
+import org.ray.core.model.RayParameters;
+import org.ray.spi.NopRemoteFunctionManager;
+import org.ray.spi.PathConfig;
+import org.ray.spi.RemoteFunctionManager;
+import org.ray.spi.impl.MockLocalScheduler;
+import org.ray.spi.impl.MockObjectStore;
+import org.ray.util.exception.TaskExecutionException;
+import org.ray.util.logger.RayLog;
+
+public class RayDevRuntime extends RayRuntime {
+
+  protected RayDevRuntime() {
+  }
+
+  @Override
+  public void start(RayParameters params) {
+    PathConfig pathConfig = new PathConfig(configReader);
+    RemoteFunctionManager rfm = new NopRemoteFunctionManager(params.driver_id);
+    MockObjectStore store = new MockObjectStore();
+    MockLocalScheduler scheduler = new MockLocalScheduler(store);
+    init(scheduler, store, rfm, pathConfig);
+    scheduler.setLocalFunctionManager(this.functions);
+  }
+
+  private final ConcurrentHashMap<UniqueID, Object> actors = new ConcurrentHashMap<>();
+
+  @RayRemote
+  private static byte[] createActor(String className) {
+    return ((RayDevRuntime) RayRuntime.getInstance()).createLocalActor(className);
+  }
+
+  private byte[] createLocalActor(String className) {
+    UniqueID taskId = WorkerContext.currentTask().taskId;
+    UniqueID actorId = UniqueIdHelper.taskComputeReturnId(taskId, 0, false);
+    try {
+      Class<?> cls = Class.forName(className);
+
+      Constructor<?>[] cts = cls.getConstructors();
+      for (Constructor<?> ct : cts) {
+        System.err.println(ct.getName() + ", param count = " + ct.getParameterCount());
+      }
+
+      Object r = cls.getConstructor().newInstance();
+      actors.put(actorId, r);
+      RayLog.core.info("TaskId " + taskId + ", create actor ok " + actorId);
+      return actorId.getBytes();
+    } catch (ClassNotFoundException | InstantiationException | IllegalAccessException | IllegalArgumentException | InvocationTargetException | NoSuchMethodException | SecurityException e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+      String logInfo =
+          "TaskId " + taskId + " error at RayDevRuntime createLocalActor, create actor " + actorId
+              + " for " + className + "  failed";
+      System.err.println(logInfo + ", ex = " + e.getMessage());
+      RayLog.core.error(logInfo, e);
+      throw new TaskExecutionException(logInfo, e);
+    }
+  }
+
+  @Override
+  public <T> RayActor<T> create(Class<T> cls) {
+    return new RayActor<>(Ray.call(RayDevRuntime::createActor, cls.getName()).getId());
+  }
+
+  @Override
+  public Object getLocalActor(UniqueID id) {
+    return actors.get(id);
+  }
+
+  @Override
+  public void cleanUp() {
+    // nothing to do
+  }
+}

java/runtime-dev/src/main/java/org/ray/spi/impl/MockLocalScheduler.java

@@ -0,0 +1,85 @@
+package org.ray.spi.impl;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import org.ray.api.UniqueID;
+import org.ray.core.LocalFunctionManager;
+import org.ray.core.Worker;
+import org.ray.spi.LocalSchedulerLink;
+import org.ray.spi.model.FunctionArg;
+import org.ray.spi.model.TaskSpec;
+
+/**
+ * A mock implementation of {@code org.ray.spi.LocalSchedulerLink}, which stores waiting tasks in a
+ * Map, and cooperates with a {@code org.ray.spi.impl.MockObjectStore}.
+ */
+public class MockLocalScheduler implements LocalSchedulerLink {
+
+  private final Map<UniqueID, Map<UniqueID, TaskSpec>> waitTasks_ = new ConcurrentHashMap<>();
+  private final MockObjectStore store_;
+  private LocalFunctionManager functions_ = null;
+
+  public MockLocalScheduler(MockObjectStore store) {
+    store_ = store;
+    store.registerScheduler(this);
+  }
+
+  public void setLocalFunctionManager(LocalFunctionManager mgr) {
+    functions_ = mgr;
+  }
+
+  public void onObjectPut(UniqueID id) {
+    Map<UniqueID, TaskSpec> bucket = waitTasks_.get(id);
+    if (bucket != null) {
+      waitTasks_.remove(id);
+      for (TaskSpec ts : bucket.values()) {
+        submitTask(ts);
+      }
+    }
+  }
+
+  private UniqueID isTaskReady(TaskSpec spec) {
+    for (FunctionArg arg : spec.args) {
+      if (arg.ids != null) {
+        for (UniqueID id : arg.ids) {
+          if (!store_.isObjectReady(id)) {
+            return id;
+          }
+        }
+      }
+    }
+    return null;
+  }
+
+  @Override
+  public void submitTask(TaskSpec task) {
+    UniqueID id = isTaskReady(task);
+    if (id == null) {
+      Worker.execute(task, functions_);
+    } else {
+      Map<UniqueID, TaskSpec> bucket = waitTasks_
+          .computeIfAbsent(id, id_ -> new ConcurrentHashMap<>());
+      bucket.put(id, task);
+    }
+  }
+
+  @Override
+  public TaskSpec getTaskTodo() {
+    throw new RuntimeException("invalid execution flow here");
+  }
+
+  @Override
+  public void markTaskPutDependency(UniqueID taskId, UniqueID objectId) {
+
+  }
+
+  @Override
+  public void reconstructObject(UniqueID objectId) {
+
+  }
+
+  @Override
+  public void notifyUnblocked() {
+
+  }
+}

java/runtime-dev/src/main/java/org/ray/spi/impl/MockObjectStore.java

@@ -0,0 +1,110 @@
+package org.ray.spi.impl;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.arrow.plasma.ObjectStoreLink;
+import org.ray.api.UniqueID;
+import org.ray.core.WorkerContext;
+import org.ray.util.logger.RayLog;
+
+/**
+ * A mock implementation of {@code org.ray.spi.ObjectStoreLink}, which use Map to store data.
+ */
+public class MockObjectStore implements ObjectStoreLink {
+
+  private final Map<UniqueID, byte[]> data_ = new ConcurrentHashMap<>();
+  private final Map<UniqueID, byte[]> metadata_ = new ConcurrentHashMap<>();
+  private MockLocalScheduler scheduler_ = null;
+
+  @Override
+  public void put(byte[] objectId, byte[] value, byte[] metadataValue) {
+    if (objectId == null || objectId.length == 0 || value == null) {
+      RayLog.core
+          .error(logPrefix() + "cannot put null: " + objectId + "," + Arrays.toString(value));
+      System.exit(-1);
+    }
+    UniqueID uniqueID = new UniqueID(objectId);
+    data_.put(uniqueID, value);
+    metadata_.put(uniqueID, metadataValue);
+
+    if (scheduler_ != null) {
+      scheduler_.onObjectPut(uniqueID);
+    }
+  }
+
+  @Override
+  public List<byte[]> get(byte[][] objectIds, int timeoutMs, boolean isMetadata) {
+    final Map<UniqueID, byte[]> dataMap = isMetadata ? metadata_ : data_;
+    ArrayList<byte[]> rets = new ArrayList<>(objectIds.length);
+    for (byte[] objId : objectIds) {
+      UniqueID uniqueID = new UniqueID(objId);
+      RayLog.core.info(logPrefix() + " is notified for objectid " + uniqueID);
+      rets.add(dataMap.get(uniqueID));
+    }
+    return rets;
+  }
+
+  private String logPrefix() {
+    return WorkerContext.currentTask().taskId + "-" + getUserTrace() + " -> ";
+  }
+
+  private String getUserTrace() {
+    StackTraceElement stes[] = Thread.currentThread().getStackTrace();
+    int k = 1;
+    while (stes[k].getClassName().startsWith("org.ray")
+        && !stes[k].getClassName().contains("test")) {
+      k++;
+    }
+    return stes[k].getFileName() + ":" + stes[k].getLineNumber();
+  }
+
+  @Override
+  public List<byte[]> wait(byte[][] objectIds, int timeoutMs, int numReturns) {
+    ArrayList<byte[]> rets = new ArrayList<>();
+    for (byte[] objId : objectIds) {
+      //tod test
+      if (data_.containsKey(new UniqueID(objId))) {
+        rets.add(objId);
+      }
+    }
+    return rets;
+  }
+
+  @Override
+  public byte[] hash(byte[] objectId) {
+    return null;
+  }
+
+  @Override
+  public void fetch(byte[][] objectIds) {
+
+  }
+
+  @Override
+  public long evict(long numBytes) {
+    return 0;
+  }
+
+  public boolean isObjectReady(UniqueID id) {
+    return data_.containsKey(id);
+  }
+
+  public void registerScheduler(MockLocalScheduler s) {
+    scheduler_ = s;
+  }
+
+  @Override
+  public void release(byte[] objectId) {
+    return;
+  }
+
+  @Override
+  public boolean contains(byte[] objectId) {
+
+    return data_.containsKey(new UniqueID(objectId));
+  }
+}