wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-28 08:40:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Java] improve Java API module (#2783)

Browse Source 
API module (`ray/java/api` dir) includes all public APIs provided by Ray, it should be the only module that normal Ray users need to face.

The purpose of this PR to first improve the code quality of the API module. Subsequent PRs will improve other modules later. The changes of this PR include the following aspects: 
1) Only keep interfaces in api module, to hide implementation details from users and fix circular dependencies among modules.
2) Document everything in the api module. 
3) Improve naming.
4) Add more tests for API. 
5) Also fix/improve related code in other modules.
6) Remove some unused code.

(Apologize for posting such a large PR. Java worker code has been lack of maintenance for a while. There're a lot of code quality issues that need to be fixed. We plan to use a couple of large PRs to address them. After that, future changes will come in small PRs.)
This commit is contained in:
Hao Chen
2018-09-03 02:51:16 +08:00
committed by Robert Nishihara
parent 2691b3a11a
commit 3b0a2c4197
98 changed files with 2232 additions and 2158 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/runtime-dev/src/main/java/org/ray/core/impl/RayDevRuntime.java
+3 -63
View File
@@ -1,64 +1,14 @@
package org.ray.core.impl;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.concurrent.ConcurrentHashMap;
import org.ray.api.Ray;
import org.ray.api.RayActor;
import org.ray.api.RayRemote;
import org.ray.api.UniqueID;
import org.ray.core.RayRuntime;
import org.ray.core.UniqueIdHelper;
import org.ray.core.WorkerContext;
import org.ray.core.AbstractRayRuntime;
import org.ray.core.model.RayParameters;
import org.ray.spi.NopRemoteFunctionManager;
import org.ray.spi.PathConfig;
import org.ray.spi.RemoteFunctionManager;
import org.ray.spi.impl.MockLocalScheduler;
import org.ray.spi.impl.MockObjectStore;
import org.ray.util.exception.TaskExecutionException;
import org.ray.util.logger.RayLog;
public class RayDevRuntime extends RayRuntime {
private final ConcurrentHashMap<UniqueID, Object> actors = new ConcurrentHashMap<>();
protected RayDevRuntime() {
}
@RayRemote
private static byte[] createActor(String className) {
return ((RayDevRuntime) RayRuntime.getInstance()).createLocalActor(className);
}
private byte[] createLocalActor(String className) {
UniqueID taskId = WorkerContext.currentTask().taskId;
UniqueID actorId = UniqueIdHelper.computeReturnId(taskId, 0);
try {
Class<?> cls = Class.forName(className);
Constructor<?>[] cts = cls.getConstructors();
for (Constructor<?> ct : cts) {
System.err.println(ct.getName() + ", param count = " + ct.getParameterCount());
}
Object r = cls.getConstructor().newInstance();
actors.put(actorId, r);
RayLog.core.info("TaskId " + taskId + ", create actor ok " + actorId);
return actorId.getBytes();
} catch (ClassNotFoundException | InstantiationException | IllegalAccessException
| IllegalArgumentException | InvocationTargetException | NoSuchMethodException
| SecurityException e) {
// TODO Auto-generated catch block
e.printStackTrace();
String logInfo =
"TaskId " + taskId + " error at RayDevRuntime createLocalActor, create actor " + actorId
+ " for " + className + " failed";
System.err.println(logInfo + ", ex = " + e.getMessage());
RayLog.core.error(logInfo, e);
throw new TaskExecutionException(logInfo, e);
}
}
public class RayDevRuntime extends AbstractRayRuntime {
@Override
public void start(RayParameters params) {
@@ -71,17 +21,7 @@ public class RayDevRuntime extends RayRuntime {
}
@Override
public void cleanUp() {
public void shutdown() {
// nothing to do
}
@Override
public Object getLocalActor(UniqueID id) {
return actors.get(id);
}
@Override
public <T> RayActor<T> create(Class<T> cls) {
return new RayActor<>(Ray.call(RayDevRuntime::createActor, cls.getName()).getId());
}
}
java/runtime-dev/src/main/java/org/ray/spi/impl/MockLocalScheduler.java
+14 -16
View File
@@ -3,7 +3,7 @@ package org.ray.spi.impl;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.ray.api.UniqueID;
import org.ray.api.id.UniqueId;
import org.ray.core.LocalFunctionManager;
import org.ray.core.Worker;
import org.ray.spi.LocalSchedulerLink;
@@ -16,7 +16,7 @@ import org.ray.spi.model.TaskSpec;
*/
public class MockLocalScheduler implements LocalSchedulerLink {
private final Map<UniqueID, Map<UniqueID, TaskSpec>> waitTasks = new ConcurrentHashMap<>();
private final Map<UniqueId, Map<UniqueId, TaskSpec>> waitTasks = new ConcurrentHashMap<>();
private final MockObjectStore store;
private LocalFunctionManager functions = null;
@@ -29,8 +29,8 @@ public class MockLocalScheduler implements LocalSchedulerLink {
functions = mgr;
}
public void onObjectPut(UniqueID id) {
Map<UniqueID, TaskSpec> bucket = waitTasks.get(id);
public void onObjectPut(UniqueId id) {
Map<UniqueId, TaskSpec> bucket = waitTasks.get(id);
if (bucket != null) {
waitTasks.remove(id);
for (TaskSpec ts : bucket.values()) {
@@ -41,23 +41,21 @@ public class MockLocalScheduler implements LocalSchedulerLink {
@Override
public void submitTask(TaskSpec task) {
UniqueID id = isTaskReady(task);
UniqueId id = isTaskReady(task);
if (id == null) {
Worker.execute(task, functions);
} else {
Map<UniqueID, TaskSpec> bucket = waitTasks
Map<UniqueId, TaskSpec> bucket = waitTasks
.computeIfAbsent(id, id_ -> new ConcurrentHashMap<>());
bucket.put(id, task);
}
}
private UniqueID isTaskReady(TaskSpec spec) {
private UniqueId isTaskReady(TaskSpec spec) {
for (FunctionArg arg : spec.args) {
if (arg.ids != null) {
for (UniqueID id : arg.ids) {
if (!store.isObjectReady(id)) {
return id;
}
if (arg.id != null) {
if (!store.isObjectReady(arg.id)) {
return arg.id;
}
}
}
@@ -70,17 +68,17 @@ public class MockLocalScheduler implements LocalSchedulerLink {
}
@Override
public void markTaskPutDependency(UniqueID taskId, UniqueID objectId) {
public void markTaskPutDependency(UniqueId taskId, UniqueId objectId) {
}
@Override
public void reconstructObject(UniqueID objectId, boolean fetchOnly) {
public void reconstructObject(UniqueId objectId, boolean fetchOnly) {
}
@Override
public void reconstructObjects(List<UniqueID> objectIds, boolean fetchOnly) {
public void reconstructObjects(List<UniqueId> objectIds, boolean fetchOnly) {
}
@@ -90,7 +88,7 @@ public class MockLocalScheduler implements LocalSchedulerLink {
}
@Override
public UniqueID generateTaskId(UniqueID driverId, UniqueID parentTaskId, int taskIndex) {
public UniqueId generateTaskId(UniqueId driverId, UniqueId parentTaskId, int taskIndex) {
throw new RuntimeException("Not implemented here.");
}
java/runtime-dev/src/main/java/org/ray/spi/impl/MockObjectStore.java
+9 -9
View File
@@ -6,7 +6,7 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.arrow.plasma.ObjectStoreLink;
import org.ray.api.UniqueID;
import org.ray.api.id.UniqueId;
import org.ray.core.WorkerContext;
import org.ray.util.logger.RayLog;
@@ -15,8 +15,8 @@ import org.ray.util.logger.RayLog;
*/
public class MockObjectStore implements ObjectStoreLink {
private final Map<UniqueID, byte[]> data = new ConcurrentHashMap<>();
private final Map<UniqueID, byte[]> metadata = new ConcurrentHashMap<>();
private final Map<UniqueId, byte[]> data = new ConcurrentHashMap<>();
private final Map<UniqueId, byte[]> metadata = new ConcurrentHashMap<>();
private MockLocalScheduler scheduler = null;
@Override
@@ -26,7 +26,7 @@ public class MockObjectStore implements ObjectStoreLink {
.error(logPrefix() + "cannot put null: " + objectId + "," + Arrays.toString(value));
System.exit(-1);
}
UniqueID uniqueId = new UniqueID(objectId);
UniqueId uniqueId = new UniqueId(objectId);
data.put(uniqueId, value);
metadata.put(uniqueId, metadataValue);
@@ -37,10 +37,10 @@ public class MockObjectStore implements ObjectStoreLink {
@Override
public List<byte[]> get(byte[][] objectIds, int timeoutMs, boolean isMetadata) {
final Map<UniqueID, byte[]> dataMap = isMetadata ? metadata : data;
final Map<UniqueId, byte[]> dataMap = isMetadata ? metadata : data;
ArrayList<byte[]> rets = new ArrayList<>(objectIds.length);
for (byte[] objId : objectIds) {
UniqueID uniqueId = new UniqueID(objId);
UniqueId uniqueId = new UniqueId(objId);
RayLog.core.info(logPrefix() + " is notified for objectid " + uniqueId);
rets.add(dataMap.get(uniqueId));
}
@@ -52,7 +52,7 @@ public class MockObjectStore implements ObjectStoreLink {
ArrayList<byte[]> rets = new ArrayList<>();
for (byte[] objId : objectIds) {
//tod test
if (data.containsKey(new UniqueID(objId))) {
if (data.containsKey(new UniqueId(objId))) {
rets.add(objId);
}
}
@@ -82,7 +82,7 @@ public class MockObjectStore implements ObjectStoreLink {
@Override
public boolean contains(byte[] objectId) {
return data.containsKey(new UniqueID(objectId));
return data.containsKey(new UniqueId(objectId));
}
private String logPrefix() {
@@ -99,7 +99,7 @@ public class MockObjectStore implements ObjectStoreLink {
return stes[k].getFileName() + ":" + stes[k].getLineNumber();
}
public boolean isObjectReady(UniqueID id) {
public boolean isObjectReady(UniqueId id) {
return data.containsKey(id);
}