wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-28 19:33:06 +08:00
Code Issues Packages Projects Releases Wiki Activity

Remove dependencies from TaskExecutionSpecification (#5166)

Browse Source
This commit is contained in:
Edward Oakes
2019-07-15 18:15:21 -07:00
committed by Philipp Moritz
parent fd71ffde2f
commit e5be5fd46d
33 changed files with 136 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/runtime/src/main/java/org/ray/runtime/AbstractRayRuntime.java
+14 -9
View File
@@ -244,7 +244,7 @@ public abstract class AbstractRayRuntime implements RayRuntime {
@Override
public RayObject call(RayFunc func, Object[] args, CallOptions options) {
TaskSpec spec = createTaskSpec(func, null, RayActorImpl.NIL, args, false, options);
TaskSpec spec = createTaskSpec(func, null, RayActorImpl.NIL, args, false, false, options);
rayletClient.submitTask(spec);
return new RayObjectImpl(spec.returnIds[0]);
}
@@ -257,8 +257,7 @@ public abstract class AbstractRayRuntime implements RayRuntime {
RayActorImpl<?> actorImpl = (RayActorImpl) actor;
TaskSpec spec;
synchronized (actor) {
spec = createTaskSpec(func, null, actorImpl, args, false, null);
spec.getExecutionDependencies().add(((RayActorImpl) actor).getTaskCursor());
spec = createTaskSpec(func, null, actorImpl, args, false, true, null);
actorImpl.setTaskCursor(spec.returnIds[1]);
actorImpl.clearNewActorHandles();
}
@@ -271,7 +270,7 @@ public abstract class AbstractRayRuntime implements RayRuntime {
public <T> RayActor<T> createActor(RayFunc actorFactoryFunc,
Object[] args, ActorCreationOptions options) {
TaskSpec spec = createTaskSpec(actorFactoryFunc, null, RayActorImpl.NIL,
args, true, options);
args, true, false, options);
RayActorImpl<?> actor = new RayActorImpl(new UniqueId(spec.returnIds[0].getBytes()));
actor.increaseTaskCounter();
actor.setTaskCursor(spec.returnIds[0]);
@@ -293,7 +292,7 @@ public abstract class AbstractRayRuntime implements RayRuntime {
CallOptions options) {
checkPyArguments(args);
PyFunctionDescriptor desc = new PyFunctionDescriptor(moduleName, "", functionName);
TaskSpec spec = createTaskSpec(null, desc, RayPyActorImpl.NIL, args, false, options);
TaskSpec spec = createTaskSpec(null, desc, RayPyActorImpl.NIL, args, false, false, options);
rayletClient.submitTask(spec);
return new RayObjectImpl(spec.returnIds[0]);
}
@@ -306,8 +305,7 @@ public abstract class AbstractRayRuntime implements RayRuntime {
RayPyActorImpl actorImpl = (RayPyActorImpl) pyActor;
TaskSpec spec;
synchronized (pyActor) {
spec = createTaskSpec(null, desc, actorImpl, args, false, null);
spec.getExecutionDependencies().add(actorImpl.getTaskCursor());
spec = createTaskSpec(null, desc, actorImpl, args, false, true, null);
actorImpl.setTaskCursor(spec.returnIds[1]);
actorImpl.clearNewActorHandles();
}
@@ -320,7 +318,7 @@ public abstract class AbstractRayRuntime implements RayRuntime {
ActorCreationOptions options) {
checkPyArguments(args);
PyFunctionDescriptor desc = new PyFunctionDescriptor(moduleName, className, "__init__");
TaskSpec spec = createTaskSpec(null, desc, RayPyActorImpl.NIL, args, true, options);
TaskSpec spec = createTaskSpec(null, desc, RayPyActorImpl.NIL, args, true, false, options);
RayPyActorImpl actor = new RayPyActorImpl(spec.actorCreationId, moduleName, className);
actor.increaseTaskCounter();
actor.setTaskCursor(spec.returnIds[0]);
@@ -337,11 +335,12 @@ public abstract class AbstractRayRuntime implements RayRuntime {
* @param actor The actor handle. If the task is not an actor task, actor id must be NIL.
* @param args The arguments for the remote function.
* @param isActorCreationTask Whether this task is an actor creation task.
* @param isActorTask Whether this task is an actor task.
* @return A TaskSpec object.
*/
private TaskSpec createTaskSpec(RayFunc func, PyFunctionDescriptor pyFunctionDescriptor,
RayActorImpl<?> actor, Object[] args,
boolean isActorCreationTask, BaseTaskOptions taskOptions) {
boolean isActorCreationTask, boolean isActorTask, BaseTaskOptions taskOptions) {
Preconditions.checkArgument((func == null) != (pyFunctionDescriptor == null));
TaskId taskId = rayletClient.generateTaskId(workerContext.getCurrentJobId(),
@@ -382,6 +381,11 @@ public abstract class AbstractRayRuntime implements RayRuntime {
functionDescriptor = pyFunctionDescriptor;
}
ObjectId previousActorTaskDummyObjectId = ObjectId.NIL;
if (isActorTask) {
previousActorTaskDummyObjectId = actor.getTaskCursor();
}
return new TaskSpec(
workerContext.getCurrentJobId(),
taskId,
@@ -392,6 +396,7 @@ public abstract class AbstractRayRuntime implements RayRuntime {
actor.getId(),
actor.getHandleId(),
actor.increaseTaskCounter(),
previousActorTaskDummyObjectId,
actor.getNewActorHandles().toArray(new UniqueId[0]),
ArgumentsBuilder.wrap(args, language == TaskLanguage.PYTHON),
numReturns,
java/runtime/src/main/java/org/ray/runtime/raylet/MockRayletClient.java
+3 -4
View File
@@ -138,10 +138,9 @@ public class MockRayletClient implements RayletClient {
}
}
}
// Check whether task dependencies are ready.
for (ObjectId id : spec.getExecutionDependencies()) {
if (!store.isObjectReady(id)) {
unreadyObjects.add(id);
if (spec.isActorTask()) {
if (!store.isObjectReady(spec.previousActorTaskDummyObjectId)) {
unreadyObjects.add(spec.previousActorTaskDummyObjectId);
}
}
return unreadyObjects;
java/runtime/src/main/java/org/ray/runtime/raylet/RayletClientImpl.java
+10 -9
View File
@@ -81,12 +81,7 @@ public class RayletClientImpl implements RayletClient {
Preconditions.checkState(!spec.jobId.isNil());
byte[] taskSpec = convertTaskSpecToProtobuf(spec);
byte[] cursorId = null;
if (!spec.getExecutionDependencies().isEmpty()) {
//TODO(hchen): handle more than one dependencies.
cursorId = spec.getExecutionDependencies().get(0).getBytes();
}
nativeSubmitTask(client, cursorId, taskSpec);
nativeSubmitTask(client, taskSpec);
}
@Override
@@ -195,6 +190,7 @@ public class RayletClientImpl implements RayletClient {
// Parse ActorTaskSpec.
UniqueId actorId = UniqueId.NIL;
UniqueId actorHandleId = UniqueId.NIL;
ObjectId previousActorTaskDummyObjectId = ObjectId.NIL;
int actorCounter = 0;
if (taskSpec.getType() == Common.TaskType.ACTOR_TASK) {
Common.ActorTaskSpec actorTaskSpec = taskSpec.getActorTaskSpec();
@@ -202,14 +198,17 @@ public class RayletClientImpl implements RayletClient {
actorHandleId = UniqueId
.fromByteBuffer(actorTaskSpec.getActorHandleId().asReadOnlyByteBuffer());
actorCounter = (int) actorTaskSpec.getActorCounter();
previousActorTaskDummyObjectId = ObjectId.fromByteBuffer(
actorTaskSpec.getPreviousActorTaskDummyObjectId().asReadOnlyByteBuffer());
newActorHandles = actorTaskSpec.getNewActorHandlesList().stream()
.map(byteString -> UniqueId.fromByteBuffer(byteString.asReadOnlyByteBuffer()))
.toArray(UniqueId[]::new);
}
return new TaskSpec(jobId, taskId, parentTaskId, parentCounter, actorCreationId,
maxActorReconstructions, actorId, actorHandleId, actorCounter, newActorHandles,
args, numReturns, resources, TaskLanguage.JAVA, functionDescriptor, dynamicWorkerOptions);
maxActorReconstructions, actorId, actorHandleId, actorCounter,
previousActorTaskDummyObjectId, newActorHandles, args, numReturns, resources,
TaskLanguage.JAVA, functionDescriptor, dynamicWorkerOptions);
}
/**
@@ -275,6 +274,8 @@ public class RayletClientImpl implements RayletClient {
.setActorId(ByteString.copyFrom(task.actorId.getBytes()))
.setActorHandleId(ByteString.copyFrom(task.actorHandleId.getBytes()))
.setActorCreationDummyObjectId(ByteString.copyFrom(task.actorId.getBytes()))
.setPreviousActorTaskDummyObjectId(
ByteString.copyFrom(task.previousActorTaskDummyObjectId.getBytes()))
.setActorCounter(task.actorCounter)
.addAllNewActorHandles(newHandles)
);
@@ -310,7 +311,7 @@ public class RayletClientImpl implements RayletClient {
private static native long nativeInit(String localSchedulerSocket, byte[] workerId,
boolean isWorker, byte[] driverTaskId);
private static native void nativeSubmitTask(long client, byte[] cursorId, byte[] taskSpec)
private static native void nativeSubmitTask(long client, byte[] taskSpec)
throws RayException;
private static native byte[] nativeGetTask(long client) throws RayException;
java/runtime/src/main/java/org/ray/runtime/task/TaskSpec.java
+8 -10
View File
@@ -46,6 +46,9 @@ public class TaskSpec {
// Number of tasks that have been submitted to this actor so far.
public final int actorCounter;
// Object id returned by the previous task submitted to the same actor.
public final ObjectId previousActorTaskDummyObjectId;
// Task arguments.
public final UniqueId[] newActorHandles;
@@ -55,7 +58,7 @@ public class TaskSpec {
// number of return objects.
public final int numReturns;
// returns ids.
// Return ids.
public final ObjectId[] returnIds;
// The task's resource demands.
@@ -71,8 +74,6 @@ public class TaskSpec {
// is Python, the type is PyFunctionDescriptor.
private final FunctionDescriptor functionDescriptor;
private List<ObjectId> executionDependencies;
public boolean isActorTask() {
return !actorId.isNil();
}
@@ -91,6 +92,7 @@ public class TaskSpec {
UniqueId actorId,
UniqueId actorHandleId,
int actorCounter,
ObjectId previousActorTaskDummyObjectId,
UniqueId[] newActorHandles,
FunctionArg[] args,
int numReturns,
@@ -107,6 +109,7 @@ public class TaskSpec {
this.actorId = actorId;
this.actorHandleId = actorHandleId;
this.actorCounter = actorCounter;
this.previousActorTaskDummyObjectId = previousActorTaskDummyObjectId;
this.newActorHandles = newActorHandles;
this.args = args;
this.numReturns = numReturns;
@@ -128,7 +131,6 @@ public class TaskSpec {
Preconditions.checkArgument(false, "Unknown task language: {}.", language);
}
this.functionDescriptor = functionDescriptor;
this.executionDependencies = new ArrayList<>();
}
public JavaFunctionDescriptor getJavaFunctionDescriptor() {
@@ -141,10 +143,6 @@ public class TaskSpec {
return (PyFunctionDescriptor) functionDescriptor;
}
public List<ObjectId> getExecutionDependencies() {
return executionDependencies;
}
@Override
public String toString() {
return "TaskSpec{" +
@@ -157,14 +155,14 @@ public class TaskSpec {
", actorId=" + actorId +
", actorHandleId=" + actorHandleId +
", actorCounter=" + actorCounter +
", previousActorTaskDummyObjectId=" + previousActorTaskDummyObjectId +
", newActorHandles=" + Arrays.toString(newActorHandles) +
", args=" + Arrays.toString(args) +
", numReturns=" + numReturns +
", resources=" + resources +
", language=" + language +
", functionDescriptor=" + functionDescriptor +
", dynamicWorkerOptions=" + dynamicWorkerOptions +
", executionDependencies=" + executionDependencies +
", dynamicWorkerOptions=" + dynamicWorkerOptions +
'}';
}
}