wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-29 01:59:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

[Core]Remove checkpoint table (#12235)

Browse Source 
* Delete an actor entry from node manager.

* Remove checkpoint table

* remote checkpoint interface

* remove checkpoint interface

* fix ExitActorTest

Co-authored-by: chaokunyang <shawn.ck.yang@gmail.com>
This commit is contained in:
SangBin Cho
2020-12-01 08:58:36 -08:00
committed by GitHub
parent 9021f15b2a
commit f6f3cc9af1
52 changed files with 6 additions and 2012 deletions
Download Patch File Download Diff File Expand all files Collapse all files
java/runtime/src/main/java/io/ray/runtime/gcs/GcsClient.java
-29
View File
@@ -2,7 +2,6 @@ package io.ray.runtime.gcs;
import com.google.common.base.Preconditions;
import com.google.protobuf.InvalidProtocolBufferException;
import io.ray.api.Checkpointable.Checkpoint;
import io.ray.api.id.ActorId;
import io.ray.api.id.BaseId;
import io.ray.api.id.JobId;
@@ -12,7 +11,6 @@ import io.ray.api.id.UniqueId;
import io.ray.api.placementgroup.PlacementGroup;
import io.ray.api.runtimecontext.NodeInfo;
import io.ray.runtime.generated.Gcs;
import io.ray.runtime.generated.Gcs.ActorCheckpointIdData;
import io.ray.runtime.generated.Gcs.GcsNodeInfo;
import io.ray.runtime.generated.Gcs.TablePrefix;
import io.ray.runtime.placementgroup.PlacementGroupUtils;
@@ -175,33 +173,6 @@ public class GcsClient {
return client.exists(key);
}
/**
* Get the available checkpoints for the given actor ID.
*/
public List<Checkpoint> getCheckpointsForActor(ActorId actorId) {
List<Checkpoint> checkpoints = new ArrayList<>();
byte[] result = globalStateAccessor.getActorCheckpointId(actorId);
if (result != null) {
ActorCheckpointIdData data = null;
try {
data = ActorCheckpointIdData.parseFrom(result);
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException("Received invalid protobuf data from GCS.");
}
UniqueId[] checkpointIds = new UniqueId[data.getCheckpointIdsCount()];
for (int i = 0; i < checkpointIds.length; i++) {
checkpointIds[i] = UniqueId
.fromByteBuffer(data.getCheckpointIds(i).asReadOnlyByteBuffer());
}
for (int i = 0; i < checkpointIds.length; i++) {
checkpoints.add(new Checkpoint(checkpointIds[i], data.getTimestamps(i)));
}
}
checkpoints.sort((x, y) -> Long.compare(y.timestamp, x.timestamp));
return checkpoints;
}
public JobId nextJobId() {
int jobCounter = (int) primary.incr("JobCounter".getBytes());
return JobId.fromInt(jobCounter);
java/runtime/src/main/java/io/ray/runtime/gcs/GlobalStateAccessor.java
-13
View File
@@ -125,17 +125,6 @@ public class GlobalStateAccessor {
}
}
/**
* @return An actor checkpoint id data with ActorCheckpointIdData protobuf schema.
*/
public byte[] getActorCheckpointId(ActorId actorId) {
// Fetch an actor checkpoint id with protobuf bytes format from GCS.
synchronized (GlobalStateAccessor.class) {
validateGlobalStateAccessorPointer();
return this.nativeGetActorCheckpointId(globalStateAccessorNativePointer, actorId.getBytes());
}
}
private void destroyGlobalStateAccessor() {
synchronized (GlobalStateAccessor.class) {
if (0 == globalStateAccessorNativePointer) {
@@ -164,8 +153,6 @@ public class GlobalStateAccessor {
private native byte[] nativeGetActorInfo(long nativePtr, byte[] actorId);
private native byte[] nativeGetActorCheckpointId(long nativePtr, byte[] actorId);
private native byte[] nativeGetPlacementGroupInfo(long nativePtr,
byte[] placementGroupId);
java/runtime/src/main/java/io/ray/runtime/task/LocalModeTaskExecutor.java
-8
View File
@@ -1,6 +1,5 @@
package io.ray.runtime.task;
import io.ray.api.id.ActorId;
import io.ray.api.id.UniqueId;
import io.ray.runtime.RayRuntimeInternal;
@@ -34,11 +33,4 @@ public class LocalModeTaskExecutor extends TaskExecutor<LocalModeTaskExecutor.Lo
return new LocalActorContext(runtime.getWorkerContext().getCurrentWorkerId());
}
@Override
protected void maybeSaveCheckpoint(Object actor, ActorId actorId) {
}
@Override
protected void maybeLoadCheckpoint(Object actor, ActorId actorId) {
}
}
java/runtime/src/main/java/io/ray/runtime/task/NativeTaskExecutor.java
-84
View File
@@ -1,39 +1,14 @@
package io.ray.runtime.task;
import com.google.common.base.Preconditions;
import io.ray.api.Checkpointable;
import io.ray.api.Checkpointable.Checkpoint;
import io.ray.api.Checkpointable.CheckpointContext;
import io.ray.api.id.ActorId;
import io.ray.api.id.UniqueId;
import io.ray.runtime.RayRuntimeInternal;
import java.util.ArrayList;
import java.util.List;
/**
* Task executor for cluster mode.
*/
public class NativeTaskExecutor extends TaskExecutor<NativeTaskExecutor.NativeActorContext> {
// TODO(hchen): Use the C++ config.
private static final int NUM_ACTOR_CHECKPOINTS_TO_KEEP = 20;
static class NativeActorContext extends TaskExecutor.ActorContext {
/**
* Number of tasks executed since last actor checkpoint.
*/
private int numTasksSinceLastCheckpoint = 0;
/**
* IDs of this actor's previous checkpoints.
*/
private List<UniqueId> checkpointIds;
/**
* Timestamp of the last actor checkpoint.
*/
private long lastCheckpointTimestamp = 0;
}
public NativeTaskExecutor(RayRuntimeInternal runtime) {
@@ -49,63 +24,4 @@ public class NativeTaskExecutor extends TaskExecutor<NativeTaskExecutor.NativeAc
// This is to make sure no memory leak when `Ray.exitActor()` is called.
removeActorContext(new UniqueId(workerIdBytes));
}
@Override
protected void maybeSaveCheckpoint(Object actor, ActorId actorId) {
if (!(actor instanceof Checkpointable)) {
return;
}
NativeActorContext actorContext = getActorContext();
CheckpointContext checkpointContext = new CheckpointContext(actorId,
++actorContext.numTasksSinceLastCheckpoint,
System.currentTimeMillis() - actorContext.lastCheckpointTimestamp);
Checkpointable checkpointable = (Checkpointable) actor;
if (!checkpointable.shouldCheckpoint(checkpointContext)) {
return;
}
actorContext.numTasksSinceLastCheckpoint = 0;
actorContext.lastCheckpointTimestamp = System.currentTimeMillis();
UniqueId checkpointId = new UniqueId(nativePrepareCheckpoint());
List<UniqueId> checkpointIds = actorContext.checkpointIds;
checkpointIds.add(checkpointId);
if (checkpointIds.size() > NUM_ACTOR_CHECKPOINTS_TO_KEEP) {
((Checkpointable) actor).checkpointExpired(actorId, checkpointIds.get(0));
checkpointIds.remove(0);
}
checkpointable.saveCheckpoint(actorId, checkpointId);
}
@Override
protected void maybeLoadCheckpoint(Object actor, ActorId actorId) {
if (!(actor instanceof Checkpointable)) {
return;
}
NativeActorContext actorContext = getActorContext();
actorContext.numTasksSinceLastCheckpoint = 0;
actorContext.lastCheckpointTimestamp = System.currentTimeMillis();
actorContext.checkpointIds = new ArrayList<>();
List<Checkpoint> availableCheckpoints
= runtime.getGcsClient().getCheckpointsForActor(actorId);
if (availableCheckpoints.isEmpty()) {
return;
}
UniqueId checkpointId = ((Checkpointable) actor).loadCheckpoint(actorId, availableCheckpoints);
if (checkpointId != null) {
boolean checkpointValid = false;
for (Checkpoint checkpoint : availableCheckpoints) {
if (checkpoint.checkpointId.equals(checkpointId)) {
checkpointValid = true;
break;
}
}
Preconditions.checkArgument(checkpointValid,
"'loadCheckpoint' must return a checkpoint ID that exists in the "
+ "'availableCheckpoints' list, or null.");
nativeNotifyActorResumedFromCheckpoint(checkpointId.getBytes());
}
}
private static native byte[] nativePrepareCheckpoint();
private static native void nativeNotifyActorResumedFromCheckpoint(byte[] checkpointId);
}
java/runtime/src/main/java/io/ray/runtime/task/TaskExecutor.java
-10
View File
@@ -1,7 +1,6 @@
package io.ray.runtime.task;
import com.google.common.base.Preconditions;
import io.ray.api.id.ActorId;
import io.ray.api.id.JobId;
import io.ray.api.id.TaskId;
import io.ray.api.id.UniqueId;
@@ -150,16 +149,10 @@ public abstract class TaskExecutor<T extends TaskExecutor.ActorContext> {
}
// Set result
if (taskType != TaskType.ACTOR_CREATION_TASK) {
if (taskType == TaskType.ACTOR_TASK) {
// TODO (kfstorm): handle checkpoint in core worker.
maybeSaveCheckpoint(actor, runtime.getWorkerContext().getCurrentActorId());
}
if (rayFunction.hasReturn()) {
returnObjects.add(ObjectSerializer.serialize(result));
}
} else {
// TODO (kfstorm): handle checkpoint in core worker.
maybeLoadCheckpoint(result, runtime.getWorkerContext().getCurrentActorId());
actorContext.currentActor = result;
}
LOGGER.debug("Finished executing task {}", taskId);
@@ -195,7 +188,4 @@ public abstract class TaskExecutor<T extends TaskExecutor.ActorContext> {
rayFunctionInfo.get(2));
}
protected abstract void maybeSaveCheckpoint(Object actor, ActorId actorId);
protected abstract void maybeLoadCheckpoint(Object actor, ActorId actorId);
}