Implement actor checkpointing (#3839)

* Implement Actor checkpointing

* docs

* fix

* fix

* fix

* move restore-from-checkpoint to HandleActorStateTransition

* Revert "move restore-from-checkpoint to HandleActorStateTransition"

This reverts commit 9aa4447c1e3e321f42a1d895d72f17098b72de12.

* resubmit waiting tasks when actor frontier restored

* add doc about num_actor_checkpoints_to_keep=1

* add num_actor_checkpoints_to_keep to Cython

* add checkpoint_expired api

* check if actor class is abstract

* change checkpoint_ids to long string

* implement java

* Refactor to delay actor creation publish until checkpoint is resumed

* debug, lint

* Erase from checkpoints to restore if task fails

* fix lint

* update comments

* avoid duplicated actor notification log

* fix unintended change

* add actor_id to checkpoint_expired

* small java updates

* make checkpoint info per actor

* lint

* Remove logging

* Remove old actor checkpointing Python code, move new checkpointing code to FunctionActionManager

* Replace old actor checkpointing tests

* Fix test and lint

* address comments

* consolidate kill_actor

* Remove __ray_checkpoint__

* fix non-ascii char

* Loosen test checks

* fix java

* fix sphinx-build

python/ray/includes/unique_ids.pxi

@@ -7,9 +7,21 @@ See https://github.com/ray-project/ray/issues/3721.
 # WARNING: Any additional ID types defined in this file must be added to the
 # _ID_TYPES list at the bottom of this file.
 from ray.includes.common cimport (
-    CUniqueID, CTaskID, CObjectID, CFunctionID, CActorClassID, CActorID,
-    CActorHandleID, CWorkerID, CDriverID, CConfigID, CClientID,
-    ComputePutId, ComputeTaskId)
+    CActorCheckpointID,
+    CActorClassID,
+    CActorHandleID,
+    CActorID,
+    CClientID,
+    CConfigID,
+    CDriverID,
+    CFunctionID,
+    CObjectID,
+    CTaskID,
+    CUniqueID,
+    CWorkerID,
+    ComputePutId,
+    ComputeTaskId,
+)
 
 from ray.utils import decode
 
@@ -236,6 +248,29 @@ cdef class ActorHandleID(UniqueID):
         return "ActorHandleID(" + self.hex() + ")"
 
 
+cdef class ActorCheckpointID(UniqueID):
+
+    def __init__(self, id):
+        if not id:
+            self.data = CUniqueID()
+        else:
+            check_id(id)
+            self.data = CUniqueID.from_binary(id)
+
+    @staticmethod
+    cdef from_native(const CActorCheckpointID& cpp_id):
+        cdef ActorCheckpointID self = ActorCheckpointID.__new__(ActorHandleID)
+        self.data = cpp_id
+        return self
+
+    @staticmethod
+    def nil():
+        return ActorCheckpointID.from_native(CActorCheckpointID.nil())
+
+    def __repr__(self):
+        return "ActorCheckpointID(" + self.hex() + ")"
+
+
 cdef class FunctionID(UniqueID):
 
     def __init__(self, id):