[Streaming] Fault Tolerance Implementation (#10595)

streaming/python/config.py

@@ -8,7 +8,6 @@ class Config:
     NATIVE_CHANNEL = "native_channel"
     CHANNEL_SIZE = "channel_size"
     CHANNEL_SIZE_DEFAULT = 10**8
-    IS_RECREATE = "streaming.is_recreate"
     # return from StreamingReader.getBundle if only empty message read in this
     # interval.
     TIMER_INTERVAL_MS = "timer_interval_ms"
@@ -26,3 +25,38 @@ class Config:
     FLOW_CONTROL_TYPE = "streaming.flow_control_type"
     WRITER_CONSUMED_STEP = "streaming.writer.consumed_step"
     READER_CONSUMED_STEP = "streaming.reader.consumed_step"
+
+    # state backend
+    CP_STATE_BACKEND_TYPE = "streaming.context-backend.type"
+    CP_STATE_BACKEND_MEMORY = "memory"
+    CP_STATE_BACKEND_LOCAL_FILE = "local_file"
+    CP_STATE_BACKEND_DEFAULT = CP_STATE_BACKEND_MEMORY
+
+    # local disk
+    FILE_STATE_ROOT_PATH = "streaming.context-backend.file-state.root"
+    FILE_STATE_ROOT_PATH_DEFAULT = "/tmp/ray_streaming_state"
+
+    # checkpoint
+    JOB_WORKER_CONTEXT_KEY = "jobworker_context_"
+
+    # reliability level
+    REQUEST_ROLLBACK_RETRY_TIMES = 3
+
+    # checkpoint prefix key
+    JOB_WORKER_OP_CHECKPOINT_PREFIX_KEY = "jobwk_op_"
+
+
+class ConfigHelper(object):
+    @staticmethod
+    def get_cp_local_file_root_dir(conf):
+        value = conf.get(Config.FILE_STATE_ROOT_PATH)
+        if value is not None:
+            return value
+        return Config.FILE_STATE_ROOT_PATH_DEFAULT
+
+    @staticmethod
+    def get_cp_context_backend_type(conf):
+        value = conf.get(Config.CP_STATE_BACKEND_TYPE)
+        if value is not None:
+            return value
+        return Config.CP_STATE_BACKEND_DEFAULT