Revert "[Streaming] Fault Tolerance Implementation (#10008)" (#10582)

This reverts commit 1b1466748f.
This commit is contained in:
SangBin Cho
2020-09-04 13:21:18 -07:00
committed by GitHub
parent da83bbd764
commit cb919c5e5c
158 changed files with 1227 additions and 7040 deletions
+1 -35
View File
@@ -8,6 +8,7 @@ class Config:
NATIVE_CHANNEL = "native_channel"
CHANNEL_SIZE = "channel_size"
CHANNEL_SIZE_DEFAULT = 10**8
IS_RECREATE = "streaming.is_recreate"
# return from StreamingReader.getBundle if only empty message read in this
# interval.
TIMER_INTERVAL_MS = "timer_interval_ms"
@@ -25,38 +26,3 @@ class Config:
FLOW_CONTROL_TYPE = "streaming.flow_control_type"
WRITER_CONSUMED_STEP = "streaming.writer.consumed_step"
READER_CONSUMED_STEP = "streaming.reader.consumed_step"
# state backend
CP_STATE_BACKEND_TYPE = "streaming.context-backend.type"
CP_STATE_BACKEND_MEMORY = "memory"
CP_STATE_BACKEND_LOCAL_FILE = "local_file"
CP_STATE_BACKEND_DEFAULT = CP_STATE_BACKEND_MEMORY
# local disk
FILE_STATE_ROOT_PATH = "streaming.context-backend.file-state.root"
FILE_STATE_ROOT_PATH_DEFAULT = "/tmp/ray_streaming_state"
# checkpoint
JOB_WORKER_CONTEXT_KEY = "jobworker_context_"
# reliability level
REQUEST_ROLLBACK_RETRY_TIMES = 3
# checkpoint prefix key
JOB_WORKER_OP_CHECKPOINT_PREFIX_KEY = "jobwk_op_"
class ConfigHelper(object):
@staticmethod
def get_cp_local_file_root_dir(conf):
value = conf.get(Config.FILE_STATE_ROOT_PATH)
if value is not None:
return value
return Config.FILE_STATE_ROOT_PATH_DEFAULT
@staticmethod
def get_cp_context_backend_type(conf):
value = conf.get(Config.CP_STATE_BACKEND_TYPE)
if value is not None:
return value
return Config.CP_STATE_BACKEND_DEFAULT