Revert "[Streaming] Fault Tolerance Implementation (#10008)" (#10582)

This reverts commit 1b1466748f.
This commit is contained in:
SangBin Cho
2020-09-04 13:21:18 -07:00
committed by GitHub
parent da83bbd764
commit cb919c5e5c
158 changed files with 1227 additions and 7040 deletions
+3 -17
View File
@@ -22,12 +22,6 @@ class Function(ABC):
def close(self):
pass
def save_checkpoint(self):
pass
def load_checkpoint(self, checkpoint_obj):
pass
class EmptyFunction(Function):
"""Default function which does nothing"""
@@ -64,15 +58,12 @@ class SourceFunction(Function):
pass
@abstractmethod
def fetch(self, ctx: SourceContext):
def run(self, ctx: SourceContext):
"""Starts the source. Implementations can use the
:class:`SourceContext` to emit elements.
"""
pass
def close(self):
pass
class MapFunction(Function):
"""
@@ -185,29 +176,24 @@ class CollectionSourceFunction(SourceFunction):
def init(self, parallel, index):
pass
def fetch(self, ctx: SourceContext):
def run(self, ctx: SourceContext):
for v in self.values:
ctx.collect(v)
self.values = []
class LocalFileSourceFunction(SourceFunction):
def __init__(self, filename):
self.filename = filename
self.done = False
def init(self, parallel, index):
pass
def fetch(self, ctx: SourceContext):
if self.done:
return
def run(self, ctx: SourceContext):
with open(self.filename, "r") as f:
line = f.readline()
while line != "":
ctx.collect(line[:-1])
line = f.readline()
self.done = True
class SimpleMapFunction(MapFunction):