[Streaming] Streaming Python API (#6755)

This commit is contained in:
chaokunyang
2020-02-25 10:33:33 +08:00
committed by GitHub
parent 2c1f4fd82c
commit 8b6784de06
71 changed files with 2701 additions and 1928 deletions
+15 -10
View File
@@ -1,18 +1,23 @@
import ray
from ray.streaming.config import Config
from ray.streaming.streaming import Environment, Conf
from ray.streaming import StreamingContext
def test_word_count():
ray.init()
env = Environment(config=Conf(channel_type=Config.NATIVE_CHANNEL))
env.read_text_file(__file__) \
ray.init(load_code_from_local=True, include_java=True)
ctx = StreamingContext.Builder() \
.build()
ctx.read_text_file(__file__) \
.set_parallelism(1) \
.filter(lambda x: "word" in x) \
.inspect(lambda x: print("result", x))
env_handle = env.execute()
ray.get(env_handle) # Stay alive until execution finishes
env.wait_finish()
.flat_map(lambda x: x.split()) \
.map(lambda x: (x, 1)) \
.key_by(lambda x: x[0]) \
.reduce(lambda old_value, new_value:
(old_value[0], old_value[1] + new_value[1])) \
.filter(lambda x: "ray" not in x) \
.sink(lambda x: print("result", x))
ctx.submit("word_count")
import time
time.sleep(3)
ray.shutdown()