[Streaming] Streaming Cross-Lang API (#7464)

2026-07-07 05:35:39 +08:00 · 2020-04-29 13:42:08 +08:00
parent 101255f782
commit 91f630f709
72 changed files with 1612 additions and 408 deletions
@@ -1,9 +1,14 @@
 package io.ray.streaming.runtime.core.collector;

-import io.ray.runtime.serializer.Serializer;
+import io.ray.api.BaseActor;
+import io.ray.api.RayPyActor;
+import io.ray.streaming.api.Language;
 import io.ray.streaming.api.collector.Collector;
 import io.ray.streaming.api.partition.Partition;
 import io.ray.streaming.message.Record;
+import io.ray.streaming.runtime.serialization.CrossLangSerializer;
+import io.ray.streaming.runtime.serialization.JavaSerializer;
+import io.ray.streaming.runtime.serialization.Serializer;
 import io.ray.streaming.runtime.transfer.ChannelID;
 import io.ray.streaming.runtime.transfer.DataWriter;
 import java.nio.ByteBuffer;
@@ -14,15 +19,24 @@ import org.slf4j.LoggerFactory;
 public class OutputCollector implements Collector<Record> {
  private static final Logger LOGGER = LoggerFactory.getLogger(OutputCollector.class);

-  private Partition partition;
-  private DataWriter writer;
-  private ChannelID[] outputQueues;
+  private final DataWriter writer;
+  private final ChannelID[] outputQueues;
+  private final Collection<BaseActor> targetActors;
+  private final Language[] targetLanguages;
+  private final Partition partition;
+  private final Serializer javaSerializer = new JavaSerializer();
+  private final Serializer crossLangSerializer = new CrossLangSerializer();

-  public OutputCollector(Collection<String> outputQueueIds,
-                         DataWriter writer,
+  public OutputCollector(DataWriter writer,
+                         Collection<String> outputQueueIds,
+                         Collection<BaseActor> targetActors,
                         Partition partition) {
-    this.outputQueues = outputQueueIds.stream().map(ChannelID::from).toArray(ChannelID[]::new);
    this.writer = writer;
+    this.outputQueues = outputQueueIds.stream().map(ChannelID::from).toArray(ChannelID[]::new);
+    this.targetActors = targetActors;
+    this.targetLanguages = targetActors.stream()
+        .map(actor -> actor instanceof RayPyActor ? Language.PYTHON : Language.JAVA)
+        .toArray(Language[]::new);
    this.partition = partition;
    LOGGER.debug("OutputCollector constructed, outputQueueIds:{}, partition:{}.",
        outputQueueIds, this.partition);
@@ -31,9 +45,32 @@ public class OutputCollector implements Collector<Record> {
  @Override
  public void collect(Record record) {
    int[] partitions = this.partition.partition(record, outputQueues.length);
-    ByteBuffer msgBuffer = ByteBuffer.wrap(Serializer.encode(record).getLeft());
+    ByteBuffer javaBuffer = null;
+    ByteBuffer crossLangBuffer = null;
    for (int partition : partitions) {
-      writer.write(outputQueues[partition], msgBuffer);
+      if (targetLanguages[partition] == Language.JAVA) {
+        // avoid repeated serialization
+        if (javaBuffer == null) {
+          byte[] bytes = javaSerializer.serialize(record);
+          javaBuffer = ByteBuffer.allocate(1 + bytes.length);
+          javaBuffer.put(Serializer.JAVA_TYPE_ID);
+          // TODO(chaokunyang) remove copy
+          javaBuffer.put(bytes);
+          javaBuffer.flip();
+        }
+        writer.write(outputQueues[partition], javaBuffer.duplicate());
+      } else {
+        // avoid repeated serialization
+        if (crossLangBuffer == null) {
+          byte[] bytes = crossLangSerializer.serialize(record);
+          crossLangBuffer = ByteBuffer.allocate(1 + bytes.length);
+          crossLangBuffer.put(Serializer.CROSS_LANG_TYPE_ID);
+          // TODO(chaokunyang) remove copy
+          crossLangBuffer.put(bytes);
+          crossLangBuffer.flip();
+        }
+        writer.write(outputQueues[partition], crossLangBuffer.duplicate());
+      }
    }
  }

@@ -12,6 +12,7 @@ import io.ray.streaming.runtime.core.graph.ExecutionNode;
 import io.ray.streaming.runtime.core.graph.ExecutionTask;
 import io.ray.streaming.runtime.generated.RemoteCall;
 import io.ray.streaming.runtime.generated.Streaming;
+import io.ray.streaming.runtime.serialization.MsgPackSerializer;
 import java.util.Arrays;

 public class GraphPbBuilder {
@@ -74,11 +75,10 @@ public class GraphPbBuilder {
  private byte[] serializeFunction(Function function) {
    if (function instanceof PythonFunction) {
      PythonFunction pyFunc = (PythonFunction) function;
-      // function_bytes, module_name, class_name, function_name, function_interface
+      // function_bytes, module_name, function_name, function_interface
      return serializer.serialize(Arrays.asList(
          pyFunc.getFunction(), pyFunc.getModuleName(),
-          pyFunc.getClassName(), pyFunc.getFunctionName(),
-          pyFunc.getFunctionInterface()
+          pyFunc.getFunctionName(), pyFunc.getFunctionInterface()
      ));
    } else {
      return new byte[0];
@@ -88,10 +88,10 @@ public class GraphPbBuilder {
  private byte[] serializePartition(Partition partition) {
    if (partition instanceof PythonPartition) {
      PythonPartition pythonPartition = (PythonPartition) partition;
-      // partition_bytes, module_name, class_name, function_name
+      // partition_bytes, module_name, function_name
      return serializer.serialize(Arrays.asList(
          pythonPartition.getPartition(), pythonPartition.getModuleName(),
-          pythonPartition.getClassName(), pythonPartition.getFunctionName()
+          pythonPartition.getFunctionName()
      ));
    } else {
      return new byte[0];
@@ -1,16 +1,21 @@
 package io.ray.streaming.runtime.python;

+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Primitives;
 import io.ray.streaming.api.context.StreamingContext;
 import io.ray.streaming.python.PythonFunction;
 import io.ray.streaming.python.PythonPartition;
 import io.ray.streaming.python.stream.PythonStreamSource;
+import io.ray.streaming.runtime.serialization.MsgPackSerializer;
 import io.ray.streaming.runtime.util.ReflectionUtils;
 import java.lang.reflect.Method;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
+import java.util.function.Function;
 import java.util.stream.Collectors;
-import org.msgpack.core.Preconditions;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

@@ -68,7 +73,7 @@ public class PythonGateway {
    Preconditions.checkNotNull(streamingContext);
    try {
      PythonStreamSource pythonStreamSource = PythonStreamSource.from(
-          streamingContext, PythonFunction.fromFunction(pySourceFunc));
+          streamingContext, new PythonFunction(pySourceFunc));
      referenceMap.put(getReferenceId(pythonStreamSource), pythonStreamSource);
      return serializer.serialize(getReferenceId(pythonStreamSource));
    } catch (Exception e) {
@@ -84,7 +89,7 @@ public class PythonGateway {
  }

  public byte[] createPyFunc(byte[] pyFunc) {
-    PythonFunction function = PythonFunction.fromFunction(pyFunc);
+    PythonFunction function = new PythonFunction(pyFunc);
    referenceMap.put(getReferenceId(function), function);
    return serializer.serialize(getReferenceId(function));
  }
@@ -98,15 +103,21 @@ public class PythonGateway {
  public byte[] callFunction(byte[] paramsBytes) {
    try {
      List<Object> params = (List<Object>) serializer.deserialize(paramsBytes);
-      params = processReferenceParameters(params);
+      params = processParameters(params);
      LOG.info("callFunction params {}", params);
      String className = (String) params.get(0);
      String funcName = (String) params.get(1);
      Class<?> clz = Class.forName(className, true, this.getClass().getClassLoader());
-      Method method = ReflectionUtils.findMethod(clz, funcName);
+      Class[] paramsTypes = params.subList(2, params.size()).stream()
+          .map(Object::getClass).toArray(Class[]::new);
+      Method method = findMethod(clz, funcName, paramsTypes);
      Object result = method.invoke(null, params.subList(2, params.size()).toArray());
-      referenceMap.put(getReferenceId(result), result);
-      return serializer.serialize(getReferenceId(result));
+      if (returnReference(result)) {
+        referenceMap.put(getReferenceId(result), result);
+        return serializer.serialize(getReferenceId(result));
+      } else {
+        return serializer.serialize(result);
+      }
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
@@ -115,31 +126,78 @@ public class PythonGateway {
  public byte[] callMethod(byte[] paramsBytes) {
    try {
      List<Object> params = (List<Object>) serializer.deserialize(paramsBytes);
-      params = processReferenceParameters(params);
+      params = processParameters(params);
      LOG.info("callMethod params {}", params);
      Object obj = params.get(0);
      String methodName = (String) params.get(1);
-      Method method = ReflectionUtils.findMethod(obj.getClass(), methodName);
+      Class<?> clz = obj.getClass();
+      Class[] paramsTypes = params.subList(2, params.size()).stream()
+          .map(Object::getClass).toArray(Class[]::new);
+      Method method = findMethod(clz, methodName, paramsTypes);
      Object result = method.invoke(obj, params.subList(2, params.size()).toArray());
-      referenceMap.put(getReferenceId(result), result);
-      return serializer.serialize(getReferenceId(result));
+      if (returnReference(result)) {
+        referenceMap.put(getReferenceId(result), result);
+        return serializer.serialize(getReferenceId(result));
+      } else {
+        return serializer.serialize(result);
+      }
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }

-  private List<Object> processReferenceParameters(List<Object> params) {
-    return params.stream().map(this::processReferenceParameter)
+  private static Method findMethod(Class<?> cls, String methodName, Class[] paramsTypes) {
+    List<Method> methods = ReflectionUtils.findMethods(cls, methodName);
+    if (methods.size() == 1) {
+      return methods.get(0);
+    }
+    // Convert all params types to primitive types if it's boxed type
+    Class[] unwrappedTypes = Arrays.stream(paramsTypes)
+        .map((Function<Class, Class>) Primitives::unwrap)
+        .toArray(Class[]::new);
+    Optional<Method> any = methods.stream()
+        .filter(m -> Arrays.equals(m.getParameterTypes(), paramsTypes) ||
+            Arrays.equals(m.getParameterTypes(), unwrappedTypes))
+        .findAny();
+    Preconditions.checkArgument(any.isPresent(),
+        String.format("Method %s with type %s doesn't exist on class %s",
+            methodName, Arrays.toString(paramsTypes), cls));
+    return any.get();
+  }
+
+  private static boolean returnReference(Object value) {
+    return !(value instanceof Number) && !(value instanceof String) && !(value instanceof byte[]);
+  }
+
+  public byte[] newInstance(byte[] classNameBytes) {
+    String className = (String) serializer.deserialize(classNameBytes);
+    try {
+      Class<?> clz = Class.forName(className, true, this.getClass().getClassLoader());
+      Object instance = clz.newInstance();
+      referenceMap.put(getReferenceId(instance), instance);
+      return serializer.serialize(getReferenceId(instance));
+    } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) {
+      throw new IllegalArgumentException(
+          String.format("Create instance for class %s failed", className), e);
+    }
+  }
+
+  private List<Object> processParameters(List<Object> params) {
+    return params.stream().map(this::processParameter)
        .collect(Collectors.toList());
  }

-  private Object processReferenceParameter(Object o) {
+  private Object processParameter(Object o) {
    if (o instanceof String) {
      Object value = referenceMap.get(o);
      if (value != null) {
        return value;
      }
    }
+    // Since python can't represent byte/short, we convert all Byte/Short to Integer
+    if (o instanceof Byte || o instanceof Short) {
+      return ((Number) o).intValue();
+    }
    return o;
  }

@@ -41,15 +41,11 @@ public class JobSchedulerImpl implements JobScheduler {
  public void schedule(JobGraph jobGraph, Map<String, String> jobConfig) {
    this.jobConfig = jobConfig;
    this.jobGraph = jobGraph;
-    if (Ray.internal() == null) {
-      System.setProperty("ray.raylet.config.num_workers_per_process_java", "1");
-      Ray.init();
-    }

    ExecutionGraph executionGraph = this.taskAssigner.assign(this.jobGraph);
    List<ExecutionNode> executionNodes = executionGraph.getExecutionNodeList();
    boolean hasPythonNode = executionNodes.stream()
-        .allMatch(node -> node.getLanguage() == Language.PYTHON);
+        .anyMatch(node -> node.getLanguage() == Language.PYTHON);
    RemoteCall.ExecutionGraph executionGraphPb = null;
    if (hasPythonNode) {
      executionGraphPb = new GraphPbBuilder().buildExecutionGraphPb(executionGraph);
@@ -2,6 +2,8 @@ package io.ray.streaming.runtime.schedule;

 import io.ray.api.BaseActor;
 import io.ray.api.Ray;
+import io.ray.api.RayActor;
+import io.ray.api.RayPyActor;
 import io.ray.api.function.PyActorClass;
 import io.ray.streaming.jobgraph.JobEdge;
 import io.ray.streaming.jobgraph.JobGraph;
@@ -15,8 +17,11 @@ import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;

 public class TaskAssignerImpl implements TaskAssigner {
+  private static final Logger LOG = LoggerFactory.getLogger(TaskAssignerImpl.class);

  /**
   * Assign an optimized logical plan to execution graph.
@@ -61,11 +66,17 @@ public class TaskAssignerImpl implements TaskAssigner {

  private BaseActor createWorker(JobVertex jobVertex) {
    switch (jobVertex.getLanguage()) {
-      case PYTHON:
-        return Ray.createActor(
+      case PYTHON: {
+        RayPyActor worker = Ray.createActor(
            new PyActorClass("ray.streaming.runtime.worker", "JobWorker"));
-      case JAVA:
-        return Ray.createActor(JobWorker::new);
+        LOG.info("Created python worker {}", worker);
+        return worker;
+      }
+      case JAVA: {
+        RayActor<JobWorker> worker = Ray.createActor(JobWorker::new);
+        LOG.info("Created java worker {}", worker);
+        return worker;
+      }
      default:
        throw new UnsupportedOperationException(
            "Unsupported language " + jobVertex.getLanguage());
@@ -0,0 +1,62 @@
+package io.ray.streaming.runtime.serialization;
+
+import io.ray.streaming.message.KeyRecord;
+import io.ray.streaming.message.Record;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * A serializer for cross-lang serialization between java/python.
+ * TODO implements a more sophisticated serialization framework
+ */
+public class CrossLangSerializer implements Serializer {
+  private static final byte RECORD_TYPE_ID = 0;
+  private static final byte KEY_RECORD_TYPE_ID = 1;
+
+  private MsgPackSerializer msgPackSerializer = new MsgPackSerializer();
+
+  public byte[] serialize(Object object) {
+    Record record = (Record) object;
+    Object value = record.getValue();
+    Class<? extends Record> clz = record.getClass();
+    if (clz == Record.class) {
+      return msgPackSerializer.serialize(Arrays.asList(
+          RECORD_TYPE_ID, record.getStream(), value));
+    } else if (clz == KeyRecord.class) {
+      KeyRecord keyRecord = (KeyRecord) record;
+      Object key = keyRecord.getKey();
+      return msgPackSerializer.serialize(Arrays.asList(
+          KEY_RECORD_TYPE_ID, keyRecord.getStream(), key, value));
+    } else {
+      throw new UnsupportedOperationException(
+          String.format("Serialize %s is unsupported.", record));
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  public Record deserialize(byte[] bytes) {
+    List list = (List) msgPackSerializer.deserialize(bytes);
+    Byte typeId = (Byte) list.get(0);
+    switch (typeId) {
+      case RECORD_TYPE_ID: {
+        String stream = (String) list.get(1);
+        Object value = list.get(2);
+        Record record = new Record(value);
+        record.setStream(stream);
+        return record;
+      }
+      case KEY_RECORD_TYPE_ID: {
+        String stream = (String) list.get(1);
+        Object key = list.get(2);
+        Object value = list.get(3);
+        KeyRecord keyRecord = new KeyRecord(key, value);
+        keyRecord.setStream(stream);
+        return keyRecord;
+      }
+      default:
+        throw new UnsupportedOperationException("Unsupported type " + typeId);
+
+    }
+  }
+
+}
@@ -0,0 +1,15 @@
+package io.ray.streaming.runtime.serialization;
+
+import io.ray.runtime.serializer.FstSerializer;
+
+public class JavaSerializer implements Serializer {
+  @Override
+  public byte[] serialize(Object object) {
+    return FstSerializer.encode(object);
+  }
+
+  @Override
+  public <T> T deserialize(byte[] bytes) {
+    return FstSerializer.decode(bytes);
+  }
+}
@@ -1,4 +1,4 @@
-package io.ray.streaming.runtime.python;
+package io.ray.streaming.runtime.serialization;

 import com.google.common.io.BaseEncoding;
 import java.util.ArrayList;
@@ -31,6 +31,10 @@ public class MsgPackSerializer {
        Class<?> clz = obj.getClass();
        if (clz == Boolean.class) {
          packer.packBoolean((Boolean) obj);
+        } else if (clz == Byte.class) {
+          packer.packByte((Byte) obj);
+        } else if (clz == Short.class) {
+          packer.packShort((Short) obj);
        } else if (clz == Integer.class) {
          packer.packInt((Integer) obj);
        } else if (clz == Long.class) {
@@ -84,7 +88,11 @@ public class MsgPackSerializer {
        return value.asBooleanValue().getBoolean();
      case INTEGER:
        IntegerValue iv = value.asIntegerValue();
-        if (iv.isInIntRange()) {
+        if (iv.isInByteRange()) {
+          return iv.toByte();
+        } else if (iv.isInShortRange()) {
+          return iv.toShort();
+        } else if (iv.isInIntRange()) {
          return iv.toInt();
        } else if (iv.isInLongRange()) {
          return iv.toLong();
@@ -0,0 +1,12 @@
+package io.ray.streaming.runtime.serialization;
+
+public interface Serializer {
+  byte CROSS_LANG_TYPE_ID = 0;
+  byte JAVA_TYPE_ID = 1;
+  byte PYTHON_TYPE_ID = 2;
+
+  byte[] serialize(Object object);
+
+  <T> T deserialize(byte[] bytes);
+
+}
@@ -20,7 +20,7 @@ import java.util.Map;
 */
 public class ChannelCreationParametersBuilder {

-  public class Parameter {
+  public static class Parameter {

    private ActorId actorId;
    private FunctionDescriptor asyncFunctionDescriptor;
@@ -138,7 +138,7 @@ public class ChannelCreationParametersBuilder {
        parameter.setAsyncFunctionDescriptor(pyAsyncFunctionDesc);
        parameter.setSyncFunctionDescriptor(pySyncFunctionDesc);
      } else {
-        Preconditions.checkArgument(false, "Invalid actor type");
+        throw new IllegalArgumentException("Invalid actor type");
      }
      parameters.add(parameter);
    }
@@ -152,10 +152,10 @@ public class ChannelCreationParametersBuilder {
  }

  public String toString() {
-    String str = "";
+    StringBuilder str = new StringBuilder();
    for (Parameter param : parameters) {
-      str += param.toString();
+      str.append(param.toString());
    }
-    return str;
+    return str.toString();
  }
 }
@@ -40,7 +40,7 @@ public class DataReader {
    }
    long timerInterval = Long.parseLong(
        conf.getOrDefault(Config.TIMER_INTERVAL_MS, "-1"));
-    String channelType = conf.getOrDefault(Config.CHANNEL_TYPE, Config.DEFAULT_CHANNEL_TYPE);
+    String channelType = conf.get(Config.CHANNEL_TYPE);
    boolean isMock = false;
    if (Config.MEMORY_CHANNEL.equals(channelType)) {
      isMock = true;
@@ -37,7 +37,7 @@ public class DataWriter {
                    Map<String, String> conf) {
    Preconditions.checkArgument(!outputChannels.isEmpty());
    Preconditions.checkArgument(outputChannels.size() == toActors.size());
-    ChannelCreationParametersBuilder initialParameters =
+    ChannelCreationParametersBuilder initParameters =
        new ChannelCreationParametersBuilder().buildOutputQueueParameters(outputChannels, toActors);
    byte[][] outputChannelsBytes = outputChannels.stream()
        .map(ChannelID::idStrToBytes).toArray(byte[][]::new);
@@ -47,13 +47,14 @@ public class DataWriter {
    for (int i = 0; i < outputChannels.size(); i++) {
      msgIds[i] = 0;
    }
-    String channelType = conf.getOrDefault(Config.CHANNEL_TYPE, Config.DEFAULT_CHANNEL_TYPE);
+    String channelType = conf.get(Config.CHANNEL_TYPE);
    boolean isMock = false;
-    if (Config.MEMORY_CHANNEL.equals(channelType)) {
+    if (Config.MEMORY_CHANNEL.equalsIgnoreCase(channelType)) {
      isMock = true;
+      LOGGER.info("Using memory channel");
    }
    this.nativeWriterPtr = createWriterNative(
-        initialParameters,
+        initParameters,
        outputChannelsBytes,
        msgIds,
        channelSize,
@@ -19,6 +19,7 @@ public class ReflectionUtils {

  /**
   * For covariant return type, return the most specific method.
+   *
   * @return all methods named by {@code methodName},
   */
  public static List<Method> findMethods(Class<?> cls, String methodName) {
@@ -1,5 +1,6 @@
 package io.ray.streaming.runtime.worker;

+import io.ray.api.Ray;
 import io.ray.streaming.runtime.core.graph.ExecutionGraph;
 import io.ray.streaming.runtime.core.graph.ExecutionNode;
 import io.ray.streaming.runtime.core.graph.ExecutionNode.NodeType;
@@ -14,11 +15,8 @@ import io.ray.streaming.runtime.worker.context.WorkerContext;
 import io.ray.streaming.runtime.worker.tasks.OneInputStreamTask;
 import io.ray.streaming.runtime.worker.tasks.SourceStreamTask;
 import io.ray.streaming.runtime.worker.tasks.StreamTask;
-import io.ray.streaming.util.Config;
-
 import java.io.Serializable;
 import java.util.Map;
-
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

@@ -27,6 +25,8 @@ import org.slf4j.LoggerFactory;
 */
 public class JobWorker implements Serializable {
  private static final Logger LOGGER = LoggerFactory.getLogger(JobWorker.class);
+  // special flag to indicate this actor not ready
+  private static final byte[] NOT_READY_FLAG = new byte[4];

  static {
    EnvUtil.loadNativeLibraries();
@@ -53,12 +53,11 @@ public class JobWorker implements Serializable {

    this.nodeType = executionNode.getNodeType();
    this.streamProcessor = ProcessBuilder
-      .buildProcessor(executionNode.getStreamOperator());
-    LOGGER.debug("Initializing StreamWorker, taskId: {}, operator: {}.", taskId, streamProcessor);
+        .buildProcessor(executionNode.getStreamOperator());
+    LOGGER.info("Initializing StreamWorker, pid {}, taskId: {}, operator: {}.",
+        EnvUtil.getJvmPid(), taskId, streamProcessor);

-    String channelType = (String) this.config.getOrDefault(
-        Config.CHANNEL_TYPE, Config.DEFAULT_CHANNEL_TYPE);
-    if (channelType.equals(Config.NATIVE_CHANNEL)) {
+    if (!Ray.getRuntimeContext().isSingleProcess()) {
      transferHandler = new TransferHandler();
    }
    task = createStreamTask();
@@ -124,6 +123,9 @@ public class JobWorker implements Serializable {
   * and receive result from this actor
   */
  public byte[] onReaderMessageSync(byte[] buffer) {
+    if (transferHandler == null) {
+      return NOT_READY_FLAG;
+    }
    return transferHandler.onReaderMessageSync(buffer);
  }

@@ -139,6 +141,9 @@ public class JobWorker implements Serializable {
   * and receive result from this actor
   */
  public byte[] onWriterMessageSync(byte[] buffer) {
+    if (transferHandler == null) {
+      return NOT_READY_FLAG;
+    }
    return transferHandler.onWriterMessageSync(buffer);
  }
 }
@@ -1,7 +1,9 @@
 package io.ray.streaming.runtime.worker.tasks;

-import io.ray.runtime.serializer.Serializer;
 import io.ray.streaming.runtime.core.processor.Processor;
+import io.ray.streaming.runtime.serialization.CrossLangSerializer;
+import io.ray.streaming.runtime.serialization.JavaSerializer;
+import io.ray.streaming.runtime.serialization.Serializer;
 import io.ray.streaming.runtime.transfer.Message;
 import io.ray.streaming.runtime.worker.JobWorker;
 import io.ray.streaming.util.Config;
@@ -10,11 +12,15 @@ public abstract class InputStreamTask extends StreamTask {
  private volatile boolean running = true;
  private volatile boolean stopped = false;
  private long readTimeoutMillis;
+  private final io.ray.streaming.runtime.serialization.Serializer javaSerializer;
+  private final io.ray.streaming.runtime.serialization.Serializer crossLangSerializer;

  public InputStreamTask(int taskId, Processor processor, JobWorker streamWorker) {
    super(taskId, processor, streamWorker);
    readTimeoutMillis = Long.parseLong((String) streamWorker.getConfig()
        .getOrDefault(Config.READ_TIMEOUT_MS, Config.DEFAULT_READ_TIMEOUT_MS));
+    javaSerializer = new JavaSerializer();
+    crossLangSerializer = new CrossLangSerializer();
  }

  @Override
@@ -26,9 +32,15 @@ public abstract class InputStreamTask extends StreamTask {
    while (running) {
      Message item = reader.read(readTimeoutMillis);
      if (item != null) {
-        byte[] bytes = new byte[item.body().remaining()];
+        byte[] bytes = new byte[item.body().remaining() - 1];
+        byte typeId = item.body().get();
        item.body().get(bytes);
-        Object obj = Serializer.decode(bytes, Object.class);
+        Object obj;
+        if (typeId == Serializer.JAVA_TYPE_ID) {
+          obj = javaSerializer.deserialize(bytes);
+        } else {
+          obj = crossLangSerializer.deserialize(bytes);
+        }
        processor.process(obj);
      }
    }
@@ -26,7 +26,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;

 public abstract class StreamTask implements Runnable {
-
  private static final Logger LOG = LoggerFactory.getLogger(StreamTask.class);

  protected int taskId;
@@ -53,8 +52,8 @@ public abstract class StreamTask implements Runnable {
    String queueSize = worker.getConfig()
        .getOrDefault(Config.CHANNEL_SIZE, Config.CHANNEL_SIZE_DEFAULT);
    queueConf.put(Config.CHANNEL_SIZE, queueSize);
-    String channelType = worker.getConfig()
-        .getOrDefault(Config.CHANNEL_TYPE, Config.MEMORY_CHANNEL);
+    String channelType = Ray.getRuntimeContext().isSingleProcess() ?
+        Config.MEMORY_CHANNEL : Config.NATIVE_CHANNEL;
    queueConf.put(Config.CHANNEL_TYPE, channelType);

    ExecutionGraph executionGraph = worker.getExecutionGraph();
@@ -82,7 +81,7 @@ public abstract class StreamTask implements Runnable {
        LOG.info("Create DataWriter succeed.");
        writers.put(edge, writer);
        Partition partition = edge.getPartition();
-        collectors.add(new OutputCollector(channelIDs, writer, partition));
+        collectors.add(new OutputCollector(writer, channelIDs, outputActors.values(), partition));
      }
    }

@@ -106,8 +105,8 @@ public abstract class StreamTask implements Runnable {
      reader = new DataReader(channelIDs, inputActors, queueConf);
    }

-    RuntimeContext runtimeContext = new RayRuntimeContext(worker.getExecutionTask(),
-        worker.getConfig(), executionNode.getParallelism());
+    RuntimeContext runtimeContext = new RayRuntimeContext(
+        worker.getExecutionTask(), worker.getConfig(), executionNode.getParallelism());

    processor.open(collectors, runtimeContext);

@@ -24,11 +24,13 @@ public abstract class BaseUnitTest {

  @BeforeMethod
  public void testBegin(Method method) {
-    LOG.info(">>>>>>>>>>>>>>>>>>>> Test case: " + method.getName() + " began >>>>>>>>>>>>>>>>>>>>");
+    LOG.info(">>>>>>>>>>>>>>>>>>>> Test case: {}.{} began >>>>>>>>>>>>>>>>>>>>",
+        method.getDeclaringClass(),  method.getName());
  }

  @AfterMethod
  public void testEnd(Method method) {
-    LOG.info(">>>>>>>>>>>>>>>>>>>> Test case: " + method.getName() + " end >>>>>>>>>>>>>>>>>>");
+    LOG.info(">>>>>>>>>>>>>>>>>>>> Test case: {}.{} end >>>>>>>>>>>>>>>>>>>>",
+        method.getDeclaringClass(),  method.getName());
  }
 }
@@ -80,7 +80,7 @@ public class ExecutionGraphTest extends BaseUnitTest {

  public static JobGraph buildJobGraph() {
    StreamingContext streamingContext = StreamingContext.buildContext();
-    DataStream<String> dataStream = DataStreamSource.buildSource(streamingContext,
+    DataStream<String> dataStream = DataStreamSource.fromCollection(streamingContext,
        Lists.newArrayList("a", "b", "c"));
    StreamSink streamSink = dataStream.sink(x -> LOG.info(x));

@@ -0,0 +1,56 @@
+package io.ray.streaming.runtime.demo;
+
+import io.ray.api.Ray;
+import io.ray.streaming.api.context.StreamingContext;
+import io.ray.streaming.api.function.impl.FilterFunction;
+import io.ray.streaming.api.function.impl.MapFunction;
+import io.ray.streaming.api.stream.DataStreamSource;
+import io.ray.streaming.runtime.BaseUnitTest;
+import java.io.Serializable;
+import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.Test;
+
+public class HybridStreamTest extends BaseUnitTest implements Serializable {
+  private static final Logger LOG = LoggerFactory.getLogger(HybridStreamTest.class);
+
+  public static class Mapper1 implements MapFunction<Object, Object> {
+
+    @Override
+    public Object map(Object value) {
+      LOG.info("HybridStreamTest Mapper1 {}", value);
+      return value.toString();
+    }
+  }
+
+  public static class Filter1 implements FilterFunction<Object> {
+
+    @Override
+    public boolean filter(Object value) throws Exception {
+      LOG.info("HybridStreamTest Filter1 {}", value);
+      return !value.toString().contains("b");
+    }
+  }
+
+  @Test
+  public void testHybridDataStream() throws InterruptedException {
+    Ray.shutdown();
+    StreamingContext context = StreamingContext.buildContext();
+    DataStreamSource<String> streamSource =
+        DataStreamSource.fromCollection(context, Arrays.asList("a", "b", "c"));
+    streamSource
+        .map(x -> x + x)
+        .asPythonStream()
+        .map("ray.streaming.tests.test_hybrid_stream", "map_func1")
+        .filter("ray.streaming.tests.test_hybrid_stream", "filter_func1")
+        .asJavaStream()
+        .sink(x -> System.out.println("HybridStreamTest: " + x));
+    context.execute("HybridStreamTestJob");
+    TimeUnit.SECONDS.sleep(3);
+    context.stop();
+    LOG.info("HybridStreamTest succeed");
+  }
+
+}
@@ -1,6 +1,7 @@
 package io.ray.streaming.runtime.demo;

 import com.google.common.collect.ImmutableMap;
+import io.ray.api.Ray;
 import io.ray.streaming.api.context.StreamingContext;
 import io.ray.streaming.api.function.impl.FlatMapFunction;
 import io.ray.streaming.api.function.impl.ReduceFunction;
@@ -29,6 +30,7 @@ public class WordCountTest extends BaseUnitTest implements Serializable {

  @Test
  public void testWordCount() {
+    Ray.shutdown();
    StreamingContext streamingContext = StreamingContext.buildContext();
    Map<String, String> config = new HashMap<>();
    config.put(Config.STREAMING_BATCH_MAX_COUNT, "1");
@@ -36,7 +38,7 @@ public class WordCountTest extends BaseUnitTest implements Serializable {
    streamingContext.withConfig(config);
    List<String> text = new ArrayList<>();
    text.add("hello world eagle eagle eagle");
-    DataStreamSource<String> streamSource = DataStreamSource.buildSource(streamingContext, text);
+    DataStreamSource<String> streamSource = DataStreamSource.fromCollection(streamingContext, text);
    streamSource
        .flatMap((FlatMapFunction<String, WordAndCount>) (value, collector) -> {
          String[] records = value.split(" ");
@@ -62,6 +64,7 @@ public class WordCountTest extends BaseUnitTest implements Serializable {
      }
    }
    Assert.assertEquals(wordCount, ImmutableMap.of("eagle", 3, "hello", 1));
+    streamingContext.stop();
  }

  private static class WordAndCount implements Serializable {
@@ -3,6 +3,7 @@ package io.ray.streaming.runtime.python;
 import io.ray.streaming.api.stream.StreamSink;
 import io.ray.streaming.jobgraph.JobGraph;
 import io.ray.streaming.jobgraph.JobGraphBuilder;
+import io.ray.streaming.runtime.serialization.MsgPackSerializer;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.List;
@@ -57,7 +57,7 @@ public class TaskAssignerImplTest extends BaseUnitTest {

  public JobGraph buildDataSyncPlan() {
    StreamingContext streamingContext = StreamingContext.buildContext();
-    DataStream<String> dataStream = DataStreamSource.buildSource(streamingContext,
+    DataStream<String> dataStream = DataStreamSource.fromCollection(streamingContext,
        Lists.newArrayList("a", "b", "c"));
    DataStreamSink streamSink = dataStream.sink(LOGGER::info);
    JobGraphBuilder jobGraphBuilder = new JobGraphBuilder(Lists.newArrayList(streamSink));
@@ -0,0 +1,26 @@
+package io.ray.streaming.runtime.serialization;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import io.ray.streaming.message.KeyRecord;
+import io.ray.streaming.message.Record;
+import org.testng.annotations.Test;
+
+public class CrossLangSerializerTest {
+
+  @Test
+  @SuppressWarnings("unchecked")
+  public void testSerialize() {
+    CrossLangSerializer serializer = new CrossLangSerializer();
+    Record record = new Record("value");
+    record.setStream("stream1");
+    assertTrue(EqualsBuilder.reflectionEquals(record,
+      serializer.deserialize(serializer.serialize(record))));
+    KeyRecord keyRecord = new KeyRecord("key", "value");
+    keyRecord.setStream("stream2");
+    assertEquals(keyRecord,
+      serializer.deserialize(serializer.serialize(keyRecord)));
+  }
+}
@@ -1,4 +1,7 @@
-package io.ray.streaming.runtime.python;
+package io.ray.streaming.runtime.serialization;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;

 import java.util.ArrayList;
 import java.util.Arrays;
@@ -6,25 +9,37 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import org.testng.annotations.Test;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;

@SuppressWarnings("unchecked")
 public class MsgPackSerializerTest {

+  @Test
+  public void testSerializeByte() {
+    MsgPackSerializer serializer = new MsgPackSerializer();
+
+    assertEquals(serializer.deserialize(
+        serializer.serialize((byte)1)), (byte)1);
+  }
+
  @Test
  public void testSerialize() {
    MsgPackSerializer serializer = new MsgPackSerializer();

+    assertEquals(serializer.deserialize
+        (serializer.serialize(Short.MAX_VALUE)), Short.MAX_VALUE);
+    assertEquals(serializer.deserialize(
+        serializer.serialize(Integer.MAX_VALUE)), Integer.MAX_VALUE);
+    assertEquals(serializer.deserialize(
+        serializer.serialize(Long.MAX_VALUE)), Long.MAX_VALUE);
+
    Map map = new HashMap();
    List list = new ArrayList<>();
    list.add(null);
    list.add(true);
-    list.add(1);
    list.add(1.0d);
    list.add("str");
    map.put("k1", "value1");
-    map.put("k2", 2);
+    map.put("k2", new HashMap<>());
    map.put("k3", list);
    byte[] bytes = serializer.serialize(map);
    Object o = serializer.deserialize(bytes);
@@ -5,6 +5,7 @@ import io.ray.api.Ray;
 import io.ray.api.RayActor;
 import io.ray.api.options.ActorCreationOptions;
 import io.ray.api.options.ActorCreationOptions.Builder;
+import io.ray.runtime.config.RayConfig;
 import io.ray.streaming.api.context.StreamingContext;
 import io.ray.streaming.api.function.impl.FlatMapFunction;
 import io.ray.streaming.api.function.impl.ReduceFunction;
@@ -67,7 +68,7 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
    System.setProperty("ray.raylet.config.num_workers_per_process_java", "1");
    System.setProperty("ray.run-mode", "CLUSTER");
    System.setProperty("ray.redirect-output", "true");
-    // ray init
+    RayConfig.reset();
    Ray.init();
  }

@@ -142,6 +143,14 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {

  @Test(timeOut = 60000)
  public void testWordCount() {
+    Ray.shutdown();
+    System.setProperty("ray.resources", "CPU:4,RES-A:4");
+    System.setProperty("ray.raylet.config.num_workers_per_process_java", "1");
+
+    System.setProperty("ray.run-mode", "CLUSTER");
+    System.setProperty("ray.redirect-output", "true");
+    // ray init
+    Ray.init();
    LOGGER.info("testWordCount");
    LOGGER.info("StreamingQueueTest.testWordCount run-mode: {}",
        System.getProperty("ray.run-mode"));
@@ -157,7 +166,7 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
    streamingContext.withConfig(config);
    List<String> text = new ArrayList<>();
    text.add("hello world eagle eagle eagle");
-    DataStreamSource<String> streamSource = DataStreamSource.buildSource(streamingContext, text);
+    DataStreamSource<String> streamSource = DataStreamSource.fromCollection(streamingContext, text);
    streamSource
        .flatMap((FlatMapFunction<String, WordAndCount>) (value, collector) -> {
          String[] records = value.split(" ");
@@ -176,7 +185,7 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
          serializeResultToFile(resultFile, wordCount);
        });

-    streamingContext.execute("testWordCount");
+    streamingContext.execute("testSQWordCount");

    Map<String, Integer> checkWordCount =
        (Map<String, Integer>) deserializeResultFromFile(resultFile);