[Streaming]Format java code using IDEA (#10440)

This commit is contained in:
Lixin Wei
2020-08-31 19:45:00 +08:00
committed by GitHub
parent afde3db4f0
commit 6bde6b493e
125 changed files with 548 additions and 433 deletions
@@ -17,6 +17,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
class ClusterStarter {
private static final Logger LOG = LoggerFactory.getLogger(ClusterStarter.class);
private static final String PLASMA_STORE_SOCKET_NAME = "/tmp/ray/plasma_store_socket";
private static final String RAYLET_SOCKET_NAME = "/tmp/ray/raylet_socket";
@@ -23,6 +23,7 @@ import org.slf4j.LoggerFactory;
* Encapsulate the context information of a streaming Job.
*/
public class StreamingContext implements Serializable {
private static final Logger LOG = LoggerFactory.getLogger(StreamingContext.class);
private transient AtomicInteger idGenerator;
@@ -3,21 +3,21 @@ package io.ray.streaming.api.function;
import io.ray.streaming.api.context.RuntimeContext;
/**
* An interface for all user-defined functions to define the life cycle methods of the
* functions, and access the task context where the functions get executed.
* An interface for all user-defined functions to define the life cycle methods of the functions,
* and access the task context where the functions get executed.
*/
public interface RichFunction extends Function {
/**
* Initialization method for user function which called before the first call to the user
* function.
*
* @param runtimeContext runtime context
*/
void open(RuntimeContext runtimeContext);
/**
* Tear-down method for the user function which called after the last call to
* the user function.
* Tear-down method for the user function which called after the last call to the user function.
*/
void close();
@@ -3,8 +3,8 @@ package io.ray.streaming.api.function.impl;
import io.ray.streaming.api.function.Function;
/**
* A filter function is a predicate applied individually to each record.
* The predicate decides whether to keep the element, or to discard it.
* A filter function is a predicate applied individually to each record. The predicate decides
* whether to keep the element, or to discard it.
*
* @param <T> type of the input data.
*/
@@ -10,6 +10,7 @@ import io.ray.streaming.api.function.RichFunction;
public class Functions {
private static class DefaultRichFunction implements RichFunction {
private final Function function;
private DefaultRichFunction(Function function) {
@@ -14,7 +14,7 @@ public interface Partition<T> extends Function {
* Given a record and downstream partitions, determine which partition(s) should receive the
* record.
*
* @param record The record.
* @param record The record.
* @param numPartition num of partitions
* @return IDs of the downstream partitions that should receive the record.
*/
@@ -7,6 +7,7 @@ import java.util.stream.IntStream;
* Broadcast the record to all downstream partitions.
*/
public class BroadcastPartition<T> implements Partition<T> {
private int[] partitions = new int[0];
public BroadcastPartition() {
@@ -10,6 +10,7 @@ import io.ray.streaming.api.partition.Partition;
* @param <T> Type of the input record.
*/
public class ForwardPartition<T> implements Partition<T> {
private int[] partitions = new int[] {0};
@Override
@@ -10,6 +10,7 @@ import io.ray.streaming.message.KeyRecord;
* @param <T> Type of the input record.
*/
public class KeyPartition<K, T> implements Partition<KeyRecord<K, T>> {
private int[] partitions = new int[1];
@Override
@@ -8,6 +8,7 @@ import io.ray.streaming.api.partition.Partition;
* @param <T> Type of the input record.
*/
public class RoundRobinPartition<T> implements Partition<T> {
private int seq;
private int[] partitions = new int[1];
@@ -22,7 +22,6 @@ import java.util.List;
/**
* Represents a stream of data.
*
* <p>This class defines all the streaming operations.
*
* @param <T> Type of data in the stream.
@@ -33,9 +32,10 @@ public class DataStream<T> extends Stream<DataStream<T>, T> {
super(streamingContext, streamOperator);
}
public DataStream(StreamingContext streamingContext,
StreamOperator streamOperator,
Partition<T> partition) {
public DataStream(
StreamingContext streamingContext,
StreamOperator streamOperator,
Partition<T> partition) {
super(streamingContext, streamOperator, partition);
}
@@ -43,15 +43,16 @@ public class DataStream<T> extends Stream<DataStream<T>, T> {
super(input, streamOperator);
}
public <R> DataStream(DataStream<R> input,
StreamOperator streamOperator,
Partition<T> partition) {
public <R> DataStream(
DataStream<R> input,
StreamOperator streamOperator,
Partition<T> partition) {
super(input, streamOperator, partition);
}
/**
* Create a java stream that reference passed python stream.
* Changes in new stream will be reflected in referenced stream and vice versa
* Create a java stream that reference passed python stream. Changes in new stream will be
* reflected in referenced stream and vice versa
*/
public DataStream(PythonDataStream referencedStream) {
super(referencedStream);
@@ -84,8 +85,8 @@ public class DataStream<T> extends Stream<DataStream<T>, T> {
}
/**
* Apply union transformations to this stream by merging {@link DataStream} outputs of
* the same type with each other.
* Apply union transformations to this stream by merging {@link DataStream} outputs of the same
* type with each other.
*
* @param stream The DataStream to union output with.
* @param others The other DataStreams to union output with.
@@ -100,8 +101,8 @@ public class DataStream<T> extends Stream<DataStream<T>, T> {
}
/**
* Apply union transformations to this stream by merging {@link DataStream} outputs of
* the same type with each other.
* Apply union transformations to this stream by merging {@link DataStream} outputs of the same
* type with each other.
*
* @param streams The DataStreams to union output with.
* @return A new UnionStream.
@@ -177,8 +178,8 @@ public class DataStream<T> extends Stream<DataStream<T>, T> {
}
/**
* If parent stream is a python stream, we can't call partition related methods
* in the java stream.
* If parent stream is a python stream, we can't call partition related methods in the java
* stream.
*/
private void checkPartitionCall() {
if (getInputStream() != null && getInputStream().getLanguage() == Language.PYTHON) {
@@ -188,9 +189,9 @@ public class DataStream<T> extends Stream<DataStream<T>, T> {
}
/**
* Convert this stream as a python stream.
* The converted stream and this stream are the same logical stream, which has same stream id.
* Changes in converted stream will be reflected in this stream and vice versa.
* Convert this stream as a python stream. The converted stream and this stream are the same
* logical stream, which has same stream id. Changes in converted stream will be reflected in this
* stream and vice versa.
*/
public PythonDataStream asPythonStream() {
return new PythonDataStream(this);
@@ -13,6 +13,7 @@ import java.io.Serializable;
* @param <O> Type of the data in the joined stream.
*/
public class JoinStream<L, R, O> extends DataStream<L> {
private final DataStream<R> rightStream;
public JoinStream(DataStream<L> leftStream, DataStream<R> rightStream) {
@@ -37,6 +38,7 @@ public class JoinStream<L, R, O> extends DataStream<L> {
* @param <K> Type of the join key.
*/
class Where<K> implements Serializable {
private JoinStream<L, R, O> joinStream;
private KeyFunction<L, K> leftKeyByFunction;
@@ -56,12 +58,14 @@ public class JoinStream<L, R, O> extends DataStream<L> {
* @param <K> Type of the join key.
*/
class Equal<K> implements Serializable {
private JoinStream<L, R, O> joinStream;
private KeyFunction<L, K> leftKeyByFunction;
private KeyFunction<R, K> rightKeyByFunction;
Equal(JoinStream<L, R, O> joinStream, KeyFunction<L, K> leftKeyByFunction,
KeyFunction<R, K> rightKeyByFunction) {
Equal(
JoinStream<L, R, O> joinStream, KeyFunction<L, K> leftKeyByFunction,
KeyFunction<R, K> rightKeyByFunction) {
this.joinStream = joinStream;
this.leftKeyByFunction = leftKeyByFunction;
this.rightKeyByFunction = rightKeyByFunction;
@@ -23,8 +23,8 @@ public class KeyDataStream<K, T> extends DataStream<T> {
}
/**
* Create a java stream that reference passed python stream.
* Changes in new stream will be reflected in referenced stream and vice versa
* Create a java stream that reference passed python stream. Changes in new stream will be
* reflected in referenced stream and vice versa
*/
public KeyDataStream(PythonDataStream referencedStream) {
super(referencedStream);
@@ -53,9 +53,9 @@ public class KeyDataStream<K, T> extends DataStream<T> {
}
/**
* Convert this stream as a python stream.
* The converted stream and this stream are the same logical stream, which has same stream id.
* Changes in converted stream will be reflected in this stream and vice versa.
* Convert this stream as a python stream. The converted stream and this stream are the same
* logical stream, which has same stream id. Changes in converted stream will be reflected in this
* stream and vice versa.
*/
public PythonKeyDataStream asPythonStream() {
return new PythonKeyDataStream(this);
@@ -21,6 +21,7 @@ import java.util.Map;
*/
public abstract class Stream<S extends Stream<S, T>, T>
implements Serializable {
private final int id;
private final StreamingContext streamingContext;
private final Stream inputStream;
@@ -34,9 +35,10 @@ public abstract class Stream<S extends Stream<S, T>, T>
this(streamingContext, null, streamOperator, getForwardPartition(streamOperator));
}
public Stream(StreamingContext streamingContext,
StreamOperator streamOperator,
Partition<T> partition) {
public Stream(
StreamingContext streamingContext,
StreamOperator streamOperator,
Partition<T> partition) {
this(streamingContext, null, streamOperator, partition);
}
@@ -49,10 +51,11 @@ public abstract class Stream<S extends Stream<S, T>, T>
this(inputStream.getStreamingContext(), inputStream, streamOperator, partition);
}
protected Stream(StreamingContext streamingContext,
Stream inputStream,
StreamOperator streamOperator,
Partition<T> partition) {
protected Stream(
StreamingContext streamingContext,
Stream inputStream,
StreamOperator streamOperator,
Partition<T> partition) {
this.streamingContext = streamingContext;
this.inputStream = inputStream;
this.operator = streamOperator;
@@ -64,8 +67,8 @@ public abstract class Stream<S extends Stream<S, T>, T>
}
/**
* Create a proxy stream of original stream.
* Changes in new stream will be reflected in original stream and vice versa
* Create a proxy stream of original stream. Changes in new stream will be reflected in original
* stream and vice versa
*/
protected Stream(Stream originalStream) {
this.originalStream = originalStream;
@@ -183,8 +186,8 @@ public abstract class Stream<S extends Stream<S, T>, T>
}
/**
* Set the partition function of this {@link Stream} so that output elements are forwarded to
* next operator locally.
* Set the partition function of this {@link Stream} so that output elements are forwarded to next
* operator locally.
*/
public S forward() {
return setPartition(getForwardPartition(operator));
@@ -8,6 +8,7 @@ import io.ray.streaming.operator.StreamOperator;
* @param <T> Type of the input data of this sink.
*/
public abstract class StreamSink<T> extends Stream<StreamSink<T>, T> {
public StreamSink(Stream inputStream, StreamOperator streamOperator) {
super(inputStream, streamOperator);
}
@@ -6,4 +6,5 @@ package io.ray.streaming.api.stream;
* @param <T> The type of StreamSource data.
*/
public interface StreamSource<T> {
}
@@ -6,13 +6,13 @@ import java.util.List;
/**
* Represents a union DataStream.
*
* <p>This stream does not create a physical operation, it only affects how upstream data are
* connected to downstream data.
* connected to downstream data.
*
* @param <T> The type of union data.
*/
public class UnionStream<T> extends DataStream<T> {
private List<DataStream<T>> unionStreams;
public UnionStream(DataStream<T> input, List<DataStream<T>> streams) {
@@ -30,8 +30,9 @@ public class JobGraph implements Serializable {
this.jobEdges = new ArrayList<>();
}
public JobGraph(String jobName, Map<String, String> jobConfig,
List<JobVertex> jobVertices, List<JobEdge> jobEdges) {
public JobGraph(
String jobName, Map<String, String> jobConfig,
List<JobVertex> jobVertices, List<JobEdge> jobEdges) {
this.jobName = jobName;
this.jobConfig = jobConfig;
this.jobVertices = jobVertices;
@@ -40,8 +41,8 @@ public class JobGraph implements Serializable {
}
/**
* Generate direct-graph(made up of a set of vertices and connected by edges)
* by current job graph for simple log printing.
* Generate direct-graph(made up of a set of vertices and connected by edges) by current job graph
* for simple log printing.
*
* @return Digraph in string type.
*/
@@ -19,6 +19,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class JobGraphBuilder {
private static final Logger LOG = LoggerFactory.getLogger(JobGraphBuilder.class);
private JobGraph jobGraph;
@@ -34,8 +35,9 @@ public class JobGraphBuilder {
this(streamSinkList, jobName, new HashMap<>());
}
public JobGraphBuilder(List<StreamSink> streamSinkList, String jobName,
Map<String, String> jobConfig) {
public JobGraphBuilder(
List<StreamSink> streamSinkList, String jobName,
Map<String, String> jobConfig) {
this.jobGraph = new JobGraph(jobName, jobConfig);
this.streamSinkList = streamSinkList;
this.edgeIdGenerator = new AtomicInteger(0);
@@ -98,7 +100,7 @@ public class JobGraphBuilder {
// process join stream
if (stream instanceof JoinStream) {
DataStream rightStream = ((JoinStream) stream).getRightStream();
DataStream rightStream = ((JoinStream) stream).getRightStream();
this.jobGraph.addEdge(
new JobEdge(rightStream.getId(), vertexId, rightStream.getPartition()));
processStream(rightStream);
@@ -21,10 +21,11 @@ import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.Pair;
/**
* Optimize job graph by chaining some operators so that some operators can be run in the
* same thread.
* Optimize job graph by chaining some operators so that some operators can be run in the same
* thread.
*/
public class JobGraphOptimizer {
private final JobGraph jobGraph;
private Set<JobVertex> visited = new HashSet<>();
// vertex id -> vertex
@@ -89,7 +90,8 @@ public class JobGraphOptimizer {
mergedVertex = headVertex;
} else {
List<StreamOperator> operators = verticesToMerge.stream()
.map(v -> vertexMap.get(v.getVertexId()).getStreamOperator())
.map(v -> vertexMap.get(v.getVertexId())
.getStreamOperator())
.collect(Collectors.toList());
List<Map<String, String>> configs = verticesToMerge.stream()
.map(v -> vertexMap.get(v.getVertexId()).getConfig())
@@ -99,7 +101,8 @@ public class JobGraphOptimizer {
operator = ChainedOperator.newChainedOperator(operators, configs);
} else {
List<PythonOperator> pythonOperators = operators.stream()
.map(o -> (PythonOperator) o).collect(Collectors.toList());
.map(o -> (PythonOperator) o)
.collect(Collectors.toList());
operator = new ChainedPythonOperator(pythonOperators, configs);
}
// chained operator config is placed into `ChainedOperator`.
@@ -156,9 +159,10 @@ public class JobGraphOptimizer {
}
}
private boolean canBeChained(JobVertex precedingVertex,
JobVertex succeedingVertex,
JobEdge edge) {
private boolean canBeChained(
JobVertex precedingVertex,
JobVertex succeedingVertex,
JobEdge edge) {
if (jobGraph.getVertexOutputEdges(precedingVertex.getVertexId()).size() > 1 ||
jobGraph.getVertexInputEdges(succeedingVertex.getVertexId()).size() > 1) {
return false;
@@ -18,11 +18,12 @@ public class JobVertex implements Serializable {
private StreamOperator streamOperator;
private Map<String, String> config;
public JobVertex(int vertexId,
int parallelism,
VertexType vertexType,
StreamOperator streamOperator,
Map<String, String> config) {
public JobVertex(
int vertexId,
int parallelism,
VertexType vertexType,
StreamOperator streamOperator,
Map<String, String> config) {
this.vertexId = vertexId;
this.parallelism = parallelism;
this.vertexType = vertexType;
@@ -4,6 +4,7 @@ import java.io.Serializable;
import java.util.Objects;
public class Record<T> implements Serializable {
protected transient String stream;
protected T value;
@@ -11,6 +11,7 @@ import io.ray.streaming.message.Record;
import java.util.List;
public abstract class StreamOperator<F extends Function> implements Operator {
protected final String name;
protected F function;
protected RichFunction richFunction;
@@ -23,6 +23,7 @@ import java.util.stream.Collectors;
* Abstract base class for chained operators.
*/
public abstract class ChainedOperator extends StreamOperator<Function> {
protected final List<StreamOperator> operators;
protected final Operator headOperator;
protected final Operator tailOperator;
@@ -43,7 +44,8 @@ public abstract class ChainedOperator extends StreamOperator<Function> {
public void open(List<Collector> collectorList, RuntimeContext runtimeContext) {
// Dont' call super.open() as we `open` every operator separately.
List<ForwardCollector> succeedingCollectors = operators.stream().skip(1)
.map(operator -> new ForwardCollector((OneInputOperator) operator))
.map(operator -> new ForwardCollector(
(OneInputOperator) operator))
.collect(Collectors.toList());
for (int i = 0; i < operators.size() - 1; i++) {
StreamOperator operator = operators.get(i);
@@ -113,6 +115,7 @@ public abstract class ChainedOperator extends StreamOperator<Function> {
static class ChainedSourceOperator<T> extends ChainedOperator
implements SourceOperator<T> {
private final SourceOperator<T> sourceOperator;
@SuppressWarnings("unchecked")
@@ -135,6 +138,7 @@ public abstract class ChainedOperator extends StreamOperator<Function> {
static class ChainedOneInputOperator<T> extends ChainedOperator
implements OneInputOperator<T> {
private final OneInputOperator<T> inputOperator;
@SuppressWarnings("unchecked")
@@ -152,6 +156,7 @@ public abstract class ChainedOperator extends StreamOperator<Function> {
static class ChainedTwoInputOperator<L, R> extends ChainedOperator
implements TwoInputOperator<L, R> {
private final TwoInputOperator<L, R> inputOperator;
@SuppressWarnings("unchecked")
@@ -5,6 +5,7 @@ import io.ray.streaming.message.Record;
import io.ray.streaming.operator.OneInputOperator;
class ForwardCollector implements Collector<Record> {
private final OneInputOperator succeedingOperator;
ForwardCollector(OneInputOperator succeedingOperator) {
@@ -17,6 +17,7 @@ import io.ray.streaming.operator.TwoInputOperator;
*/
public class JoinOperator<L, R, K, O> extends StreamOperator<JoinFunction<L, R, O>> implements
TwoInputOperator<L, R> {
public JoinOperator() {
}
@@ -13,6 +13,7 @@ import java.util.List;
public class SourceOperatorImpl<T> extends StreamOperator<SourceFunction<T>>
implements SourceOperator {
private SourceContextImpl sourceContext;
public SourceOperatorImpl(SourceFunction<T> function) {
@@ -47,6 +48,7 @@ public class SourceOperatorImpl<T> extends StreamOperator<SourceFunction<T>>
}
class SourceContextImpl implements SourceContext<T> {
private List<Collector> collectors;
public SourceContextImpl(List<Collector> collectors) {
@@ -7,20 +7,18 @@ import org.apache.commons.lang3.StringUtils;
/**
* Represents a user defined python function.
*
* <p>Python worker can use information in this class to create a function object.</p>
*
* <p>If this object is constructed from serialized python function,
* python worker can deserialize it to create python function directly.
* If this object is constructed from moduleName and className/functionName,
* python worker will use `importlib` to load python function.</p>
*
* python worker can deserialize it to create python function directly. If this object is
* constructed from moduleName and className/functionName, python worker will use `importlib` to
* load python function.</p>
* <p>If the python data stream api is invoked from python, `function` will be not null.</p>
* <p>If the python data stream api is invoked from java, `moduleName` and
* `functionName` will be not null.</p>
* <p>
*/
public class PythonFunction implements Function {
public enum FunctionInterface {
SOURCE_FUNCTION("SourceFunction"),
MAP_FUNCTION("MapFunction"),
@@ -47,8 +45,8 @@ public class PythonFunction implements Function {
// null if this function is constructed from serialized python function.
private final String functionName;
/**
* FunctionInterface can be used to validate python function,
* and look up operator class from FunctionInterface.
* FunctionInterface can be used to validate python function, and look up operator class from
* FunctionInterface.
*/
private String functionInterface;
@@ -69,10 +67,12 @@ public class PythonFunction implements Function {
*
* @param moduleName module name of streaming function.
* @param functionName function name of streaming function. {@code functionName} is the name
* of a python function, or class name of subclass of `ray.streaming.function.`
* of a
* python function, or class name of subclass of `ray.streaming.function.`
*/
public PythonFunction(String moduleName,
String functionName) {
public PythonFunction(
String moduleName,
String functionName) {
Preconditions.checkArgument(StringUtils.isNotBlank(moduleName));
Preconditions.checkArgument(StringUtils.isNotBlank(functionName));
this.function = null;
@@ -17,6 +17,7 @@ import java.util.stream.Collectors;
*/
@SuppressWarnings("unchecked")
public class PythonOperator extends StreamOperator {
private final String moduleName;
private final String className;
@@ -80,7 +81,7 @@ public class PythonOperator extends StreamOperator {
StringBuilder builder = new StringBuilder();
builder.append(PythonOperator.class.getSimpleName()).append("[");
if (function != null) {
builder.append(((PythonFunction)function).toSimpleString());
builder.append(((PythonFunction) function).toSimpleString());
} else {
builder.append(moduleName).append(".").append(className);
}
@@ -101,6 +102,7 @@ public class PythonOperator extends StreamOperator {
}
public static class ChainedPythonOperator extends PythonOperator {
private final List<PythonOperator> operators;
private final PythonOperator headOperator;
private final PythonOperator tailOperator;
@@ -8,16 +8,15 @@ import org.apache.commons.lang3.StringUtils;
/**
* Represents a python partition function.
* <p>
* Python worker can create a partition object using information in this
* PythonPartition.
* Python worker can create a partition object using information in this PythonPartition.
* <p>
* If this object is constructed from serialized python partition,
* python worker can deserialize it to create python partition directly.
* If this object is constructed from moduleName and className/functionName,
* python worker will use `importlib` to load python partition function.
* If this object is constructed from serialized python partition, python worker can deserialize it
* to create python partition directly. If this object is constructed from moduleName and
* className/functionName, python worker will use `importlib` to load python partition function.
* <p>
*/
public class PythonPartition implements Partition<Object> {
public static final PythonPartition BroadcastPartition = new PythonPartition(
"ray.streaming.partition", "BroadcastPartition");
public static final PythonPartition KeyPartition = new PythonPartition(
@@ -18,14 +18,16 @@ import java.util.List;
*/
public class PythonDataStream extends Stream<PythonDataStream, Object> implements PythonStream {
protected PythonDataStream(StreamingContext streamingContext,
PythonOperator pythonOperator) {
protected PythonDataStream(
StreamingContext streamingContext,
PythonOperator pythonOperator) {
super(streamingContext, pythonOperator);
}
protected PythonDataStream(StreamingContext streamingContext,
PythonOperator pythonOperator,
Partition<Object> partition) {
protected PythonDataStream(
StreamingContext streamingContext,
PythonOperator pythonOperator,
Partition<Object> partition) {
super(streamingContext, pythonOperator, partition);
}
@@ -33,15 +35,16 @@ public class PythonDataStream extends Stream<PythonDataStream, Object> implement
super(input, pythonOperator);
}
public PythonDataStream(PythonDataStream input,
PythonOperator pythonOperator,
Partition<Object> partition) {
public PythonDataStream(
PythonDataStream input,
PythonOperator pythonOperator,
Partition<Object> partition) {
super(input, pythonOperator, partition);
}
/**
* Create a python stream that reference passed java stream.
* Changes in new stream will be reflected in referenced stream and vice versa
* Create a python stream that reference passed java stream. Changes in new stream will be
* reflected in referenced stream and vice versa
*/
public PythonDataStream(DataStream referencedStream) {
super(referencedStream);
@@ -85,8 +88,8 @@ public class PythonDataStream extends Stream<PythonDataStream, Object> implement
* Apply a filter function to this stream.
*
* @param func The python FilterFunction.
* @return A new PythonDataStream that contains only the elements satisfying
* the given filter predicate.
* @return A new PythonDataStream that contains only the elements satisfying the given filter
* predicate.
*/
public PythonDataStream filter(PythonFunction func) {
func.setFunctionInterface(FunctionInterface.FILTER_FUNCTION);
@@ -94,8 +97,8 @@ public class PythonDataStream extends Stream<PythonDataStream, Object> implement
}
/**
* Apply union transformations to this stream by merging {@link PythonDataStream} outputs of
* the same type with each other.
* Apply union transformations to this stream by merging {@link PythonDataStream} outputs of the
* same type with each other.
*
* @param stream The DataStream to union output with.
* @param others The other DataStreams to union output with.
@@ -109,8 +112,8 @@ public class PythonDataStream extends Stream<PythonDataStream, Object> implement
}
/**
* Apply union transformations to this stream by merging {@link PythonDataStream} outputs of
* the same type with each other.
* Apply union transformations to this stream by merging {@link PythonDataStream} outputs of the
* same type with each other.
*
* @param streams The DataStreams to union output with.
* @return A new UnionStream.
@@ -178,8 +181,8 @@ public class PythonDataStream extends Stream<PythonDataStream, Object> implement
}
/**
* If parent stream is a python stream, we can't call partition related methods
* in the java stream.
* If parent stream is a python stream, we can't call partition related methods in the java
* stream.
*/
private void checkPartitionCall() {
if (getInputStream() != null && getInputStream().getLanguage() == Language.JAVA) {
@@ -189,9 +192,9 @@ public class PythonDataStream extends Stream<PythonDataStream, Object> implement
}
/**
* Convert this stream as a java stream.
* The converted stream and this stream are the same logical stream, which has same stream id.
* Changes in converted stream will be reflected in this stream and vice versa.
* Convert this stream as a java stream. The converted stream and this stream are the same logical
* stream, which has same stream id. Changes in converted stream will be reflected in this stream
* and vice versa.
*/
public DataStream<Object> asJavaStream() {
return new DataStream<>(this);
@@ -19,8 +19,8 @@ public class PythonKeyDataStream extends PythonDataStream implements PythonStrea
}
/**
* Create a python stream that reference passed python stream.
* Changes in new stream will be reflected in referenced stream and vice versa
* Create a python stream that reference passed python stream. Changes in new stream will be
* reflected in referenced stream and vice versa
*/
public PythonKeyDataStream(DataStream referencedStream) {
super(referencedStream);
@@ -44,9 +44,9 @@ public class PythonKeyDataStream extends PythonDataStream implements PythonStrea
}
/**
* Convert this stream as a java stream.
* The converted stream and this stream are the same logical stream, which has same stream id.
* Changes in converted stream will be reflected in this stream and vice versa.
* Convert this stream as a java stream. The converted stream and this stream are the same logical
* stream, which has same stream id. Changes in converted stream will be reflected in this stream
* and vice versa.
*/
public KeyDataStream<Object, Object> asJavaStream() {
return new KeyDataStream(this);
@@ -4,4 +4,5 @@ package io.ray.streaming.python.stream;
* A marker interface used to identify all python streams.
*/
public interface PythonStream {
}
@@ -8,6 +8,7 @@ import io.ray.streaming.python.PythonOperator;
* Represents a sink of the PythonStream.
*/
public class PythonStreamSink extends StreamSink implements PythonStream {
public PythonStreamSink(PythonDataStream input, PythonOperator sinkOperator) {
super(input, sinkOperator);
getStreamingContext().addSink(this);
@@ -17,8 +17,9 @@ public class PythonStreamSource extends PythonDataStream implements StreamSource
withChainStrategy(ChainStrategy.HEAD);
}
public static PythonStreamSource from(StreamingContext streamingContext,
PythonFunction sourceFunction) {
public static PythonStreamSource from(
StreamingContext streamingContext,
PythonFunction sourceFunction) {
sourceFunction.setFunctionInterface(FunctionInterface.SOURCE_FUNCTION);
return new PythonStreamSource(streamingContext, sourceFunction);
}
@@ -6,11 +6,11 @@ import java.util.List;
/**
* Represents a union DataStream.
*
* <p>This stream does not create a physical operation, it only affects how upstream data are
* connected to downstream data.
* connected to downstream data.
*/
public class PythonUnionStream extends PythonDataStream {
private List<PythonDataStream> unionStreams;
public PythonUnionStream(PythonDataStream input, List<PythonDataStream> others) {
@@ -11,7 +11,7 @@ public class Config {
public static final String MEMORY_CHANNEL = "memory_channel";
public static final String NATIVE_CHANNEL = "native_channel";
public static final String CHANNEL_SIZE = "channel_size";
public static final String CHANNEL_SIZE_DEFAULT = String.valueOf((long)Math.pow(10, 8));
public static final String CHANNEL_SIZE_DEFAULT = String.valueOf((long) Math.pow(10, 8));
public static final String IS_RECREATE = "streaming.is_recreate";
// return from DataReader.getBundle if only empty message read in this interval.
public static final String TIMER_INTERVAL_MS = "timer_interval_ms";
@@ -2,6 +2,7 @@ package io.ray.streaming.api.stream;
import static org.testng.Assert.assertEquals;
import io.ray.streaming.api.context.StreamingContext;
import io.ray.streaming.operator.impl.MapOperator;
import io.ray.streaming.python.stream.PythonDataStream;
@@ -2,6 +2,7 @@ package io.ray.streaming.jobgraph;
import static org.testng.Assert.assertEquals;
import com.google.common.collect.Lists;
import io.ray.streaming.api.context.StreamingContext;
import io.ray.streaming.api.stream.DataStream;
@@ -12,13 +13,14 @@ import org.slf4j.LoggerFactory;
import org.testng.annotations.Test;
public class JobGraphOptimizerTest {
private static final Logger LOG = LoggerFactory.getLogger( JobGraphOptimizerTest.class );
private static final Logger LOG = LoggerFactory.getLogger(JobGraphOptimizerTest.class);
@Test
public void testOptimize() {
StreamingContext context = StreamingContext.buildContext();
DataStream<Integer> source1 = DataStreamSource.fromCollection(context,
Lists.newArrayList(1 ,2 ,3));
Lists.newArrayList(1, 2, 3));
DataStream<String> source2 = DataStreamSource.fromCollection(context,
Lists.newArrayList("1", "2", "3"));
DataStream<String> source3 = DataStreamSource.fromCollection(context,
@@ -43,7 +45,7 @@ public class JobGraphOptimizerTest {
public void testOptimizeHybridStream() {
StreamingContext context = StreamingContext.buildContext();
DataStream<Integer> source1 = DataStreamSource.fromCollection(context,
Lists.newArrayList(1 ,2 ,3));
Lists.newArrayList(1, 2, 3));
DataStream<String> source2 = DataStreamSource.fromCollection(context,
Lists.newArrayList("1", "2", "3"));
source1.asPythonStream()