mirror of
https://github.com/wassname/ray.git
synced 2026-07-09 04:47:52 +08:00
[Streaming]Format java code using IDEA (#10440)
This commit is contained in:
+1
@@ -17,6 +17,7 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
class ClusterStarter {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(ClusterStarter.class);
|
||||
private static final String PLASMA_STORE_SOCKET_NAME = "/tmp/ray/plasma_store_socket";
|
||||
private static final String RAYLET_SOCKET_NAME = "/tmp/ray/raylet_socket";
|
||||
|
||||
+1
@@ -23,6 +23,7 @@ import org.slf4j.LoggerFactory;
|
||||
* Encapsulate the context information of a streaming Job.
|
||||
*/
|
||||
public class StreamingContext implements Serializable {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(StreamingContext.class);
|
||||
|
||||
private transient AtomicInteger idGenerator;
|
||||
|
||||
+4
-4
@@ -3,21 +3,21 @@ package io.ray.streaming.api.function;
|
||||
import io.ray.streaming.api.context.RuntimeContext;
|
||||
|
||||
/**
|
||||
* An interface for all user-defined functions to define the life cycle methods of the
|
||||
* functions, and access the task context where the functions get executed.
|
||||
* An interface for all user-defined functions to define the life cycle methods of the functions,
|
||||
* and access the task context where the functions get executed.
|
||||
*/
|
||||
public interface RichFunction extends Function {
|
||||
|
||||
/**
|
||||
* Initialization method for user function which called before the first call to the user
|
||||
* function.
|
||||
*
|
||||
* @param runtimeContext runtime context
|
||||
*/
|
||||
void open(RuntimeContext runtimeContext);
|
||||
|
||||
/**
|
||||
* Tear-down method for the user function which called after the last call to
|
||||
* the user function.
|
||||
* Tear-down method for the user function which called after the last call to the user function.
|
||||
*/
|
||||
void close();
|
||||
|
||||
|
||||
+2
-2
@@ -3,8 +3,8 @@ package io.ray.streaming.api.function.impl;
|
||||
import io.ray.streaming.api.function.Function;
|
||||
|
||||
/**
|
||||
* A filter function is a predicate applied individually to each record.
|
||||
* The predicate decides whether to keep the element, or to discard it.
|
||||
* A filter function is a predicate applied individually to each record. The predicate decides
|
||||
* whether to keep the element, or to discard it.
|
||||
*
|
||||
* @param <T> type of the input data.
|
||||
*/
|
||||
|
||||
+1
@@ -10,6 +10,7 @@ import io.ray.streaming.api.function.RichFunction;
|
||||
public class Functions {
|
||||
|
||||
private static class DefaultRichFunction implements RichFunction {
|
||||
|
||||
private final Function function;
|
||||
|
||||
private DefaultRichFunction(Function function) {
|
||||
|
||||
+1
-1
@@ -14,7 +14,7 @@ public interface Partition<T> extends Function {
|
||||
* Given a record and downstream partitions, determine which partition(s) should receive the
|
||||
* record.
|
||||
*
|
||||
* @param record The record.
|
||||
* @param record The record.
|
||||
* @param numPartition num of partitions
|
||||
* @return IDs of the downstream partitions that should receive the record.
|
||||
*/
|
||||
|
||||
+1
@@ -7,6 +7,7 @@ import java.util.stream.IntStream;
|
||||
* Broadcast the record to all downstream partitions.
|
||||
*/
|
||||
public class BroadcastPartition<T> implements Partition<T> {
|
||||
|
||||
private int[] partitions = new int[0];
|
||||
|
||||
public BroadcastPartition() {
|
||||
|
||||
+1
@@ -10,6 +10,7 @@ import io.ray.streaming.api.partition.Partition;
|
||||
* @param <T> Type of the input record.
|
||||
*/
|
||||
public class ForwardPartition<T> implements Partition<T> {
|
||||
|
||||
private int[] partitions = new int[] {0};
|
||||
|
||||
@Override
|
||||
|
||||
+1
@@ -10,6 +10,7 @@ import io.ray.streaming.message.KeyRecord;
|
||||
* @param <T> Type of the input record.
|
||||
*/
|
||||
public class KeyPartition<K, T> implements Partition<KeyRecord<K, T>> {
|
||||
|
||||
private int[] partitions = new int[1];
|
||||
|
||||
@Override
|
||||
|
||||
+1
@@ -8,6 +8,7 @@ import io.ray.streaming.api.partition.Partition;
|
||||
* @param <T> Type of the input record.
|
||||
*/
|
||||
public class RoundRobinPartition<T> implements Partition<T> {
|
||||
|
||||
private int seq;
|
||||
private int[] partitions = new int[1];
|
||||
|
||||
|
||||
+19
-18
@@ -22,7 +22,6 @@ import java.util.List;
|
||||
|
||||
/**
|
||||
* Represents a stream of data.
|
||||
*
|
||||
* <p>This class defines all the streaming operations.
|
||||
*
|
||||
* @param <T> Type of data in the stream.
|
||||
@@ -33,9 +32,10 @@ public class DataStream<T> extends Stream<DataStream<T>, T> {
|
||||
super(streamingContext, streamOperator);
|
||||
}
|
||||
|
||||
public DataStream(StreamingContext streamingContext,
|
||||
StreamOperator streamOperator,
|
||||
Partition<T> partition) {
|
||||
public DataStream(
|
||||
StreamingContext streamingContext,
|
||||
StreamOperator streamOperator,
|
||||
Partition<T> partition) {
|
||||
super(streamingContext, streamOperator, partition);
|
||||
}
|
||||
|
||||
@@ -43,15 +43,16 @@ public class DataStream<T> extends Stream<DataStream<T>, T> {
|
||||
super(input, streamOperator);
|
||||
}
|
||||
|
||||
public <R> DataStream(DataStream<R> input,
|
||||
StreamOperator streamOperator,
|
||||
Partition<T> partition) {
|
||||
public <R> DataStream(
|
||||
DataStream<R> input,
|
||||
StreamOperator streamOperator,
|
||||
Partition<T> partition) {
|
||||
super(input, streamOperator, partition);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a java stream that reference passed python stream.
|
||||
* Changes in new stream will be reflected in referenced stream and vice versa
|
||||
* Create a java stream that reference passed python stream. Changes in new stream will be
|
||||
* reflected in referenced stream and vice versa
|
||||
*/
|
||||
public DataStream(PythonDataStream referencedStream) {
|
||||
super(referencedStream);
|
||||
@@ -84,8 +85,8 @@ public class DataStream<T> extends Stream<DataStream<T>, T> {
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply union transformations to this stream by merging {@link DataStream} outputs of
|
||||
* the same type with each other.
|
||||
* Apply union transformations to this stream by merging {@link DataStream} outputs of the same
|
||||
* type with each other.
|
||||
*
|
||||
* @param stream The DataStream to union output with.
|
||||
* @param others The other DataStreams to union output with.
|
||||
@@ -100,8 +101,8 @@ public class DataStream<T> extends Stream<DataStream<T>, T> {
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply union transformations to this stream by merging {@link DataStream} outputs of
|
||||
* the same type with each other.
|
||||
* Apply union transformations to this stream by merging {@link DataStream} outputs of the same
|
||||
* type with each other.
|
||||
*
|
||||
* @param streams The DataStreams to union output with.
|
||||
* @return A new UnionStream.
|
||||
@@ -177,8 +178,8 @@ public class DataStream<T> extends Stream<DataStream<T>, T> {
|
||||
}
|
||||
|
||||
/**
|
||||
* If parent stream is a python stream, we can't call partition related methods
|
||||
* in the java stream.
|
||||
* If parent stream is a python stream, we can't call partition related methods in the java
|
||||
* stream.
|
||||
*/
|
||||
private void checkPartitionCall() {
|
||||
if (getInputStream() != null && getInputStream().getLanguage() == Language.PYTHON) {
|
||||
@@ -188,9 +189,9 @@ public class DataStream<T> extends Stream<DataStream<T>, T> {
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert this stream as a python stream.
|
||||
* The converted stream and this stream are the same logical stream, which has same stream id.
|
||||
* Changes in converted stream will be reflected in this stream and vice versa.
|
||||
* Convert this stream as a python stream. The converted stream and this stream are the same
|
||||
* logical stream, which has same stream id. Changes in converted stream will be reflected in this
|
||||
* stream and vice versa.
|
||||
*/
|
||||
public PythonDataStream asPythonStream() {
|
||||
return new PythonDataStream(this);
|
||||
|
||||
+6
-2
@@ -13,6 +13,7 @@ import java.io.Serializable;
|
||||
* @param <O> Type of the data in the joined stream.
|
||||
*/
|
||||
public class JoinStream<L, R, O> extends DataStream<L> {
|
||||
|
||||
private final DataStream<R> rightStream;
|
||||
|
||||
public JoinStream(DataStream<L> leftStream, DataStream<R> rightStream) {
|
||||
@@ -37,6 +38,7 @@ public class JoinStream<L, R, O> extends DataStream<L> {
|
||||
* @param <K> Type of the join key.
|
||||
*/
|
||||
class Where<K> implements Serializable {
|
||||
|
||||
private JoinStream<L, R, O> joinStream;
|
||||
private KeyFunction<L, K> leftKeyByFunction;
|
||||
|
||||
@@ -56,12 +58,14 @@ public class JoinStream<L, R, O> extends DataStream<L> {
|
||||
* @param <K> Type of the join key.
|
||||
*/
|
||||
class Equal<K> implements Serializable {
|
||||
|
||||
private JoinStream<L, R, O> joinStream;
|
||||
private KeyFunction<L, K> leftKeyByFunction;
|
||||
private KeyFunction<R, K> rightKeyByFunction;
|
||||
|
||||
Equal(JoinStream<L, R, O> joinStream, KeyFunction<L, K> leftKeyByFunction,
|
||||
KeyFunction<R, K> rightKeyByFunction) {
|
||||
Equal(
|
||||
JoinStream<L, R, O> joinStream, KeyFunction<L, K> leftKeyByFunction,
|
||||
KeyFunction<R, K> rightKeyByFunction) {
|
||||
this.joinStream = joinStream;
|
||||
this.leftKeyByFunction = leftKeyByFunction;
|
||||
this.rightKeyByFunction = rightKeyByFunction;
|
||||
|
||||
+5
-5
@@ -23,8 +23,8 @@ public class KeyDataStream<K, T> extends DataStream<T> {
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a java stream that reference passed python stream.
|
||||
* Changes in new stream will be reflected in referenced stream and vice versa
|
||||
* Create a java stream that reference passed python stream. Changes in new stream will be
|
||||
* reflected in referenced stream and vice versa
|
||||
*/
|
||||
public KeyDataStream(PythonDataStream referencedStream) {
|
||||
super(referencedStream);
|
||||
@@ -53,9 +53,9 @@ public class KeyDataStream<K, T> extends DataStream<T> {
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert this stream as a python stream.
|
||||
* The converted stream and this stream are the same logical stream, which has same stream id.
|
||||
* Changes in converted stream will be reflected in this stream and vice versa.
|
||||
* Convert this stream as a python stream. The converted stream and this stream are the same
|
||||
* logical stream, which has same stream id. Changes in converted stream will be reflected in this
|
||||
* stream and vice versa.
|
||||
*/
|
||||
public PythonKeyDataStream asPythonStream() {
|
||||
return new PythonKeyDataStream(this);
|
||||
|
||||
+14
-11
@@ -21,6 +21,7 @@ import java.util.Map;
|
||||
*/
|
||||
public abstract class Stream<S extends Stream<S, T>, T>
|
||||
implements Serializable {
|
||||
|
||||
private final int id;
|
||||
private final StreamingContext streamingContext;
|
||||
private final Stream inputStream;
|
||||
@@ -34,9 +35,10 @@ public abstract class Stream<S extends Stream<S, T>, T>
|
||||
this(streamingContext, null, streamOperator, getForwardPartition(streamOperator));
|
||||
}
|
||||
|
||||
public Stream(StreamingContext streamingContext,
|
||||
StreamOperator streamOperator,
|
||||
Partition<T> partition) {
|
||||
public Stream(
|
||||
StreamingContext streamingContext,
|
||||
StreamOperator streamOperator,
|
||||
Partition<T> partition) {
|
||||
this(streamingContext, null, streamOperator, partition);
|
||||
}
|
||||
|
||||
@@ -49,10 +51,11 @@ public abstract class Stream<S extends Stream<S, T>, T>
|
||||
this(inputStream.getStreamingContext(), inputStream, streamOperator, partition);
|
||||
}
|
||||
|
||||
protected Stream(StreamingContext streamingContext,
|
||||
Stream inputStream,
|
||||
StreamOperator streamOperator,
|
||||
Partition<T> partition) {
|
||||
protected Stream(
|
||||
StreamingContext streamingContext,
|
||||
Stream inputStream,
|
||||
StreamOperator streamOperator,
|
||||
Partition<T> partition) {
|
||||
this.streamingContext = streamingContext;
|
||||
this.inputStream = inputStream;
|
||||
this.operator = streamOperator;
|
||||
@@ -64,8 +67,8 @@ public abstract class Stream<S extends Stream<S, T>, T>
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a proxy stream of original stream.
|
||||
* Changes in new stream will be reflected in original stream and vice versa
|
||||
* Create a proxy stream of original stream. Changes in new stream will be reflected in original
|
||||
* stream and vice versa
|
||||
*/
|
||||
protected Stream(Stream originalStream) {
|
||||
this.originalStream = originalStream;
|
||||
@@ -183,8 +186,8 @@ public abstract class Stream<S extends Stream<S, T>, T>
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the partition function of this {@link Stream} so that output elements are forwarded to
|
||||
* next operator locally.
|
||||
* Set the partition function of this {@link Stream} so that output elements are forwarded to next
|
||||
* operator locally.
|
||||
*/
|
||||
public S forward() {
|
||||
return setPartition(getForwardPartition(operator));
|
||||
|
||||
@@ -8,6 +8,7 @@ import io.ray.streaming.operator.StreamOperator;
|
||||
* @param <T> Type of the input data of this sink.
|
||||
*/
|
||||
public abstract class StreamSink<T> extends Stream<StreamSink<T>, T> {
|
||||
|
||||
public StreamSink(Stream inputStream, StreamOperator streamOperator) {
|
||||
super(inputStream, streamOperator);
|
||||
}
|
||||
|
||||
+1
@@ -6,4 +6,5 @@ package io.ray.streaming.api.stream;
|
||||
* @param <T> The type of StreamSource data.
|
||||
*/
|
||||
public interface StreamSource<T> {
|
||||
|
||||
}
|
||||
|
||||
+2
-2
@@ -6,13 +6,13 @@ import java.util.List;
|
||||
|
||||
/**
|
||||
* Represents a union DataStream.
|
||||
*
|
||||
* <p>This stream does not create a physical operation, it only affects how upstream data are
|
||||
* connected to downstream data.
|
||||
* connected to downstream data.
|
||||
*
|
||||
* @param <T> The type of union data.
|
||||
*/
|
||||
public class UnionStream<T> extends DataStream<T> {
|
||||
|
||||
private List<DataStream<T>> unionStreams;
|
||||
|
||||
public UnionStream(DataStream<T> input, List<DataStream<T>> streams) {
|
||||
|
||||
@@ -30,8 +30,9 @@ public class JobGraph implements Serializable {
|
||||
this.jobEdges = new ArrayList<>();
|
||||
}
|
||||
|
||||
public JobGraph(String jobName, Map<String, String> jobConfig,
|
||||
List<JobVertex> jobVertices, List<JobEdge> jobEdges) {
|
||||
public JobGraph(
|
||||
String jobName, Map<String, String> jobConfig,
|
||||
List<JobVertex> jobVertices, List<JobEdge> jobEdges) {
|
||||
this.jobName = jobName;
|
||||
this.jobConfig = jobConfig;
|
||||
this.jobVertices = jobVertices;
|
||||
@@ -40,8 +41,8 @@ public class JobGraph implements Serializable {
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate direct-graph(made up of a set of vertices and connected by edges)
|
||||
* by current job graph for simple log printing.
|
||||
* Generate direct-graph(made up of a set of vertices and connected by edges) by current job graph
|
||||
* for simple log printing.
|
||||
*
|
||||
* @return Digraph in string type.
|
||||
*/
|
||||
|
||||
+5
-3
@@ -19,6 +19,7 @@ import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
public class JobGraphBuilder {
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(JobGraphBuilder.class);
|
||||
|
||||
private JobGraph jobGraph;
|
||||
@@ -34,8 +35,9 @@ public class JobGraphBuilder {
|
||||
this(streamSinkList, jobName, new HashMap<>());
|
||||
}
|
||||
|
||||
public JobGraphBuilder(List<StreamSink> streamSinkList, String jobName,
|
||||
Map<String, String> jobConfig) {
|
||||
public JobGraphBuilder(
|
||||
List<StreamSink> streamSinkList, String jobName,
|
||||
Map<String, String> jobConfig) {
|
||||
this.jobGraph = new JobGraph(jobName, jobConfig);
|
||||
this.streamSinkList = streamSinkList;
|
||||
this.edgeIdGenerator = new AtomicInteger(0);
|
||||
@@ -98,7 +100,7 @@ public class JobGraphBuilder {
|
||||
|
||||
// process join stream
|
||||
if (stream instanceof JoinStream) {
|
||||
DataStream rightStream = ((JoinStream) stream).getRightStream();
|
||||
DataStream rightStream = ((JoinStream) stream).getRightStream();
|
||||
this.jobGraph.addEdge(
|
||||
new JobEdge(rightStream.getId(), vertexId, rightStream.getPartition()));
|
||||
processStream(rightStream);
|
||||
|
||||
+11
-7
@@ -21,10 +21,11 @@ import java.util.stream.Collectors;
|
||||
import org.apache.commons.lang3.tuple.Pair;
|
||||
|
||||
/**
|
||||
* Optimize job graph by chaining some operators so that some operators can be run in the
|
||||
* same thread.
|
||||
* Optimize job graph by chaining some operators so that some operators can be run in the same
|
||||
* thread.
|
||||
*/
|
||||
public class JobGraphOptimizer {
|
||||
|
||||
private final JobGraph jobGraph;
|
||||
private Set<JobVertex> visited = new HashSet<>();
|
||||
// vertex id -> vertex
|
||||
@@ -89,7 +90,8 @@ public class JobGraphOptimizer {
|
||||
mergedVertex = headVertex;
|
||||
} else {
|
||||
List<StreamOperator> operators = verticesToMerge.stream()
|
||||
.map(v -> vertexMap.get(v.getVertexId()).getStreamOperator())
|
||||
.map(v -> vertexMap.get(v.getVertexId())
|
||||
.getStreamOperator())
|
||||
.collect(Collectors.toList());
|
||||
List<Map<String, String>> configs = verticesToMerge.stream()
|
||||
.map(v -> vertexMap.get(v.getVertexId()).getConfig())
|
||||
@@ -99,7 +101,8 @@ public class JobGraphOptimizer {
|
||||
operator = ChainedOperator.newChainedOperator(operators, configs);
|
||||
} else {
|
||||
List<PythonOperator> pythonOperators = operators.stream()
|
||||
.map(o -> (PythonOperator) o).collect(Collectors.toList());
|
||||
.map(o -> (PythonOperator) o)
|
||||
.collect(Collectors.toList());
|
||||
operator = new ChainedPythonOperator(pythonOperators, configs);
|
||||
}
|
||||
// chained operator config is placed into `ChainedOperator`.
|
||||
@@ -156,9 +159,10 @@ public class JobGraphOptimizer {
|
||||
}
|
||||
}
|
||||
|
||||
private boolean canBeChained(JobVertex precedingVertex,
|
||||
JobVertex succeedingVertex,
|
||||
JobEdge edge) {
|
||||
private boolean canBeChained(
|
||||
JobVertex precedingVertex,
|
||||
JobVertex succeedingVertex,
|
||||
JobEdge edge) {
|
||||
if (jobGraph.getVertexOutputEdges(precedingVertex.getVertexId()).size() > 1 ||
|
||||
jobGraph.getVertexInputEdges(succeedingVertex.getVertexId()).size() > 1) {
|
||||
return false;
|
||||
|
||||
@@ -18,11 +18,12 @@ public class JobVertex implements Serializable {
|
||||
private StreamOperator streamOperator;
|
||||
private Map<String, String> config;
|
||||
|
||||
public JobVertex(int vertexId,
|
||||
int parallelism,
|
||||
VertexType vertexType,
|
||||
StreamOperator streamOperator,
|
||||
Map<String, String> config) {
|
||||
public JobVertex(
|
||||
int vertexId,
|
||||
int parallelism,
|
||||
VertexType vertexType,
|
||||
StreamOperator streamOperator,
|
||||
Map<String, String> config) {
|
||||
this.vertexId = vertexId;
|
||||
this.parallelism = parallelism;
|
||||
this.vertexType = vertexType;
|
||||
|
||||
@@ -4,6 +4,7 @@ import java.io.Serializable;
|
||||
import java.util.Objects;
|
||||
|
||||
public class Record<T> implements Serializable {
|
||||
|
||||
protected transient String stream;
|
||||
protected T value;
|
||||
|
||||
|
||||
+1
@@ -11,6 +11,7 @@ import io.ray.streaming.message.Record;
|
||||
import java.util.List;
|
||||
|
||||
public abstract class StreamOperator<F extends Function> implements Operator {
|
||||
|
||||
protected final String name;
|
||||
protected F function;
|
||||
protected RichFunction richFunction;
|
||||
|
||||
+6
-1
@@ -23,6 +23,7 @@ import java.util.stream.Collectors;
|
||||
* Abstract base class for chained operators.
|
||||
*/
|
||||
public abstract class ChainedOperator extends StreamOperator<Function> {
|
||||
|
||||
protected final List<StreamOperator> operators;
|
||||
protected final Operator headOperator;
|
||||
protected final Operator tailOperator;
|
||||
@@ -43,7 +44,8 @@ public abstract class ChainedOperator extends StreamOperator<Function> {
|
||||
public void open(List<Collector> collectorList, RuntimeContext runtimeContext) {
|
||||
// Dont' call super.open() as we `open` every operator separately.
|
||||
List<ForwardCollector> succeedingCollectors = operators.stream().skip(1)
|
||||
.map(operator -> new ForwardCollector((OneInputOperator) operator))
|
||||
.map(operator -> new ForwardCollector(
|
||||
(OneInputOperator) operator))
|
||||
.collect(Collectors.toList());
|
||||
for (int i = 0; i < operators.size() - 1; i++) {
|
||||
StreamOperator operator = operators.get(i);
|
||||
@@ -113,6 +115,7 @@ public abstract class ChainedOperator extends StreamOperator<Function> {
|
||||
|
||||
static class ChainedSourceOperator<T> extends ChainedOperator
|
||||
implements SourceOperator<T> {
|
||||
|
||||
private final SourceOperator<T> sourceOperator;
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@@ -135,6 +138,7 @@ public abstract class ChainedOperator extends StreamOperator<Function> {
|
||||
|
||||
static class ChainedOneInputOperator<T> extends ChainedOperator
|
||||
implements OneInputOperator<T> {
|
||||
|
||||
private final OneInputOperator<T> inputOperator;
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
@@ -152,6 +156,7 @@ public abstract class ChainedOperator extends StreamOperator<Function> {
|
||||
|
||||
static class ChainedTwoInputOperator<L, R> extends ChainedOperator
|
||||
implements TwoInputOperator<L, R> {
|
||||
|
||||
private final TwoInputOperator<L, R> inputOperator;
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
|
||||
+1
@@ -5,6 +5,7 @@ import io.ray.streaming.message.Record;
|
||||
import io.ray.streaming.operator.OneInputOperator;
|
||||
|
||||
class ForwardCollector implements Collector<Record> {
|
||||
|
||||
private final OneInputOperator succeedingOperator;
|
||||
|
||||
ForwardCollector(OneInputOperator succeedingOperator) {
|
||||
|
||||
+1
@@ -17,6 +17,7 @@ import io.ray.streaming.operator.TwoInputOperator;
|
||||
*/
|
||||
public class JoinOperator<L, R, K, O> extends StreamOperator<JoinFunction<L, R, O>> implements
|
||||
TwoInputOperator<L, R> {
|
||||
|
||||
public JoinOperator() {
|
||||
|
||||
}
|
||||
|
||||
+2
@@ -13,6 +13,7 @@ import java.util.List;
|
||||
|
||||
public class SourceOperatorImpl<T> extends StreamOperator<SourceFunction<T>>
|
||||
implements SourceOperator {
|
||||
|
||||
private SourceContextImpl sourceContext;
|
||||
|
||||
public SourceOperatorImpl(SourceFunction<T> function) {
|
||||
@@ -47,6 +48,7 @@ public class SourceOperatorImpl<T> extends StreamOperator<SourceFunction<T>>
|
||||
}
|
||||
|
||||
class SourceContextImpl implements SourceContext<T> {
|
||||
|
||||
private List<Collector> collectors;
|
||||
|
||||
public SourceContextImpl(List<Collector> collectors) {
|
||||
|
||||
+11
-11
@@ -7,20 +7,18 @@ import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
/**
|
||||
* Represents a user defined python function.
|
||||
*
|
||||
* <p>Python worker can use information in this class to create a function object.</p>
|
||||
*
|
||||
* <p>If this object is constructed from serialized python function,
|
||||
* python worker can deserialize it to create python function directly.
|
||||
* If this object is constructed from moduleName and className/functionName,
|
||||
* python worker will use `importlib` to load python function.</p>
|
||||
*
|
||||
* python worker can deserialize it to create python function directly. If this object is
|
||||
* constructed from moduleName and className/functionName, python worker will use `importlib` to
|
||||
* load python function.</p>
|
||||
* <p>If the python data stream api is invoked from python, `function` will be not null.</p>
|
||||
* <p>If the python data stream api is invoked from java, `moduleName` and
|
||||
* `functionName` will be not null.</p>
|
||||
* <p>
|
||||
*/
|
||||
public class PythonFunction implements Function {
|
||||
|
||||
public enum FunctionInterface {
|
||||
SOURCE_FUNCTION("SourceFunction"),
|
||||
MAP_FUNCTION("MapFunction"),
|
||||
@@ -47,8 +45,8 @@ public class PythonFunction implements Function {
|
||||
// null if this function is constructed from serialized python function.
|
||||
private final String functionName;
|
||||
/**
|
||||
* FunctionInterface can be used to validate python function,
|
||||
* and look up operator class from FunctionInterface.
|
||||
* FunctionInterface can be used to validate python function, and look up operator class from
|
||||
* FunctionInterface.
|
||||
*/
|
||||
private String functionInterface;
|
||||
|
||||
@@ -69,10 +67,12 @@ public class PythonFunction implements Function {
|
||||
*
|
||||
* @param moduleName module name of streaming function.
|
||||
* @param functionName function name of streaming function. {@code functionName} is the name
|
||||
* of a python function, or class name of subclass of `ray.streaming.function.`
|
||||
* of a
|
||||
* python function, or class name of subclass of `ray.streaming.function.`
|
||||
*/
|
||||
public PythonFunction(String moduleName,
|
||||
String functionName) {
|
||||
public PythonFunction(
|
||||
String moduleName,
|
||||
String functionName) {
|
||||
Preconditions.checkArgument(StringUtils.isNotBlank(moduleName));
|
||||
Preconditions.checkArgument(StringUtils.isNotBlank(functionName));
|
||||
this.function = null;
|
||||
|
||||
+3
-1
@@ -17,6 +17,7 @@ import java.util.stream.Collectors;
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
public class PythonOperator extends StreamOperator {
|
||||
|
||||
private final String moduleName;
|
||||
private final String className;
|
||||
|
||||
@@ -80,7 +81,7 @@ public class PythonOperator extends StreamOperator {
|
||||
StringBuilder builder = new StringBuilder();
|
||||
builder.append(PythonOperator.class.getSimpleName()).append("[");
|
||||
if (function != null) {
|
||||
builder.append(((PythonFunction)function).toSimpleString());
|
||||
builder.append(((PythonFunction) function).toSimpleString());
|
||||
} else {
|
||||
builder.append(moduleName).append(".").append(className);
|
||||
}
|
||||
@@ -101,6 +102,7 @@ public class PythonOperator extends StreamOperator {
|
||||
}
|
||||
|
||||
public static class ChainedPythonOperator extends PythonOperator {
|
||||
|
||||
private final List<PythonOperator> operators;
|
||||
private final PythonOperator headOperator;
|
||||
private final PythonOperator tailOperator;
|
||||
|
||||
+5
-6
@@ -8,16 +8,15 @@ import org.apache.commons.lang3.StringUtils;
|
||||
/**
|
||||
* Represents a python partition function.
|
||||
* <p>
|
||||
* Python worker can create a partition object using information in this
|
||||
* PythonPartition.
|
||||
* Python worker can create a partition object using information in this PythonPartition.
|
||||
* <p>
|
||||
* If this object is constructed from serialized python partition,
|
||||
* python worker can deserialize it to create python partition directly.
|
||||
* If this object is constructed from moduleName and className/functionName,
|
||||
* python worker will use `importlib` to load python partition function.
|
||||
* If this object is constructed from serialized python partition, python worker can deserialize it
|
||||
* to create python partition directly. If this object is constructed from moduleName and
|
||||
* className/functionName, python worker will use `importlib` to load python partition function.
|
||||
* <p>
|
||||
*/
|
||||
public class PythonPartition implements Partition<Object> {
|
||||
|
||||
public static final PythonPartition BroadcastPartition = new PythonPartition(
|
||||
"ray.streaming.partition", "BroadcastPartition");
|
||||
public static final PythonPartition KeyPartition = new PythonPartition(
|
||||
|
||||
+24
-21
@@ -18,14 +18,16 @@ import java.util.List;
|
||||
*/
|
||||
public class PythonDataStream extends Stream<PythonDataStream, Object> implements PythonStream {
|
||||
|
||||
protected PythonDataStream(StreamingContext streamingContext,
|
||||
PythonOperator pythonOperator) {
|
||||
protected PythonDataStream(
|
||||
StreamingContext streamingContext,
|
||||
PythonOperator pythonOperator) {
|
||||
super(streamingContext, pythonOperator);
|
||||
}
|
||||
|
||||
protected PythonDataStream(StreamingContext streamingContext,
|
||||
PythonOperator pythonOperator,
|
||||
Partition<Object> partition) {
|
||||
protected PythonDataStream(
|
||||
StreamingContext streamingContext,
|
||||
PythonOperator pythonOperator,
|
||||
Partition<Object> partition) {
|
||||
super(streamingContext, pythonOperator, partition);
|
||||
}
|
||||
|
||||
@@ -33,15 +35,16 @@ public class PythonDataStream extends Stream<PythonDataStream, Object> implement
|
||||
super(input, pythonOperator);
|
||||
}
|
||||
|
||||
public PythonDataStream(PythonDataStream input,
|
||||
PythonOperator pythonOperator,
|
||||
Partition<Object> partition) {
|
||||
public PythonDataStream(
|
||||
PythonDataStream input,
|
||||
PythonOperator pythonOperator,
|
||||
Partition<Object> partition) {
|
||||
super(input, pythonOperator, partition);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a python stream that reference passed java stream.
|
||||
* Changes in new stream will be reflected in referenced stream and vice versa
|
||||
* Create a python stream that reference passed java stream. Changes in new stream will be
|
||||
* reflected in referenced stream and vice versa
|
||||
*/
|
||||
public PythonDataStream(DataStream referencedStream) {
|
||||
super(referencedStream);
|
||||
@@ -85,8 +88,8 @@ public class PythonDataStream extends Stream<PythonDataStream, Object> implement
|
||||
* Apply a filter function to this stream.
|
||||
*
|
||||
* @param func The python FilterFunction.
|
||||
* @return A new PythonDataStream that contains only the elements satisfying
|
||||
* the given filter predicate.
|
||||
* @return A new PythonDataStream that contains only the elements satisfying the given filter
|
||||
* predicate.
|
||||
*/
|
||||
public PythonDataStream filter(PythonFunction func) {
|
||||
func.setFunctionInterface(FunctionInterface.FILTER_FUNCTION);
|
||||
@@ -94,8 +97,8 @@ public class PythonDataStream extends Stream<PythonDataStream, Object> implement
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply union transformations to this stream by merging {@link PythonDataStream} outputs of
|
||||
* the same type with each other.
|
||||
* Apply union transformations to this stream by merging {@link PythonDataStream} outputs of the
|
||||
* same type with each other.
|
||||
*
|
||||
* @param stream The DataStream to union output with.
|
||||
* @param others The other DataStreams to union output with.
|
||||
@@ -109,8 +112,8 @@ public class PythonDataStream extends Stream<PythonDataStream, Object> implement
|
||||
}
|
||||
|
||||
/**
|
||||
* Apply union transformations to this stream by merging {@link PythonDataStream} outputs of
|
||||
* the same type with each other.
|
||||
* Apply union transformations to this stream by merging {@link PythonDataStream} outputs of the
|
||||
* same type with each other.
|
||||
*
|
||||
* @param streams The DataStreams to union output with.
|
||||
* @return A new UnionStream.
|
||||
@@ -178,8 +181,8 @@ public class PythonDataStream extends Stream<PythonDataStream, Object> implement
|
||||
}
|
||||
|
||||
/**
|
||||
* If parent stream is a python stream, we can't call partition related methods
|
||||
* in the java stream.
|
||||
* If parent stream is a python stream, we can't call partition related methods in the java
|
||||
* stream.
|
||||
*/
|
||||
private void checkPartitionCall() {
|
||||
if (getInputStream() != null && getInputStream().getLanguage() == Language.JAVA) {
|
||||
@@ -189,9 +192,9 @@ public class PythonDataStream extends Stream<PythonDataStream, Object> implement
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert this stream as a java stream.
|
||||
* The converted stream and this stream are the same logical stream, which has same stream id.
|
||||
* Changes in converted stream will be reflected in this stream and vice versa.
|
||||
* Convert this stream as a java stream. The converted stream and this stream are the same logical
|
||||
* stream, which has same stream id. Changes in converted stream will be reflected in this stream
|
||||
* and vice versa.
|
||||
*/
|
||||
public DataStream<Object> asJavaStream() {
|
||||
return new DataStream<>(this);
|
||||
|
||||
+5
-5
@@ -19,8 +19,8 @@ public class PythonKeyDataStream extends PythonDataStream implements PythonStrea
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a python stream that reference passed python stream.
|
||||
* Changes in new stream will be reflected in referenced stream and vice versa
|
||||
* Create a python stream that reference passed python stream. Changes in new stream will be
|
||||
* reflected in referenced stream and vice versa
|
||||
*/
|
||||
public PythonKeyDataStream(DataStream referencedStream) {
|
||||
super(referencedStream);
|
||||
@@ -44,9 +44,9 @@ public class PythonKeyDataStream extends PythonDataStream implements PythonStrea
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert this stream as a java stream.
|
||||
* The converted stream and this stream are the same logical stream, which has same stream id.
|
||||
* Changes in converted stream will be reflected in this stream and vice versa.
|
||||
* Convert this stream as a java stream. The converted stream and this stream are the same logical
|
||||
* stream, which has same stream id. Changes in converted stream will be reflected in this stream
|
||||
* and vice versa.
|
||||
*/
|
||||
public KeyDataStream<Object, Object> asJavaStream() {
|
||||
return new KeyDataStream(this);
|
||||
|
||||
+1
@@ -4,4 +4,5 @@ package io.ray.streaming.python.stream;
|
||||
* A marker interface used to identify all python streams.
|
||||
*/
|
||||
public interface PythonStream {
|
||||
|
||||
}
|
||||
|
||||
+1
@@ -8,6 +8,7 @@ import io.ray.streaming.python.PythonOperator;
|
||||
* Represents a sink of the PythonStream.
|
||||
*/
|
||||
public class PythonStreamSink extends StreamSink implements PythonStream {
|
||||
|
||||
public PythonStreamSink(PythonDataStream input, PythonOperator sinkOperator) {
|
||||
super(input, sinkOperator);
|
||||
getStreamingContext().addSink(this);
|
||||
|
||||
+3
-2
@@ -17,8 +17,9 @@ public class PythonStreamSource extends PythonDataStream implements StreamSource
|
||||
withChainStrategy(ChainStrategy.HEAD);
|
||||
}
|
||||
|
||||
public static PythonStreamSource from(StreamingContext streamingContext,
|
||||
PythonFunction sourceFunction) {
|
||||
public static PythonStreamSource from(
|
||||
StreamingContext streamingContext,
|
||||
PythonFunction sourceFunction) {
|
||||
sourceFunction.setFunctionInterface(FunctionInterface.SOURCE_FUNCTION);
|
||||
return new PythonStreamSource(streamingContext, sourceFunction);
|
||||
}
|
||||
|
||||
+2
-2
@@ -6,11 +6,11 @@ import java.util.List;
|
||||
|
||||
/**
|
||||
* Represents a union DataStream.
|
||||
*
|
||||
* <p>This stream does not create a physical operation, it only affects how upstream data are
|
||||
* connected to downstream data.
|
||||
* connected to downstream data.
|
||||
*/
|
||||
public class PythonUnionStream extends PythonDataStream {
|
||||
|
||||
private List<PythonDataStream> unionStreams;
|
||||
|
||||
public PythonUnionStream(PythonDataStream input, List<PythonDataStream> others) {
|
||||
|
||||
@@ -11,7 +11,7 @@ public class Config {
|
||||
public static final String MEMORY_CHANNEL = "memory_channel";
|
||||
public static final String NATIVE_CHANNEL = "native_channel";
|
||||
public static final String CHANNEL_SIZE = "channel_size";
|
||||
public static final String CHANNEL_SIZE_DEFAULT = String.valueOf((long)Math.pow(10, 8));
|
||||
public static final String CHANNEL_SIZE_DEFAULT = String.valueOf((long) Math.pow(10, 8));
|
||||
public static final String IS_RECREATE = "streaming.is_recreate";
|
||||
// return from DataReader.getBundle if only empty message read in this interval.
|
||||
public static final String TIMER_INTERVAL_MS = "timer_interval_ms";
|
||||
|
||||
@@ -2,6 +2,7 @@ package io.ray.streaming.api.stream;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
|
||||
import io.ray.streaming.api.context.StreamingContext;
|
||||
import io.ray.streaming.operator.impl.MapOperator;
|
||||
import io.ray.streaming.python.stream.PythonDataStream;
|
||||
|
||||
+5
-3
@@ -2,6 +2,7 @@ package io.ray.streaming.jobgraph;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
|
||||
import com.google.common.collect.Lists;
|
||||
import io.ray.streaming.api.context.StreamingContext;
|
||||
import io.ray.streaming.api.stream.DataStream;
|
||||
@@ -12,13 +13,14 @@ import org.slf4j.LoggerFactory;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
public class JobGraphOptimizerTest {
|
||||
private static final Logger LOG = LoggerFactory.getLogger( JobGraphOptimizerTest.class );
|
||||
|
||||
private static final Logger LOG = LoggerFactory.getLogger(JobGraphOptimizerTest.class);
|
||||
|
||||
@Test
|
||||
public void testOptimize() {
|
||||
StreamingContext context = StreamingContext.buildContext();
|
||||
DataStream<Integer> source1 = DataStreamSource.fromCollection(context,
|
||||
Lists.newArrayList(1 ,2 ,3));
|
||||
Lists.newArrayList(1, 2, 3));
|
||||
DataStream<String> source2 = DataStreamSource.fromCollection(context,
|
||||
Lists.newArrayList("1", "2", "3"));
|
||||
DataStream<String> source3 = DataStreamSource.fromCollection(context,
|
||||
@@ -43,7 +45,7 @@ public class JobGraphOptimizerTest {
|
||||
public void testOptimizeHybridStream() {
|
||||
StreamingContext context = StreamingContext.buildContext();
|
||||
DataStream<Integer> source1 = DataStreamSource.fromCollection(context,
|
||||
Lists.newArrayList(1 ,2 ,3));
|
||||
Lists.newArrayList(1, 2, 3));
|
||||
DataStream<String> source2 = DataStreamSource.fromCollection(context,
|
||||
Lists.newArrayList("1", "2", "3"));
|
||||
source1.asPythonStream()
|
||||
|
||||
Reference in New Issue
Block a user