[Java] Rename group id and package name. (#7864)

* Initial

* Change streaming's

* Fix

* Fix

* Fix org_ray

* Fix cpp file name

* Fix streaming

* Fix

* Fix

* Fix testlistening

* Fix missing sth in python

* Fix

* Fix

* Fix SPI

* Fix

* Fix complation

* Fix

* Fix CI

* Fix checkstyle

Fix checkstyle

* Fix streaming tests

* Fix streaming CI

* Fix streaming checkstyle.

* Fix build

* Fix bazel dep

* Fix

* Fix ray checkstyle

* Fix streaming checkstyle

* Fix bazel checkstyle
This commit is contained in:
Qing Wang
2020-04-12 17:59:34 +08:00
committed by GitHub
parent 3061067039
commit 98bfcd53bc
358 changed files with 1717 additions and 1740 deletions
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime.config;
package io.ray.streaming.runtime.config;
import java.io.Serializable;
import javax.accessibility.Accessible;
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime.config;
package io.ray.streaming.runtime.config;
import java.io.Serializable;
import java.util.Map;
@@ -1,6 +1,7 @@
package org.ray.streaming.runtime.config;
package io.ray.streaming.runtime.config;
import com.google.common.base.Preconditions;
import io.ray.streaming.runtime.config.global.CommonConfig;
import java.io.Serializable;
import java.lang.reflect.Method;
import java.util.HashMap;
@@ -8,7 +9,6 @@ import java.util.Map;
import org.aeonbits.owner.Config.DefaultValue;
import org.aeonbits.owner.Config.Key;
import org.aeonbits.owner.ConfigFactory;
import org.ray.streaming.runtime.config.global.CommonConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,8 +1,8 @@
package org.ray.streaming.runtime.config;
package io.ray.streaming.runtime.config;
import io.ray.streaming.runtime.config.master.ResourceConfig;
import java.util.Map;
import org.aeonbits.owner.ConfigFactory;
import org.ray.streaming.runtime.config.master.ResourceConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime.config;
package io.ray.streaming.runtime.config;
import java.util.Map;
import org.slf4j.Logger;
@@ -1,6 +1,6 @@
package org.ray.streaming.runtime.config.global;
package io.ray.streaming.runtime.config.global;
import org.ray.streaming.runtime.config.Config;
import io.ray.streaming.runtime.config.Config;
/**
* Job common config.
@@ -1,6 +1,6 @@
package org.ray.streaming.runtime.config.master;
package io.ray.streaming.runtime.config.master;
import org.ray.streaming.runtime.config.Config;
import io.ray.streaming.runtime.config.Config;
/**
* Job resource management config.
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime.config.types;
package io.ray.streaming.runtime.config.types;
public enum SlotAssignStrategyType {
@@ -1,13 +1,13 @@
package org.ray.streaming.runtime.core.collector;
package io.ray.streaming.runtime.core.collector;
import io.ray.runtime.serializer.Serializer;
import io.ray.streaming.api.collector.Collector;
import io.ray.streaming.api.partition.Partition;
import io.ray.streaming.message.Record;
import io.ray.streaming.runtime.transfer.ChannelID;
import io.ray.streaming.runtime.transfer.DataWriter;
import java.nio.ByteBuffer;
import java.util.Collection;
import org.ray.runtime.serializer.Serializer;
import org.ray.streaming.api.collector.Collector;
import org.ray.streaming.api.partition.Partition;
import org.ray.streaming.message.Record;
import org.ray.streaming.runtime.transfer.ChannelID;
import org.ray.streaming.runtime.transfer.DataWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime.core.command;
package io.ray.streaming.runtime.core.command;
import java.io.Serializable;
@@ -1,11 +1,11 @@
package org.ray.streaming.runtime.core.common;
package io.ray.streaming.runtime.core.common;
import java.io.Serializable;
import java.util.Random;
/**
* Streaming system unique identity base class.
* For example, ${@link org.ray.streaming.runtime.core.resource.ContainerID }
* For example, ${@link io.ray.streaming.runtime.core.resource.ContainerID }
*/
public class AbstractID implements Comparable<AbstractID>, Serializable {
@@ -1,7 +1,7 @@
package org.ray.streaming.runtime.core.graph;
package io.ray.streaming.runtime.core.graph;
import io.ray.streaming.api.partition.Partition;
import java.io.Serializable;
import org.ray.streaming.api.partition.Partition;
/**
* An edge in the physical execution graph.
@@ -1,18 +1,18 @@
package org.ray.streaming.runtime.core.graph;
package io.ray.streaming.runtime.core.graph;
import io.ray.api.BaseActor;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.ray.api.BaseActor;
/**
* Physical execution graph.
*
* <p>Notice: Temporary implementation for now to keep functional. This will be changed to
* {@link org.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph} later when
* {@link io.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph} later when
* new stream task implementation is ready.
*/
public class ExecutionGraph implements Serializable {
@@ -1,11 +1,11 @@
package org.ray.streaming.runtime.core.graph;
package io.ray.streaming.runtime.core.graph;
import io.ray.streaming.api.Language;
import io.ray.streaming.jobgraph.VertexType;
import io.ray.streaming.operator.StreamOperator;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import org.ray.streaming.api.Language;
import org.ray.streaming.jobgraph.VertexType;
import org.ray.streaming.operator.StreamOperator;
/**
* A node in the physical execution graph.
@@ -1,7 +1,7 @@
package org.ray.streaming.runtime.core.graph;
package io.ray.streaming.runtime.core.graph;
import io.ray.api.BaseActor;
import java.io.Serializable;
import org.ray.api.BaseActor;
/**
* ExecutionTask is minimal execution unit.
@@ -1,8 +1,8 @@
package org.ray.streaming.runtime.core.graph.executiongraph;
package io.ray.streaming.runtime.core.graph.executiongraph;
import com.google.common.base.MoreObjects;
import io.ray.streaming.api.partition.Partition;
import java.io.Serializable;
import org.ray.streaming.api.partition.Partition;
/**
* An edge that connects two execution vertices.
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime.core.graph.executiongraph;
package io.ray.streaming.runtime.core.graph.executiongraph;
import java.io.Serializable;
import java.util.ArrayList;
@@ -1,8 +1,8 @@
package org.ray.streaming.runtime.core.graph.executiongraph;
package io.ray.streaming.runtime.core.graph.executiongraph;
import com.google.common.base.MoreObjects;
import org.ray.streaming.api.partition.Partition;
import org.ray.streaming.jobgraph.JobEdge;
import io.ray.streaming.api.partition.Partition;
import io.ray.streaming.jobgraph.JobEdge;
/**
* An edge that connects two execution job vertices.
@@ -1,20 +1,20 @@
package org.ray.streaming.runtime.core.graph.executiongraph;
package io.ray.streaming.runtime.core.graph.executiongraph;
import com.google.common.base.MoreObjects;
import com.google.common.base.Preconditions;
import io.ray.api.RayActor;
import io.ray.streaming.api.Language;
import io.ray.streaming.jobgraph.JobVertex;
import io.ray.streaming.jobgraph.VertexType;
import io.ray.streaming.operator.StreamOperator;
import io.ray.streaming.runtime.config.master.ResourceConfig;
import io.ray.streaming.runtime.worker.JobWorker;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import org.aeonbits.owner.ConfigFactory;
import org.ray.api.RayActor;
import org.ray.streaming.api.Language;
import org.ray.streaming.jobgraph.JobVertex;
import org.ray.streaming.jobgraph.VertexType;
import org.ray.streaming.operator.StreamOperator;
import org.ray.streaming.runtime.config.master.ResourceConfig;
import org.ray.streaming.runtime.worker.JobWorker;
/**
* Physical job vertex.
@@ -1,20 +1,20 @@
package org.ray.streaming.runtime.core.graph.executiongraph;
package io.ray.streaming.runtime.core.graph.executiongraph;
import com.google.common.base.MoreObjects;
import io.ray.api.RayActor;
import io.ray.api.id.ActorId;
import io.ray.streaming.api.Language;
import io.ray.streaming.jobgraph.VertexType;
import io.ray.streaming.operator.StreamOperator;
import io.ray.streaming.runtime.config.master.ResourceConfig;
import io.ray.streaming.runtime.core.resource.ResourceType;
import io.ray.streaming.runtime.core.resource.Slot;
import io.ray.streaming.runtime.worker.JobWorker;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.ray.api.RayActor;
import org.ray.api.id.ActorId;
import org.ray.streaming.api.Language;
import org.ray.streaming.jobgraph.VertexType;
import org.ray.streaming.operator.StreamOperator;
import org.ray.streaming.runtime.config.master.ResourceConfig;
import org.ray.streaming.runtime.core.resource.ResourceType;
import org.ray.streaming.runtime.core.resource.Slot;
import org.ray.streaming.runtime.worker.JobWorker;
/**
* Physical vertex, correspond to {@link ExecutionJobVertex}.
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime.core.graph.executiongraph;
package io.ray.streaming.runtime.core.graph.executiongraph;
import java.io.Serializable;
@@ -1,7 +1,7 @@
package org.ray.streaming.runtime.core.processor;
package io.ray.streaming.runtime.core.processor;
import org.ray.streaming.message.Record;
import org.ray.streaming.operator.OneInputOperator;
import io.ray.streaming.message.Record;
import io.ray.streaming.operator.OneInputOperator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,10 +1,10 @@
package org.ray.streaming.runtime.core.processor;
package io.ray.streaming.runtime.core.processor;
import org.ray.streaming.operator.OneInputOperator;
import org.ray.streaming.operator.OperatorType;
import org.ray.streaming.operator.StreamOperator;
import org.ray.streaming.operator.TwoInputOperator;
import org.ray.streaming.operator.impl.SourceOperator;
import io.ray.streaming.operator.OneInputOperator;
import io.ray.streaming.operator.OperatorType;
import io.ray.streaming.operator.StreamOperator;
import io.ray.streaming.operator.TwoInputOperator;
import io.ray.streaming.operator.impl.SourceOperator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,9 +1,9 @@
package org.ray.streaming.runtime.core.processor;
package io.ray.streaming.runtime.core.processor;
import io.ray.streaming.api.collector.Collector;
import io.ray.streaming.api.context.RuntimeContext;
import java.io.Serializable;
import java.util.List;
import org.ray.streaming.api.collector.Collector;
import org.ray.streaming.api.context.RuntimeContext;
public interface Processor<T> extends Serializable {
@@ -1,7 +1,7 @@
package org.ray.streaming.runtime.core.processor;
package io.ray.streaming.runtime.core.processor;
import org.ray.streaming.message.Record;
import org.ray.streaming.operator.impl.SourceOperator;
import io.ray.streaming.message.Record;
import io.ray.streaming.operator.impl.SourceOperator;
/**
* The processor for the stream sources, containing a SourceOperator.
@@ -1,9 +1,9 @@
package org.ray.streaming.runtime.core.processor;
package io.ray.streaming.runtime.core.processor;
import io.ray.streaming.api.collector.Collector;
import io.ray.streaming.api.context.RuntimeContext;
import io.ray.streaming.operator.Operator;
import java.util.List;
import org.ray.streaming.api.collector.Collector;
import org.ray.streaming.api.context.RuntimeContext;
import org.ray.streaming.operator.Operator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,7 +1,7 @@
package org.ray.streaming.runtime.core.processor;
package io.ray.streaming.runtime.core.processor;
import org.ray.streaming.message.Record;
import org.ray.streaming.operator.TwoInputOperator;
import io.ray.streaming.message.Record;
import io.ray.streaming.operator.TwoInputOperator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,12 +1,12 @@
package org.ray.streaming.runtime.core.resource;
package io.ray.streaming.runtime.core.resource;
import com.google.common.base.MoreObjects;
import io.ray.api.id.UniqueId;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.ray.api.id.UniqueId;
/**
* Resource manager unit abstraction.
@@ -0,0 +1,9 @@
package io.ray.streaming.runtime.core.resource;
import io.ray.streaming.runtime.core.common.AbstractID;
/**
* Container unique identifier.
*/
public class ContainerID extends AbstractID {
}
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime.core.resource;
package io.ray.streaming.runtime.core.resource;
/**
* Key for different type of resources.
@@ -1,13 +1,13 @@
package org.ray.streaming.runtime.core.resource;
package io.ray.streaming.runtime.core.resource;
import com.google.common.base.MoreObjects;
import io.ray.api.id.UniqueId;
import io.ray.streaming.runtime.config.master.ResourceConfig;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.ray.api.id.UniqueId;
import org.ray.streaming.runtime.config.master.ResourceConfig;
/**
* Resource description of ResourceManager.
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime.core.resource;
package io.ray.streaming.runtime.core.resource;
import com.google.common.base.MoreObjects;
import java.io.Serializable;
@@ -1,10 +1,10 @@
package org.ray.streaming.runtime.master;
package io.ray.streaming.runtime.master;
import com.google.common.base.MoreObjects;
import io.ray.streaming.jobgraph.JobGraph;
import io.ray.streaming.runtime.config.StreamingConfig;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
import java.io.Serializable;
import org.ray.streaming.jobgraph.JobGraph;
import org.ray.streaming.runtime.config.StreamingConfig;
import org.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
/**
* Runtime context for job master.
@@ -1,7 +1,7 @@
package org.ray.streaming.runtime.master.graphmanager;
package io.ray.streaming.runtime.master.graphmanager;
import org.ray.streaming.jobgraph.JobGraph;
import org.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
import io.ray.streaming.jobgraph.JobGraph;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
/**
* Graph manager is one of the important roles of JobMaster. It mainly focuses on graph management.
@@ -1,14 +1,14 @@
package org.ray.streaming.runtime.master.graphmanager;
package io.ray.streaming.runtime.master.graphmanager;
import io.ray.streaming.jobgraph.JobGraph;
import io.ray.streaming.jobgraph.JobVertex;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionEdge;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionJobEdge;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionJobVertex;
import io.ray.streaming.runtime.master.JobRuntimeContext;
import java.util.LinkedHashMap;
import java.util.Map;
import org.ray.streaming.jobgraph.JobGraph;
import org.ray.streaming.jobgraph.JobVertex;
import org.ray.streaming.runtime.core.graph.executiongraph.ExecutionEdge;
import org.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
import org.ray.streaming.runtime.core.graph.executiongraph.ExecutionJobEdge;
import org.ray.streaming.runtime.core.graph.executiongraph.ExecutionJobVertex;
import org.ray.streaming.runtime.master.JobRuntimeContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,10 +1,10 @@
package org.ray.streaming.runtime.master.resourcemanager;
package io.ray.streaming.runtime.master.resourcemanager;
import io.ray.streaming.runtime.core.resource.Container;
import io.ray.streaming.runtime.core.resource.Resources;
import io.ray.streaming.runtime.master.scheduler.strategy.SlotAssignStrategy;
import java.util.List;
import java.util.Map;
import org.ray.streaming.runtime.core.resource.Container;
import org.ray.streaming.runtime.core.resource.Resources;
import org.ray.streaming.runtime.master.scheduler.strategy.SlotAssignStrategy;
/**
* The resource manager is responsible for resource de-/allocation and monitoring ray cluster.
@@ -1,5 +1,15 @@
package org.ray.streaming.runtime.master.resourcemanager;
package io.ray.streaming.runtime.master.resourcemanager;
import io.ray.api.Ray;
import io.ray.api.runtimecontext.NodeInfo;
import io.ray.streaming.runtime.config.StreamingMasterConfig;
import io.ray.streaming.runtime.config.master.ResourceConfig;
import io.ray.streaming.runtime.config.types.SlotAssignStrategyType;
import io.ray.streaming.runtime.core.resource.Container;
import io.ray.streaming.runtime.core.resource.Resources;
import io.ray.streaming.runtime.master.JobRuntimeContext;
import io.ray.streaming.runtime.master.scheduler.strategy.SlotAssignStrategy;
import io.ray.streaming.runtime.master.scheduler.strategy.SlotAssignStrategyFactory;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -8,16 +18,6 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import org.ray.api.Ray;
import org.ray.api.runtimecontext.NodeInfo;
import org.ray.streaming.runtime.config.StreamingMasterConfig;
import org.ray.streaming.runtime.config.master.ResourceConfig;
import org.ray.streaming.runtime.config.types.SlotAssignStrategyType;
import org.ray.streaming.runtime.core.resource.Container;
import org.ray.streaming.runtime.core.resource.Resources;
import org.ray.streaming.runtime.master.JobRuntimeContext;
import org.ray.streaming.runtime.master.scheduler.strategy.SlotAssignStrategy;
import org.ray.streaming.runtime.master.scheduler.strategy.SlotAssignStrategyFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,12 +1,12 @@
package org.ray.streaming.runtime.master.scheduler.strategy;
package io.ray.streaming.runtime.master.scheduler.strategy;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
import io.ray.streaming.runtime.core.resource.Container;
import io.ray.streaming.runtime.core.resource.ContainerID;
import io.ray.streaming.runtime.core.resource.Resources;
import io.ray.streaming.runtime.core.resource.Slot;
import java.util.List;
import java.util.Map;
import org.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
import org.ray.streaming.runtime.core.resource.Container;
import org.ray.streaming.runtime.core.resource.ContainerID;
import org.ray.streaming.runtime.core.resource.Resources;
import org.ray.streaming.runtime.core.resource.Slot;
/**
* The SlotAssignStrategy managers a set of slots. When a container is
@@ -1,7 +1,7 @@
package org.ray.streaming.runtime.master.scheduler.strategy;
package io.ray.streaming.runtime.master.scheduler.strategy;
import org.ray.streaming.runtime.config.types.SlotAssignStrategyType;
import org.ray.streaming.runtime.master.scheduler.strategy.impl.PipelineFirstStrategy;
import io.ray.streaming.runtime.config.types.SlotAssignStrategyType;
import io.ray.streaming.runtime.master.scheduler.strategy.impl.PipelineFirstStrategy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,18 +1,18 @@
package org.ray.streaming.runtime.master.scheduler.strategy.impl;
package io.ray.streaming.runtime.master.scheduler.strategy.impl;
import com.google.common.base.Preconditions;
import io.ray.streaming.runtime.config.types.SlotAssignStrategyType;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionJobVertex;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionVertex;
import io.ray.streaming.runtime.core.resource.Container;
import io.ray.streaming.runtime.core.resource.ContainerID;
import io.ray.streaming.runtime.core.resource.Resources;
import io.ray.streaming.runtime.core.resource.Slot;
import io.ray.streaming.runtime.master.scheduler.strategy.SlotAssignStrategy;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.ray.streaming.runtime.config.types.SlotAssignStrategyType;
import org.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
import org.ray.streaming.runtime.core.graph.executiongraph.ExecutionJobVertex;
import org.ray.streaming.runtime.core.graph.executiongraph.ExecutionVertex;
import org.ray.streaming.runtime.core.resource.Container;
import org.ray.streaming.runtime.core.resource.ContainerID;
import org.ray.streaming.runtime.core.resource.Resources;
import org.ray.streaming.runtime.core.resource.Slot;
import org.ray.streaming.runtime.master.scheduler.strategy.SlotAssignStrategy;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,18 +1,18 @@
package org.ray.streaming.runtime.python;
package io.ray.streaming.runtime.python;
import com.google.protobuf.ByteString;
import io.ray.runtime.actor.NativeRayActor;
import io.ray.streaming.api.function.Function;
import io.ray.streaming.api.partition.Partition;
import io.ray.streaming.python.PythonFunction;
import io.ray.streaming.python.PythonPartition;
import io.ray.streaming.runtime.core.graph.ExecutionEdge;
import io.ray.streaming.runtime.core.graph.ExecutionGraph;
import io.ray.streaming.runtime.core.graph.ExecutionNode;
import io.ray.streaming.runtime.core.graph.ExecutionTask;
import io.ray.streaming.runtime.generated.RemoteCall;
import io.ray.streaming.runtime.generated.Streaming;
import java.util.Arrays;
import org.ray.runtime.actor.NativeRayActor;
import org.ray.streaming.api.function.Function;
import org.ray.streaming.api.partition.Partition;
import org.ray.streaming.python.PythonFunction;
import org.ray.streaming.python.PythonPartition;
import org.ray.streaming.runtime.core.graph.ExecutionEdge;
import org.ray.streaming.runtime.core.graph.ExecutionGraph;
import org.ray.streaming.runtime.core.graph.ExecutionNode;
import org.ray.streaming.runtime.core.graph.ExecutionTask;
import org.ray.streaming.runtime.generated.RemoteCall;
import org.ray.streaming.runtime.generated.Streaming;
public class GraphPbBuilder {
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime.python;
package io.ray.streaming.runtime.python;
import com.google.common.io.BaseEncoding;
import java.util.ArrayList;
@@ -1,16 +1,16 @@
package org.ray.streaming.runtime.python;
package io.ray.streaming.runtime.python;
import io.ray.streaming.api.context.StreamingContext;
import io.ray.streaming.python.PythonFunction;
import io.ray.streaming.python.PythonPartition;
import io.ray.streaming.python.stream.PythonStreamSource;
import io.ray.streaming.runtime.util.ReflectionUtils;
import java.lang.reflect.Method;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.msgpack.core.Preconditions;
import org.ray.streaming.api.context.StreamingContext;
import org.ray.streaming.python.PythonFunction;
import org.ray.streaming.python.PythonPartition;
import org.ray.streaming.python.stream.PythonStreamSource;
import org.ray.streaming.runtime.util.ReflectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,24 +1,24 @@
package org.ray.streaming.runtime.schedule;
package io.ray.streaming.runtime.schedule;
import io.ray.api.BaseActor;
import io.ray.api.Ray;
import io.ray.api.RayActor;
import io.ray.api.RayObject;
import io.ray.api.RayPyActor;
import io.ray.api.function.PyActorMethod;
import io.ray.streaming.api.Language;
import io.ray.streaming.jobgraph.JobGraph;
import io.ray.streaming.runtime.core.graph.ExecutionGraph;
import io.ray.streaming.runtime.core.graph.ExecutionNode;
import io.ray.streaming.runtime.core.graph.ExecutionTask;
import io.ray.streaming.runtime.generated.RemoteCall;
import io.ray.streaming.runtime.python.GraphPbBuilder;
import io.ray.streaming.runtime.worker.JobWorker;
import io.ray.streaming.runtime.worker.context.WorkerContext;
import io.ray.streaming.schedule.JobScheduler;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.ray.api.BaseActor;
import org.ray.api.Ray;
import org.ray.api.RayActor;
import org.ray.api.RayObject;
import org.ray.api.RayPyActor;
import org.ray.api.function.PyActorMethod;
import org.ray.streaming.api.Language;
import org.ray.streaming.jobgraph.JobGraph;
import org.ray.streaming.runtime.core.graph.ExecutionGraph;
import org.ray.streaming.runtime.core.graph.ExecutionNode;
import org.ray.streaming.runtime.core.graph.ExecutionTask;
import org.ray.streaming.runtime.generated.RemoteCall;
import org.ray.streaming.runtime.python.GraphPbBuilder;
import org.ray.streaming.runtime.worker.JobWorker;
import org.ray.streaming.runtime.worker.context.WorkerContext;
import org.ray.streaming.schedule.JobScheduler;
/**
* JobSchedulerImpl schedules workers by the Plan and the resource information
@@ -1,8 +1,8 @@
package org.ray.streaming.runtime.schedule;
package io.ray.streaming.runtime.schedule;
import io.ray.streaming.jobgraph.JobGraph;
import io.ray.streaming.runtime.core.graph.ExecutionGraph;
import java.io.Serializable;
import org.ray.streaming.jobgraph.JobGraph;
import org.ray.streaming.runtime.core.graph.ExecutionGraph;
/**
* Interface of the task assigning strategy.
@@ -1,20 +1,20 @@
package org.ray.streaming.runtime.schedule;
package io.ray.streaming.runtime.schedule;
import io.ray.api.BaseActor;
import io.ray.api.Ray;
import io.ray.api.function.PyActorClass;
import io.ray.streaming.jobgraph.JobEdge;
import io.ray.streaming.jobgraph.JobGraph;
import io.ray.streaming.jobgraph.JobVertex;
import io.ray.streaming.runtime.core.graph.ExecutionEdge;
import io.ray.streaming.runtime.core.graph.ExecutionGraph;
import io.ray.streaming.runtime.core.graph.ExecutionNode;
import io.ray.streaming.runtime.core.graph.ExecutionTask;
import io.ray.streaming.runtime.worker.JobWorker;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.ray.api.BaseActor;
import org.ray.api.Ray;
import org.ray.api.function.PyActorClass;
import org.ray.streaming.jobgraph.JobEdge;
import org.ray.streaming.jobgraph.JobGraph;
import org.ray.streaming.jobgraph.JobVertex;
import org.ray.streaming.runtime.core.graph.ExecutionEdge;
import org.ray.streaming.runtime.core.graph.ExecutionGraph;
import org.ray.streaming.runtime.core.graph.ExecutionNode;
import org.ray.streaming.runtime.core.graph.ExecutionTask;
import org.ray.streaming.runtime.worker.JobWorker;
public class TaskAssignerImpl implements TaskAssigner {
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime.transfer;
package io.ray.streaming.runtime.transfer;
import com.google.common.base.FinalizablePhantomReference;
import com.google.common.base.FinalizableReferenceQueue;
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime.transfer;
package io.ray.streaming.runtime.transfer;
import java.util.ArrayList;
import java.util.List;
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime.transfer;
package io.ray.streaming.runtime.transfer;
public class ChannelInterruptException extends RuntimeException {
public ChannelInterruptException() {
@@ -1,8 +1,8 @@
package org.ray.streaming.runtime.transfer;
package io.ray.streaming.runtime.transfer;
import io.ray.streaming.runtime.generated.Streaming;
import io.ray.streaming.util.Config;
import java.util.Map;
import org.ray.streaming.runtime.generated.Streaming;
import org.ray.streaming.util.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime.transfer;
package io.ray.streaming.runtime.transfer;
import java.nio.ByteBuffer;
@@ -1,15 +1,15 @@
package org.ray.streaming.runtime.transfer;
package io.ray.streaming.runtime.transfer;
import com.google.common.base.Preconditions;
import io.ray.api.id.ActorId;
import io.ray.streaming.runtime.util.Platform;
import io.ray.streaming.util.Config;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import org.ray.api.id.ActorId;
import org.ray.streaming.runtime.util.Platform;
import org.ray.streaming.util.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,14 +1,14 @@
package org.ray.streaming.runtime.transfer;
package io.ray.streaming.runtime.transfer;
import com.google.common.base.Preconditions;
import io.ray.api.id.ActorId;
import io.ray.streaming.runtime.util.Platform;
import io.ray.streaming.util.Config;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.ray.api.id.ActorId;
import org.ray.streaming.runtime.util.Platform;
import org.ray.streaming.util.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime.transfer;
package io.ray.streaming.runtime.transfer;
import java.nio.ByteBuffer;
@@ -1,9 +1,9 @@
package org.ray.streaming.runtime.transfer;
package io.ray.streaming.runtime.transfer;
import org.ray.runtime.RayNativeRuntime;
import org.ray.runtime.functionmanager.FunctionDescriptor;
import org.ray.runtime.functionmanager.JavaFunctionDescriptor;
import org.ray.runtime.util.JniUtils;
import io.ray.runtime.RayNativeRuntime;
import io.ray.runtime.functionmanager.FunctionDescriptor;
import io.ray.runtime.functionmanager.JavaFunctionDescriptor;
import io.ray.runtime.util.JniUtils;
/**
* TransferHandler is used for handle direct call based data transfer between workers.
@@ -1,8 +1,8 @@
package org.ray.streaming.runtime.util;
package io.ray.streaming.runtime.util;
import io.ray.runtime.RayNativeRuntime;
import io.ray.runtime.util.JniUtils;
import java.lang.management.ManagementFactory;
import org.ray.runtime.RayNativeRuntime;
import org.ray.runtime.util.JniUtils;
public class EnvUtil {
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime.util;
package io.ray.streaming.runtime.util;
import com.google.common.base.Preconditions;
import java.lang.reflect.Constructor;
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime.util;
package io.ray.streaming.runtime.util;
import com.google.common.base.Preconditions;
import java.lang.reflect.Method;
@@ -1,23 +1,23 @@
package org.ray.streaming.runtime.worker;
package io.ray.streaming.runtime.worker;
import io.ray.runtime.functionmanager.JavaFunctionDescriptor;
import io.ray.streaming.runtime.core.graph.ExecutionGraph;
import io.ray.streaming.runtime.core.graph.ExecutionNode;
import io.ray.streaming.runtime.core.graph.ExecutionNode.NodeType;
import io.ray.streaming.runtime.core.graph.ExecutionTask;
import io.ray.streaming.runtime.core.processor.OneInputProcessor;
import io.ray.streaming.runtime.core.processor.ProcessBuilder;
import io.ray.streaming.runtime.core.processor.SourceProcessor;
import io.ray.streaming.runtime.core.processor.StreamProcessor;
import io.ray.streaming.runtime.transfer.TransferHandler;
import io.ray.streaming.runtime.util.EnvUtil;
import io.ray.streaming.runtime.worker.context.WorkerContext;
import io.ray.streaming.runtime.worker.tasks.OneInputStreamTask;
import io.ray.streaming.runtime.worker.tasks.SourceStreamTask;
import io.ray.streaming.runtime.worker.tasks.StreamTask;
import io.ray.streaming.util.Config;
import java.io.Serializable;
import java.util.Map;
import org.ray.runtime.functionmanager.JavaFunctionDescriptor;
import org.ray.streaming.runtime.core.graph.ExecutionGraph;
import org.ray.streaming.runtime.core.graph.ExecutionNode;
import org.ray.streaming.runtime.core.graph.ExecutionNode.NodeType;
import org.ray.streaming.runtime.core.graph.ExecutionTask;
import org.ray.streaming.runtime.core.processor.OneInputProcessor;
import org.ray.streaming.runtime.core.processor.ProcessBuilder;
import org.ray.streaming.runtime.core.processor.SourceProcessor;
import org.ray.streaming.runtime.core.processor.StreamProcessor;
import org.ray.streaming.runtime.transfer.TransferHandler;
import org.ray.streaming.runtime.util.EnvUtil;
import org.ray.streaming.runtime.worker.context.WorkerContext;
import org.ray.streaming.runtime.worker.tasks.OneInputStreamTask;
import org.ray.streaming.runtime.worker.tasks.SourceStreamTask;
import org.ray.streaming.runtime.worker.tasks.StreamTask;
import org.ray.streaming.util.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,10 +1,10 @@
package org.ray.streaming.runtime.worker.context;
package io.ray.streaming.runtime.worker.context;
import static org.ray.streaming.util.Config.STREAMING_BATCH_MAX_COUNT;
import static io.ray.streaming.util.Config.STREAMING_BATCH_MAX_COUNT;
import io.ray.streaming.api.context.RuntimeContext;
import io.ray.streaming.runtime.core.graph.ExecutionTask;
import java.util.Map;
import org.ray.streaming.api.context.RuntimeContext;
import org.ray.streaming.runtime.core.graph.ExecutionTask;
/**
* Use Ray to implement RuntimeContext.
@@ -1,8 +1,8 @@
package org.ray.streaming.runtime.worker.context;
package io.ray.streaming.runtime.worker.context;
import io.ray.streaming.runtime.core.graph.ExecutionGraph;
import java.io.Serializable;
import java.util.Map;
import org.ray.streaming.runtime.core.graph.ExecutionGraph;
/**
* Encapsulate the context information for worker initialization.
@@ -1,10 +1,10 @@
package org.ray.streaming.runtime.worker.tasks;
package io.ray.streaming.runtime.worker.tasks;
import org.ray.runtime.serializer.Serializer;
import org.ray.streaming.runtime.core.processor.Processor;
import org.ray.streaming.runtime.transfer.Message;
import org.ray.streaming.runtime.worker.JobWorker;
import org.ray.streaming.util.Config;
import io.ray.runtime.serializer.Serializer;
import io.ray.streaming.runtime.core.processor.Processor;
import io.ray.streaming.runtime.transfer.Message;
import io.ray.streaming.runtime.worker.JobWorker;
import io.ray.streaming.util.Config;
public abstract class InputStreamTask extends StreamTask {
private volatile boolean running = true;
@@ -1,7 +1,7 @@
package org.ray.streaming.runtime.worker.tasks;
package io.ray.streaming.runtime.worker.tasks;
import org.ray.streaming.runtime.core.processor.Processor;
import org.ray.streaming.runtime.worker.JobWorker;
import io.ray.streaming.runtime.core.processor.Processor;
import io.ray.streaming.runtime.worker.JobWorker;
public class OneInputStreamTask<IN> extends InputStreamTask {
@@ -1,8 +1,8 @@
package org.ray.streaming.runtime.worker.tasks;
package io.ray.streaming.runtime.worker.tasks;
import org.ray.streaming.runtime.core.processor.Processor;
import org.ray.streaming.runtime.core.processor.SourceProcessor;
import org.ray.streaming.runtime.worker.JobWorker;
import io.ray.streaming.runtime.core.processor.Processor;
import io.ray.streaming.runtime.core.processor.SourceProcessor;
import io.ray.streaming.runtime.worker.JobWorker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,26 +1,26 @@
package org.ray.streaming.runtime.worker.tasks;
package io.ray.streaming.runtime.worker.tasks;
import io.ray.api.BaseActor;
import io.ray.api.Ray;
import io.ray.api.id.ActorId;
import io.ray.streaming.api.collector.Collector;
import io.ray.streaming.api.context.RuntimeContext;
import io.ray.streaming.api.partition.Partition;
import io.ray.streaming.runtime.core.collector.OutputCollector;
import io.ray.streaming.runtime.core.graph.ExecutionEdge;
import io.ray.streaming.runtime.core.graph.ExecutionGraph;
import io.ray.streaming.runtime.core.graph.ExecutionNode;
import io.ray.streaming.runtime.core.processor.Processor;
import io.ray.streaming.runtime.transfer.ChannelID;
import io.ray.streaming.runtime.transfer.DataReader;
import io.ray.streaming.runtime.transfer.DataWriter;
import io.ray.streaming.runtime.worker.JobWorker;
import io.ray.streaming.runtime.worker.context.RayRuntimeContext;
import io.ray.streaming.util.Config;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.ray.api.BaseActor;
import org.ray.api.Ray;
import org.ray.api.id.ActorId;
import org.ray.streaming.api.collector.Collector;
import org.ray.streaming.api.context.RuntimeContext;
import org.ray.streaming.api.partition.Partition;
import org.ray.streaming.runtime.core.collector.OutputCollector;
import org.ray.streaming.runtime.core.graph.ExecutionEdge;
import org.ray.streaming.runtime.core.graph.ExecutionGraph;
import org.ray.streaming.runtime.core.graph.ExecutionNode;
import org.ray.streaming.runtime.core.processor.Processor;
import org.ray.streaming.runtime.transfer.ChannelID;
import org.ray.streaming.runtime.transfer.DataReader;
import org.ray.streaming.runtime.transfer.DataWriter;
import org.ray.streaming.runtime.worker.JobWorker;
import org.ray.streaming.runtime.worker.context.RayRuntimeContext;
import org.ray.streaming.util.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -1,9 +0,0 @@
package org.ray.streaming.runtime.core.resource;
import org.ray.streaming.runtime.core.common.AbstractID;
/**
* Container unique identifier.
*/
public class ContainerID extends AbstractID {
}
@@ -0,0 +1 @@
io.ray.streaming.runtime.schedule.JobSchedulerImpl
@@ -1 +0,0 @@
org.ray.streaming.runtime.schedule.JobSchedulerImpl
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime;
package io.ray.streaming.runtime;
import java.lang.reflect.Method;
import org.slf4j.Logger;
@@ -1,4 +1,4 @@
package org.ray.streaming.runtime;
package io.ray.streaming.runtime;
public class TestHelper {
@@ -1,11 +1,11 @@
package org.ray.streaming.runtime.config;
package io.ray.streaming.runtime.config;
import io.ray.streaming.runtime.BaseUnitTest;
import io.ray.streaming.runtime.config.global.CommonConfig;
import java.util.HashMap;
import java.util.Map;
import org.aeonbits.owner.ConfigFactory;
import org.nustaq.serialization.FSTConfiguration;
import org.ray.streaming.runtime.BaseUnitTest;
import org.ray.streaming.runtime.config.global.CommonConfig;
import org.testng.Assert;
import org.testng.annotations.Test;
@@ -1,13 +1,13 @@
package org.ray.streaming.runtime.demo;
package io.ray.streaming.runtime.demo;
import com.google.common.collect.ImmutableMap;
import org.ray.streaming.api.context.StreamingContext;
import org.ray.streaming.api.function.impl.FlatMapFunction;
import org.ray.streaming.api.function.impl.ReduceFunction;
import org.ray.streaming.api.function.impl.SinkFunction;
import org.ray.streaming.api.stream.DataStreamSource;
import org.ray.streaming.runtime.BaseUnitTest;
import org.ray.streaming.util.Config;
import io.ray.streaming.api.context.StreamingContext;
import io.ray.streaming.api.function.impl.FlatMapFunction;
import io.ray.streaming.api.function.impl.ReduceFunction;
import io.ray.streaming.api.function.impl.SinkFunction;
import io.ray.streaming.api.stream.DataStreamSource;
import io.ray.streaming.runtime.BaseUnitTest;
import io.ray.streaming.util.Config;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
@@ -1,27 +1,25 @@
package org.ray.streaming.runtime.graph;
package io.ray.streaming.runtime.graph;
import com.google.common.collect.Lists;
import io.ray.streaming.api.context.StreamingContext;
import io.ray.streaming.api.stream.DataStream;
import io.ray.streaming.api.stream.DataStreamSource;
import io.ray.streaming.api.stream.StreamSink;
import io.ray.streaming.jobgraph.JobGraph;
import io.ray.streaming.jobgraph.JobGraphBuilder;
import io.ray.streaming.runtime.BaseUnitTest;
import io.ray.streaming.runtime.config.StreamingConfig;
import io.ray.streaming.runtime.config.master.ResourceConfig;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionJobVertex;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionVertex;
import io.ray.streaming.runtime.core.resource.ResourceType;
import io.ray.streaming.runtime.master.JobRuntimeContext;
import io.ray.streaming.runtime.master.graphmanager.GraphManager;
import io.ray.streaming.runtime.master.graphmanager.GraphManagerImpl;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.ray.streaming.api.context.StreamingContext;
import org.ray.streaming.api.stream.DataStream;
import org.ray.streaming.api.stream.DataStreamSource;
import org.ray.streaming.api.stream.StreamSink;
import org.ray.streaming.jobgraph.JobGraph;
import org.ray.streaming.jobgraph.JobGraphBuilder;
import org.ray.streaming.runtime.BaseUnitTest;
import org.ray.streaming.runtime.config.StreamingConfig;
import org.ray.streaming.runtime.config.master.ResourceConfig;
import org.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
import org.ray.streaming.runtime.core.graph.executiongraph.ExecutionJobVertex;
import org.ray.streaming.runtime.core.graph.executiongraph.ExecutionVertex;
import org.ray.streaming.runtime.core.resource.ResourceType;
import org.ray.streaming.runtime.master.JobRuntimeContext;
import org.ray.streaming.runtime.master.graphmanager.GraphManager;
import org.ray.streaming.runtime.master.graphmanager.GraphManagerImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
@@ -1,7 +1,4 @@
package org.ray.streaming.runtime.python;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
package io.ray.streaming.runtime.python;
import java.util.ArrayList;
import java.util.Arrays;
@@ -9,6 +6,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertTrue;
@SuppressWarnings("unchecked")
public class MsgPackSerializerTest {
@@ -1,15 +1,14 @@
package org.ray.streaming.runtime.python;
import static org.testng.Assert.assertEquals;
package io.ray.streaming.runtime.python;
import io.ray.streaming.api.stream.StreamSink;
import io.ray.streaming.jobgraph.JobGraph;
import io.ray.streaming.jobgraph.JobGraphBuilder;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.ray.streaming.api.stream.StreamSink;
import org.ray.streaming.jobgraph.JobGraph;
import org.ray.streaming.jobgraph.JobGraphBuilder;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
public class PythonGatewayTest {
@@ -1,30 +1,29 @@
package org.ray.streaming.runtime.resourcemanager;
package io.ray.streaming.runtime.resourcemanager;
import io.ray.api.Ray;
import io.ray.streaming.jobgraph.JobGraph;
import io.ray.streaming.runtime.BaseUnitTest;
import io.ray.streaming.runtime.TestHelper;
import io.ray.streaming.runtime.config.StreamingConfig;
import io.ray.streaming.runtime.config.global.CommonConfig;
import io.ray.streaming.runtime.config.master.ResourceConfig;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
import io.ray.streaming.runtime.core.resource.Container;
import io.ray.streaming.runtime.core.resource.ContainerID;
import io.ray.streaming.runtime.core.resource.ResourceType;
import io.ray.streaming.runtime.core.resource.Slot;
import io.ray.streaming.runtime.graph.ExecutionGraphTest;
import io.ray.streaming.runtime.master.JobRuntimeContext;
import io.ray.streaming.runtime.master.graphmanager.GraphManager;
import io.ray.streaming.runtime.master.graphmanager.GraphManagerImpl;
import io.ray.streaming.runtime.master.resourcemanager.ResourceManager;
import io.ray.streaming.runtime.master.resourcemanager.ResourceManagerImpl;
import io.ray.streaming.runtime.master.scheduler.strategy.SlotAssignStrategy;
import io.ray.streaming.runtime.master.scheduler.strategy.impl.PipelineFirstStrategy;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.aeonbits.owner.util.Collections;
import org.ray.api.Ray;
import org.ray.streaming.jobgraph.JobGraph;
import org.ray.streaming.runtime.BaseUnitTest;
import org.ray.streaming.runtime.TestHelper;
import org.ray.streaming.runtime.config.StreamingConfig;
import org.ray.streaming.runtime.config.global.CommonConfig;
import org.ray.streaming.runtime.config.master.ResourceConfig;
import org.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
import org.ray.streaming.runtime.core.resource.ResourceType;
import org.ray.streaming.runtime.master.resourcemanager.ResourceManager;
import org.ray.streaming.runtime.master.resourcemanager.ResourceManagerImpl;
import org.ray.streaming.runtime.master.scheduler.strategy.SlotAssignStrategy;
import org.ray.streaming.runtime.master.scheduler.strategy.impl.PipelineFirstStrategy;
import org.ray.streaming.runtime.core.resource.Container;
import org.ray.streaming.runtime.core.resource.ContainerID;
import org.ray.streaming.runtime.core.resource.Slot;
import org.ray.streaming.runtime.graph.ExecutionGraphTest;
import org.ray.streaming.runtime.master.JobRuntimeContext;
import org.ray.streaming.runtime.master.graphmanager.GraphManager;
import org.ray.streaming.runtime.master.graphmanager.GraphManagerImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
@@ -1,20 +1,20 @@
package org.ray.streaming.runtime.schedule;
package io.ray.streaming.runtime.schedule;
import com.google.common.collect.Lists;
import io.ray.api.Ray;
import io.ray.streaming.api.context.StreamingContext;
import io.ray.streaming.api.partition.impl.RoundRobinPartition;
import io.ray.streaming.api.stream.DataStream;
import io.ray.streaming.api.stream.DataStreamSink;
import io.ray.streaming.api.stream.DataStreamSource;
import io.ray.streaming.jobgraph.JobGraph;
import io.ray.streaming.jobgraph.JobGraphBuilder;
import io.ray.streaming.runtime.BaseUnitTest;
import io.ray.streaming.runtime.core.graph.ExecutionEdge;
import io.ray.streaming.runtime.core.graph.ExecutionGraph;
import io.ray.streaming.runtime.core.graph.ExecutionNode;
import io.ray.streaming.runtime.core.graph.ExecutionNode.NodeType;
import java.util.List;
import org.ray.api.Ray;
import org.ray.streaming.api.context.StreamingContext;
import org.ray.streaming.api.partition.impl.RoundRobinPartition;
import org.ray.streaming.api.stream.DataStream;
import org.ray.streaming.api.stream.DataStreamSink;
import org.ray.streaming.api.stream.DataStreamSource;
import org.ray.streaming.jobgraph.JobGraph;
import org.ray.streaming.jobgraph.JobGraphBuilder;
import org.ray.streaming.runtime.BaseUnitTest;
import org.ray.streaming.runtime.core.graph.ExecutionEdge;
import org.ray.streaming.runtime.core.graph.ExecutionGraph;
import org.ray.streaming.runtime.core.graph.ExecutionNode;
import org.ray.streaming.runtime.core.graph.ExecutionNode.NodeType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
@@ -1,30 +1,28 @@
package org.ray.streaming.runtime.schedule.strategy;
package io.ray.streaming.runtime.schedule.strategy;
import io.ray.api.id.UniqueId;
import io.ray.streaming.jobgraph.JobGraph;
import io.ray.streaming.runtime.BaseUnitTest;
import io.ray.streaming.runtime.config.StreamingConfig;
import io.ray.streaming.runtime.config.master.ResourceConfig;
import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
import io.ray.streaming.runtime.core.resource.Container;
import io.ray.streaming.runtime.core.resource.ContainerID;
import io.ray.streaming.runtime.core.resource.ResourceType;
import io.ray.streaming.runtime.core.resource.Resources;
import io.ray.streaming.runtime.core.resource.Slot;
import io.ray.streaming.runtime.graph.ExecutionGraphTest;
import io.ray.streaming.runtime.master.JobRuntimeContext;
import io.ray.streaming.runtime.master.graphmanager.GraphManager;
import io.ray.streaming.runtime.master.graphmanager.GraphManagerImpl;
import io.ray.streaming.runtime.master.scheduler.strategy.SlotAssignStrategy;
import io.ray.streaming.runtime.master.scheduler.strategy.impl.PipelineFirstStrategy;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import org.aeonbits.owner.ConfigFactory;
import org.ray.api.id.UniqueId;
import org.ray.streaming.jobgraph.JobGraph;
import org.ray.streaming.runtime.BaseUnitTest;
import org.ray.streaming.runtime.config.StreamingConfig;
import org.ray.streaming.runtime.config.master.ResourceConfig;
import org.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
import org.ray.streaming.runtime.core.resource.ResourceType;
import org.ray.streaming.runtime.master.scheduler.strategy.SlotAssignStrategy;
import org.ray.streaming.runtime.master.scheduler.strategy.impl.PipelineFirstStrategy;
import org.ray.streaming.runtime.core.resource.Container;
import org.ray.streaming.runtime.core.resource.ContainerID;
import org.ray.streaming.runtime.core.resource.Resources;
import org.ray.streaming.runtime.core.resource.Slot;
import org.ray.streaming.runtime.graph.ExecutionGraphTest;
import org.ray.streaming.runtime.master.JobRuntimeContext;
import org.ray.streaming.runtime.master.graphmanager.GraphManager;
import org.ray.streaming.runtime.master.graphmanager.GraphManagerImpl;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
@@ -1,6 +1,18 @@
package org.ray.streaming.runtime.streamingqueue;
package io.ray.streaming.runtime.streamingqueue;
import com.google.common.collect.ImmutableMap;
import io.ray.api.Ray;
import io.ray.api.RayActor;
import io.ray.api.options.ActorCreationOptions;
import io.ray.api.options.ActorCreationOptions.Builder;
import io.ray.streaming.api.context.StreamingContext;
import io.ray.streaming.api.function.impl.FlatMapFunction;
import io.ray.streaming.api.function.impl.ReduceFunction;
import io.ray.streaming.api.stream.DataStreamSource;
import io.ray.streaming.runtime.BaseUnitTest;
import io.ray.streaming.runtime.transfer.ChannelID;
import io.ray.streaming.runtime.util.EnvUtil;
import io.ray.streaming.util.Config;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -13,18 +25,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.ray.api.Ray;
import org.ray.api.RayActor;
import org.ray.api.options.ActorCreationOptions;
import org.ray.api.options.ActorCreationOptions.Builder;
import org.ray.streaming.api.context.StreamingContext;
import org.ray.streaming.api.function.impl.FlatMapFunction;
import org.ray.streaming.api.function.impl.ReduceFunction;
import org.ray.streaming.api.stream.DataStreamSource;
import org.ray.streaming.runtime.BaseUnitTest;
import org.ray.streaming.runtime.transfer.ChannelID;
import org.ray.streaming.runtime.util.EnvUtil;
import org.ray.streaming.util.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
@@ -146,7 +146,7 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable {
public void testWordCount() {
LOGGER.info("StreamingQueueTest.testWordCount run-mode: {}",
System.getProperty("ray.run-mode"));
String resultFile = "/tmp/org.ray.streaming.runtime.streamingqueue.testWordCount.txt";
String resultFile = "/tmp/io.ray.streaming.runtime.streamingqueue.testWordCount.txt";
deleteResultFile(resultFile);
Map<String, Integer> wordCount = new ConcurrentHashMap<>();
@@ -1,5 +1,15 @@
package org.ray.streaming.runtime.streamingqueue;
package io.ray.streaming.runtime.streamingqueue;
import io.ray.api.Ray;
import io.ray.api.RayActor;
import io.ray.api.id.ActorId;
import io.ray.runtime.functionmanager.JavaFunctionDescriptor;
import io.ray.streaming.runtime.transfer.ChannelID;
import io.ray.streaming.runtime.transfer.DataMessage;
import io.ray.streaming.runtime.transfer.DataReader;
import io.ray.streaming.runtime.transfer.DataWriter;
import io.ray.streaming.runtime.transfer.TransferHandler;
import io.ray.streaming.util.Config;
import java.lang.management.ManagementFactory;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -7,17 +17,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import org.ray.api.Ray;
import org.ray.api.RayActor;
import org.ray.api.id.ActorId;
import org.ray.runtime.actor.NativeRayActor;
import org.ray.runtime.functionmanager.JavaFunctionDescriptor;
import org.ray.streaming.runtime.transfer.ChannelID;
import org.ray.streaming.runtime.transfer.DataMessage;
import org.ray.streaming.runtime.transfer.DataReader;
import org.ray.streaming.runtime.transfer.DataWriter;
import org.ray.streaming.runtime.transfer.TransferHandler;
import org.ray.streaming.util.Config;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
@@ -1,10 +1,9 @@
package org.ray.streaming.runtime.transfer;
package io.ray.streaming.runtime.transfer;
import static org.testng.Assert.assertEquals;
import org.ray.streaming.runtime.BaseUnitTest;
import org.ray.streaming.runtime.util.EnvUtil;
import io.ray.streaming.runtime.BaseUnitTest;
import io.ray.streaming.runtime.util.EnvUtil;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
public class ChannelIDTest extends BaseUnitTest {
@@ -1,10 +1,9 @@
package org.ray.streaming.runtime.util;
import static org.testng.Assert.assertEquals;
package io.ray.streaming.runtime.util;
import java.io.Serializable;
import java.util.Collections;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
public class ReflectionUtilsTest {