diff --git a/bazel/ray.bzl b/bazel/ray.bzl index 169eeab24..53fc0adee 100644 --- a/bazel/ray.bzl +++ b/bazel/ray.bzl @@ -1,5 +1,4 @@ load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_library_public") -load("@com_github_checkstyle_java//checkstyle:checkstyle.bzl", "checkstyle_test") load("@bazel_skylib//rules:copy_file.bzl", "copy_file") load("@bazel_common//tools/maven:pom_file.bzl", "pom_file") @@ -77,14 +76,6 @@ def define_java_module( resources = native.glob([name + "/src/main/resources/**"]) + additional_resources, **kwargs ) - checkstyle_test( - name = "io_ray_ray_" + name + "-checkstyle", - target = ":io_ray_ray_" + name, - config = "//java:checkstyle.xml", - suppressions = "//java:checkstyle-suppressions.xml", - size = "small", - tags = ["checkstyle"], - ) if define_test_lib: test_lib_name = "io_ray_ray_" + name + "_test" pom_file_targets.append(test_lib_name) @@ -93,14 +84,6 @@ def define_java_module( srcs = native.glob([name + "/src/test/java/**/*.java"]), deps = test_deps, ) - checkstyle_test( - name = "io_ray_ray_" + name + "_test-checkstyle", - target = ":io_ray_ray_" + name + "_test", - config = "//java:checkstyle.xml", - suppressions = "//java:checkstyle-suppressions.xml", - size = "small", - tags = ["checkstyle"], - ) pom_file( name = "io_ray_ray_" + name + "_pom", targets = pom_file_targets, diff --git a/bazel/ray_deps_build_all.bzl b/bazel/ray_deps_build_all.bzl index 4d4d3632e..a81480664 100644 --- a/bazel/ray_deps_build_all.bzl +++ b/bazel/ray_deps_build_all.bzl @@ -3,7 +3,6 @@ load("@com_github_ray_project_ray//java:dependencies.bzl", "gen_java_deps") load("@com_github_ray_project_ray//streaming/java:dependencies.bzl", "gen_streaming_java_deps") load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps") load("@com_github_jupp0r_prometheus_cpp//bazel:repositories.bzl", "prometheus_cpp_repositories") -load("@com_github_checkstyle_java//:repo.bzl", "checkstyle_deps") load("@com_github_grpc_grpc//third_party/py:python_configure.bzl", "python_configure") load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_toolchains") @@ -13,7 +12,6 @@ def ray_deps_build_all(): bazel_skylib_workspace() gen_java_deps() gen_streaming_java_deps() - checkstyle_deps() boost_deps() prometheus_cpp_repositories() python_configure(name = "local_config_python") diff --git a/bazel/ray_deps_setup.bzl b/bazel/ray_deps_setup.bzl index 02bda17cc..5a26e9696 100644 --- a/bazel/ray_deps_setup.bzl +++ b/bazel/ray_deps_setup.bzl @@ -131,12 +131,6 @@ def ray_deps_setup(): sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44", ) - auto_http_archive( - name = "com_github_checkstyle_java", - url = "https://github.com/ray-project/checkstyle_java/archive/ef367030d1433877a3360bbfceca18a5d0791bdd.tar.gz", - sha256 = "847d391156d7dcc9424e6a8ba06ff23ea2914c725b18d92028074b2ed8de3da9", - ) - auto_http_archive( # This rule is used by @com_github_nelhage_rules_boost and # declaring it here allows us to avoid patching the latter. diff --git a/ci/travis/format.sh b/ci/travis/format.sh index 7a6b45751..3f4b753f4 100755 --- a/ci/travis/format.sh +++ b/ci/travis/format.sh @@ -49,6 +49,7 @@ FLAKE8_VERSION=$(flake8 --version | head -n 1 | awk '{print $1}') YAPF_VERSION=$(yapf --version | awk '{print $2}') SHELLCHECK_VERSION=$(shellcheck --version | awk '/^version:/ {print $2}') MYPY_VERSION=$(mypy --version | awk '{print $2}') +GOOGLE_JAVA_FORMAT_JAR=/tmp/google-java-format-1.7-all-deps.jar # params: tool name, tool version, required version tool_version_check() { @@ -69,6 +70,15 @@ else echo "WARNING: clang-format is not installed!" fi +if command -v java >/dev/null; then + if [ ! -f "$GOOGLE_JAVA_FORMAT_JAR" ]; then + echo "Java code format tool google-java-format.jar is not installed, start to install it." + wget https://github.com/google/google-java-format/releases/download/google-java-format-1.7/google-java-format-1.7-all-deps.jar -O "$GOOGLE_JAVA_FORMAT_JAR" + fi +else + echo "WARNING:java is not installed, skip format java files!" +fi + if [[ $(flake8 --version) != *"flake8_quotes"* ]]; then echo "WARNING: Ray uses flake8 with flake8_quotes. Might error without it. Install with: pip install flake8-quotes" fi @@ -112,6 +122,18 @@ GIT_LS_EXCLUDES=( ':(exclude)python/ray/cloudpickle/' ) +JAVA_EXCLUDES=( + 'java/api/src/main/java/io/ray/api/ActorCall.java' + 'java/api/src/main/java/io/ray/api/PyActorCall.java' + 'java/api/src/main/java/io/ray/api/RayCall.java' +) + +JAVA_EXCLUDES_REGEX="" +for f in "${JAVA_EXCLUDES[@]}"; do + JAVA_EXCLUDES_REGEX="$JAVA_EXCLUDES_REGEX|(${f//\//\/})" +done +JAVA_EXCLUDES_REGEX=${JAVA_EXCLUDES_REGEX#|} + # TODO(barakmich): This should be cleaned up. I've at least excised the copies # of these arguments to this location, but the long-term answer is to actually # make a flake8 config file @@ -132,7 +154,6 @@ mypy_on_each() { popd } - # Format specified files format_files() { local shell_files=() python_files=() bazel_files=() @@ -205,6 +226,11 @@ format_all() { git ls-files -- '*.cc' '*.h' '*.proto' "${GIT_LS_EXCLUDES[@]}" | xargs -P 5 clang-format -i fi + echo "$(date)" "format java...." + if command -v java >/dev/null & [ -f "$GOOGLE_JAVA_FORMAT_JAR" ]; then + git ls-files -- '*.java' "${GIT_LS_EXCLUDES[@]}" | sed -E "\:$JAVA_EXCLUDES_REGEX:d" | xargs -P 5 java -jar "$GOOGLE_JAVA_FORMAT_JAR" -i + fi + if command -v shellcheck >/dev/null; then local shell_files non_shell_files non_shell_files=($(git ls-files -- ':(exclude)*.sh')) @@ -254,6 +280,12 @@ format_changed() { fi fi + if command -v java >/dev/null & [ -f "$GOOGLE_JAVA_FORMAT_JAR" ]; then + if ! git diff --diff-filter=ACRM --quiet --exit-code "$MERGEBASE" -- '*.java' &>/dev/null; then + git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- '*.java' | sed -E "\:$JAVA_EXCLUDES_REGEX:d" | xargs -P 5 java -jar "$GOOGLE_JAVA_FORMAT_JAR" -i + fi + fi + if command -v shellcheck >/dev/null; then local shell_files non_shell_files non_shell_files=($(git diff --name-only --diff-filter=ACRM "$MERGEBASE" -- ':(exclude)*.sh')) diff --git a/java/api/src/main/java/io/ray/api/ActorHandle.java b/java/api/src/main/java/io/ray/api/ActorHandle.java index 0d9c499c6..682698d9f 100644 --- a/java/api/src/main/java/io/ray/api/ActorHandle.java +++ b/java/api/src/main/java/io/ray/api/ActorHandle.java @@ -1,11 +1,12 @@ package io.ray.api; /** - * A handle to a Java actor.

+ * A handle to a Java actor. * - * A handle can be used to invoke a remote actor method, with the {@code "call"} method. For + *

A handle can be used to invoke a remote actor method, with the {@code "call"} method. For * example: - *

 {@code
+ *
+ * 
{@code
  * class MyActor {
  *   public int echo(int x) {
  *     return x;
@@ -19,11 +20,9 @@ package io.ray.api;
  * Assert.assertEqual(result.get(), 1);
  * }
* - * Note, the {@code "call"} method is defined in {@link ActorCall} interface, with multiple + *

Note, the {@code "call"} method is defined in {@link ActorCall} interface, with multiple * overloaded versions. * * @param The type of the concrete actor class. */ -public interface ActorHandle extends BaseActorHandle, ActorCall { - -} +public interface ActorHandle extends BaseActorHandle, ActorCall {} diff --git a/java/api/src/main/java/io/ray/api/BaseActorHandle.java b/java/api/src/main/java/io/ray/api/BaseActorHandle.java index 07360f723..bd875d075 100644 --- a/java/api/src/main/java/io/ray/api/BaseActorHandle.java +++ b/java/api/src/main/java/io/ray/api/BaseActorHandle.java @@ -3,15 +3,13 @@ package io.ray.api; import io.ray.api.id.ActorId; /** - * A handle to an actor.

+ * A handle to an actor. * - * A handle can be used to invoke a remote actor method. + *

A handle can be used to invoke a remote actor method. */ public interface BaseActorHandle { - /** - * @return The id of this actor. - */ + /** Returns the id of this actor. */ ActorId getId(); /** diff --git a/java/api/src/main/java/io/ray/api/ObjectRef.java b/java/api/src/main/java/io/ray/api/ObjectRef.java index 6b58781dc..23e67f3f5 100644 --- a/java/api/src/main/java/io/ray/api/ObjectRef.java +++ b/java/api/src/main/java/io/ray/api/ObjectRef.java @@ -2,15 +2,14 @@ package io.ray.api; /** * Represents a reference to an object in the object store. + * * @param The object type. */ public interface ObjectRef { /** - * Fetch the object from the object store, this method will block - * until the object is locally available. + * Fetch the object from the object store, this method will block until the object is locally + * available. */ T get(); - } - diff --git a/java/api/src/main/java/io/ray/api/PyActorHandle.java b/java/api/src/main/java/io/ray/api/PyActorHandle.java index 01fb2dc4d..a2d603971 100644 --- a/java/api/src/main/java/io/ray/api/PyActorHandle.java +++ b/java/api/src/main/java/io/ray/api/PyActorHandle.java @@ -1,18 +1,11 @@ package io.ray.api; -/** - * Handle of a Python actor. - */ +/** Handle of a Python actor. */ public interface PyActorHandle extends BaseActorHandle, PyActorCall { - /** - * @return Module name of the Python actor class. - */ + /** Returns the module name of the Python actor class. */ String getModuleName(); - /** - * @return Name of the Python actor class. - */ + /** Returns the name of the Python actor class. */ String getClassName(); } - diff --git a/java/api/src/main/java/io/ray/api/Ray.java b/java/api/src/main/java/io/ray/api/Ray.java index 7763fae43..3100d60a1 100644 --- a/java/api/src/main/java/io/ray/api/Ray.java +++ b/java/api/src/main/java/io/ray/api/Ray.java @@ -12,16 +12,12 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.Callable; -/** - * This class contains all public APIs of Ray. - */ +/** This class contains all public APIs of Ray. */ public final class Ray extends RayCall { private static RayRuntime runtime = null; - /** - * Initialize Ray runtime with the default runtime implementation. - */ + /** Initialize Ray runtime with the default runtime implementation. */ public static void init() { try { Class clz = Class.forName("io.ray.runtime.DefaultRayRuntimeFactory"); @@ -30,7 +26,6 @@ public final class Ray extends RayCall { } catch (Exception e) { throw new RuntimeException("Failed to initialize Ray runtime.", e); } - } /** @@ -45,9 +40,7 @@ public final class Ray extends RayCall { } } - /** - * Shutdown Ray runtime. - */ + /** Shutdown Ray runtime. */ public static synchronized void shutdown() { if (runtime != null) { internal().shutdown(); @@ -57,7 +50,8 @@ public final class Ray extends RayCall { /** * Check if {@link #init} has been called yet. - * @return True if {@link #init} has already been called and false otherwise. + * + *

Returns True if {@link #init} has already been called and false otherwise. */ public static boolean isInitialized() { return runtime != null; @@ -66,8 +60,8 @@ public final class Ray extends RayCall { /** * Store an object in the object store. * - * @param obj The Java object to be stored. - * @return A ObjectRef instance that represents the in-store object. + * @param obj The Java object to be stored. Returns A ObjectRef instance that represents the + * in-store object. */ public static ObjectRef put(T obj) { return internal().put(obj); @@ -76,8 +70,7 @@ public final class Ray extends RayCall { /** * Get an object by `ObjectRef` from the object store. * - * @param objectRef The reference of the object to get. - * @return The Java object. + * @param objectRef The reference of the object to get. Returns The Java object. */ public static T get(ObjectRef objectRef) { return internal().get(objectRef); @@ -86,45 +79,43 @@ public final class Ray extends RayCall { /** * Get a list of objects by `ObjectRef`s from the object store. * - * @param objectList A list of object references. - * @return A list of Java objects. + * @param objectList A list of object references. Returns A list of Java objects. */ public static List get(List> objectList) { return internal().get(objectList); } /** - * Wait for a list of RayObjects to be locally available, - * until specified number of objects are ready, or specified timeout has passed. + * Wait for a list of RayObjects to be locally available, until specified number of objects are + * ready, or specified timeout has passed. * * @param waitList A list of object references to wait for. * @param numReturns The number of objects that should be returned. - * @param timeoutMs The maximum time in milliseconds to wait before returning. - * @return Two lists, one containing locally available objects, one containing the rest. + * @param timeoutMs The maximum time in milliseconds to wait before returning. Returns Two lists, + * one containing locally available objects, one containing the rest. */ - public static WaitResult wait(List> waitList, int numReturns, - int timeoutMs) { + public static WaitResult wait(List> waitList, int numReturns, int timeoutMs) { return internal().wait(waitList, numReturns, timeoutMs); } /** - * A convenient helper method for Ray.wait. It will wait infinitely until - * specified number of objects are locally available. + * A convenient helper method for Ray.wait. It will wait infinitely until specified number of + * objects are locally available. * * @param waitList A list of object references to wait for. - * @param numReturns The number of objects that should be returned. - * @return Two lists, one containing locally available objects, one containing the rest. + * @param numReturns The number of objects that should be returned. Returns Two lists, one + * containing locally available objects, one containing the rest. */ public static WaitResult wait(List> waitList, int numReturns) { return internal().wait(waitList, numReturns, Integer.MAX_VALUE); } /** - * A convenient helper method for Ray.wait. It will wait infinitely until - * all objects are locally available. + * A convenient helper method for Ray.wait. It will wait infinitely until all objects are locally + * available. * - * @param waitList A list of object references to wait for. - * @return Two lists, one containing locally available objects, one containing the rest. + * @param waitList A list of object references to wait for. Returns Two lists, one containing + * locally available objects, one containing the rest. */ public static WaitResult wait(List> waitList) { return internal().wait(waitList, waitList.size(), Integer.MAX_VALUE); @@ -132,13 +123,12 @@ public final class Ray extends RayCall { /** * Get a handle to a named actor of current job. - *

- * Gets a handle to a named actor with the given name. The actor must - * have been created with name specified. * - * @param name The name of the named actor. - * @return an ActorHandle to the actor if the actor of specified name exists or an - * Optional.empty() + *

Gets a handle to a named actor with the given name. The actor must have been created with + * name specified. + * + * @param name The name of the named actor. Returns an ActorHandle to the actor if the actor of + * specified name exists or an Optional.empty() */ public static Optional getActor(String name) { return internal().getActor(name, false); @@ -146,13 +136,12 @@ public final class Ray extends RayCall { /** * Get a handle to a global named actor. - *

- * Gets a handle to a global named actor with the given name. The actor must - * have been created with global name specified. * - * @param name The global name of the named actor. - * @return an ActorHandle to the actor if the actor of specified name exists or an - * Optional.empty() + *

Gets a handle to a global named actor with the given name. The actor must have been created + * with global name specified. + * + * @param name The global name of the named actor. Returns an ActorHandle to the actor if the + * actor of specified name exists or an Optional.empty() */ public static Optional getGlobalActor(String name) { return internal().getActor(name, true); @@ -162,7 +151,7 @@ public final class Ray extends RayCall { * If users want to use Ray API in their own threads, call this method to get the async context * and then call {@link #setAsyncContext} at the beginning of the new thread. * - * @return The async context. + *

Returns The async context. */ public static Object getAsyncContext() { return internal().getAsyncContext(); @@ -186,8 +175,7 @@ public final class Ray extends RayCall { * If users want to use Ray API in their own threads, they should wrap their {@link Runnable} * objects with this method. * - * @param runnable The runnable to wrap. - * @return The wrapped runnable. + * @param runnable The runnable to wrap. Returns The wrapped runnable. */ public static Runnable wrapRunnable(Runnable runnable) { return internal().wrapRunnable(runnable); @@ -197,16 +185,13 @@ public final class Ray extends RayCall { * If users want to use Ray API in their own threads, they should wrap their {@link Callable} * objects with this method. * - * @param callable The callable to wrap. - * @return The wrapped callable. + * @param callable The callable to wrap. Returns The wrapped callable. */ public static Callable wrapCallable(Callable callable) { return internal().wrapCallable(callable); } - /** - * Get the underlying runtime instance. - */ + /** Get the underlying runtime instance. */ public static RayRuntime internal() { if (runtime == null) { throw new IllegalStateException( @@ -215,59 +200,49 @@ public final class Ray extends RayCall { return runtime; } - /** - * Update the resource for the specified client. - * Set the resource for the specific node. - */ + /** Update the resource for the specified client. Set the resource for the specific node. */ public static void setResource(UniqueId nodeId, String resourceName, double capacity) { internal().setResource(resourceName, capacity, nodeId); } - /** - * Set the resource for local node. - */ + /** Set the resource for local node. */ public static void setResource(String resourceName, double capacity) { internal().setResource(resourceName, capacity, UniqueId.NIL); } - /** - * Get the runtime context. - */ + /** Get the runtime context. */ public static RuntimeContext getRuntimeContext() { return internal().getRuntimeContext(); } /** - * Create a placement group. - * A placement group is used to place actors according to a specific strategy - * and resource constraints. - * It will sends a request to GCS to preallocate the specified resources, which is asynchronous. - * If the specified resource cannot be allocated, it will wait for the resource - * to be updated and rescheduled. - * This function only works when gcs actor manager is turned on. + * Create a placement group. A placement group is used to place actors according to a specific + * strategy and resource constraints. It will sends a request to GCS to preallocate the specified + * resources, which is asynchronous. If the specified resource cannot be allocated, it will wait + * for the resource to be updated and rescheduled. This function only works when gcs actor manager + * is turned on. * * @param name Name of the placement group. * @param bundles Pre-allocated resource list. - * @param strategy Actor placement strategy. - * @return A handle to the created placement group. + * @param strategy Actor placement strategy. Returns A handle to the created placement group. */ - public static PlacementGroup createPlacementGroup(String name, - List> bundles, PlacementStrategy strategy) { + public static PlacementGroup createPlacementGroup( + String name, List> bundles, PlacementStrategy strategy) { return internal().createPlacementGroup(name, bundles, strategy); } - public static PlacementGroup createPlacementGroup(List> bundles, - PlacementStrategy strategy) { + public static PlacementGroup createPlacementGroup( + List> bundles, PlacementStrategy strategy) { return internal().createPlacementGroup(bundles, strategy); } /** * Intentionally exit the current actor. - *

- * This method is used to disconnect an actor and exit the worker. * - * @throws RuntimeException An exception is raised if this is a driver or this worker is not - * an actor. + *

This method is used to disconnect an actor and exit the worker. + * + * @throws RuntimeException An exception is raised if this is a driver or this worker is not an + * actor. */ public static void exitActor() { runtime.exitActor(); @@ -275,8 +250,8 @@ public final class Ray extends RayCall { /** * Get a placement group by placement group Id. - * @param id placement group id. - * @return The placement group. + * + * @param id placement group id. Returns The placement group. */ public static PlacementGroup getPlacementGroup(PlacementGroupId id) { return internal().getPlacementGroup(id); @@ -284,15 +259,16 @@ public final class Ray extends RayCall { /** * Get all placement groups in this cluster. - * @return All placement groups. + * + *

Returns All placement groups. */ public static List getAllPlacementGroups() { return internal().getAllPlacementGroups(); } /** - * Remove a placement group by id. - * Throw RayException if remove failed. + * Remove a placement group by id. Throw RayException if remove failed. + * * @param id Id of the placement group. */ public static void removePlacementGroup(PlacementGroupId id) { diff --git a/java/api/src/main/java/io/ray/api/WaitResult.java b/java/api/src/main/java/io/ray/api/WaitResult.java index c6d2e6f67..586c3dc77 100644 --- a/java/api/src/main/java/io/ray/api/WaitResult.java +++ b/java/api/src/main/java/io/ray/api/WaitResult.java @@ -3,8 +3,8 @@ package io.ray.api; import java.util.List; /** - * Represents the result of a Ray.wait call. It contains 2 lists, - * one containing the locally available objects, one containing the rest. + * Represents the result of a Ray.wait call. It contains 2 lists, one containing the locally + * available objects, one containing the rest. */ public final class WaitResult { @@ -16,18 +16,13 @@ public final class WaitResult { this.unready = unready; } - /** - * Get the list of ready objects. - */ + /** Get the list of ready objects. */ public List> getReady() { return ready; } - /** - * Get the list of unready objects. - */ + /** Get the list of unready objects. */ public List> getUnready() { return unready; } - } diff --git a/java/api/src/main/java/io/ray/api/call/ActorCreator.java b/java/api/src/main/java/io/ray/api/call/ActorCreator.java index 16a24bda6..c6bb9cce8 100644 --- a/java/api/src/main/java/io/ray/api/call/ActorCreator.java +++ b/java/api/src/main/java/io/ray/api/call/ActorCreator.java @@ -21,10 +21,9 @@ public class ActorCreator extends BaseActorCreator> { /** * Set the JVM options for the Java worker that this actor is running in. * - * Note, if this is set, this actor won't share Java worker with other actors or tasks. + *

Note, if this is set, this actor won't share Java worker with other actors or tasks. * - * @param jvmOptions JVM options for the Java worker that this actor is running in. - * @return self + * @param jvmOptions JVM options for the Java worker that this actor is running in. Returns self * @see io.ray.api.options.ActorCreationOptions.Builder#setJvmOptions(java.lang.String) */ public ActorCreator setJvmOptions(String jvmOptions) { @@ -35,10 +34,9 @@ public class ActorCreator extends BaseActorCreator> { /** * Create a java actor remotely and return a handle to the created actor. * - * @return a handle to the created java actor. + *

Returns a handle to the created java actor. */ public ActorHandle remote() { return Ray.internal().createActor(func, args, buildOptions()); } - } diff --git a/java/api/src/main/java/io/ray/api/call/ActorTaskCaller.java b/java/api/src/main/java/io/ray/api/call/ActorTaskCaller.java index 8ec64a958..4b9d25a21 100644 --- a/java/api/src/main/java/io/ray/api/call/ActorTaskCaller.java +++ b/java/api/src/main/java/io/ray/api/call/ActorTaskCaller.java @@ -25,11 +25,10 @@ public class ActorTaskCaller { * Execute an java actor method remotely and return an object reference to the result object in * the object store. * - * @return an object reference to an object in the object store. + *

Returns an object reference to an object in the object store. */ @SuppressWarnings("unchecked") public ObjectRef remote() { return Ray.internal().callActor(actor, func, args); } - } diff --git a/java/api/src/main/java/io/ray/api/call/BaseActorCreator.java b/java/api/src/main/java/io/ray/api/call/BaseActorCreator.java index 93d98ab65..5f488124b 100644 --- a/java/api/src/main/java/io/ray/api/call/BaseActorCreator.java +++ b/java/api/src/main/java/io/ray/api/call/BaseActorCreator.java @@ -14,13 +14,11 @@ public class BaseActorCreator { protected ActorCreationOptions.Builder builder = new ActorCreationOptions.Builder(); /** - * Set the actor name of a named actor. - * This named actor is only accessible from this job by this name via - * {@link Ray#getActor(java.lang.String)}. If you want create a named actor that is accessible - * from all jobs, use {@link BaseActorCreator#setGlobalName(java.lang.String)} instead. + * Set the actor name of a named actor. This named actor is only accessible from this job by this + * name via {@link Ray#getActor(java.lang.String)}. If you want create a named actor that is + * accessible from all jobs, use {@link BaseActorCreator#setGlobalName(java.lang.String)} instead. * - * @param name The name of the named actor. - * @return self + * @param name The name of the named actor. Returns self * @see io.ray.api.options.ActorCreationOptions.Builder#setName(String) */ public T setName(String name) { @@ -29,12 +27,11 @@ public class BaseActorCreator { } /** - * Set the name of this actor. This actor will be accessible from all jobs by this name via - * {@link Ray#getGlobalActor(java.lang.String)}. If you want to create a named actor that is - * only accessible from this job, use {@link BaseActorCreator#setName(java.lang.String)} instead. + * Set the name of this actor. This actor will be accessible from all jobs by this name via {@link + * Ray#getGlobalActor(java.lang.String)}. If you want to create a named actor that is only + * accessible from this job, use {@link BaseActorCreator#setName(java.lang.String)} instead. * - * @param name The name of the named actor. - * @return self + * @param name The name of the named actor. Returns self * @see io.ray.api.options.ActorCreationOptions.Builder#setGlobalName(String) */ public T setGlobalName(String name) { @@ -43,13 +40,12 @@ public class BaseActorCreator { } /** - * Set a custom resource requirement to reserve for the lifetime of this actor. - * This method can be called multiple times. If the same resource is set multiple times, - * the latest quantity will be used. + * Set a custom resource requirement to reserve for the lifetime of this actor. This method can be + * called multiple times. If the same resource is set multiple times, the latest quantity will be + * used. * * @param resourceName resource name - * @param resourceQuantity resource quantity - * @return self + * @param resourceQuantity resource quantity Returns self * @see ActorCreationOptions.Builder#setResource(java.lang.String, java.lang.Double) */ public T setResource(String resourceName, Double resourceQuantity) { @@ -58,12 +54,11 @@ public class BaseActorCreator { } /** - * Set custom resource requirements to reserve for the lifetime of this actor. - * This method can be called multiple times. If the same resource is set multiple times, - * the latest quantity will be used. + * Set custom resource requirements to reserve for the lifetime of this actor. This method can be + * called multiple times. If the same resource is set multiple times, the latest quantity will be + * used. * - * @param resources requirements for multiple resources. - * @return self + * @param resources requirements for multiple resources. Returns self * @see BaseActorCreator#setResources(java.util.Map) */ public T setResources(Map resources) { @@ -76,8 +71,7 @@ public class BaseActorCreator { * unexpectedly. The minimum valid value is 0 (default), which indicates that the actor doesn't * need to be restarted. A value of -1 indicates that an actor should be restarted indefinitely. * - * @param maxRestarts max number of actor restarts - * @return self + * @param maxRestarts max number of actor restarts Returns self * @see ActorCreationOptions.Builder#setMaxRestarts(int) */ public T setMaxRestarts(int maxRestarts) { @@ -87,12 +81,11 @@ public class BaseActorCreator { /** * Set the max number of concurrent calls to allow for this actor. - *

- * The max concurrency defaults to 1 for threaded execution. - * Note that the execution order is not guaranteed when {@code max_concurrency > 1}. * - * @param maxConcurrency The max number of concurrent calls to allow for this actor. - * @return self + *

The max concurrency defaults to 1 for threaded execution. Note that the execution order is + * not guaranteed when {@code max_concurrency > 1}. + * + * @param maxConcurrency The max number of concurrent calls to allow for this actor. Returns self * @see ActorCreationOptions.Builder#setMaxConcurrency(int) */ public T setMaxConcurrency(int maxConcurrency) { @@ -104,8 +97,7 @@ public class BaseActorCreator { * Set the placement group to place this actor in. * * @param group The placement group of the actor. - * @param bundleIndex The index of the bundle to place this actor in. - * @return self + * @param bundleIndex The index of the bundle to place this actor in. Returns self * @see ActorCreationOptions.Builder#setPlacementGroup(PlacementGroup, int) */ public T setPlacementGroup(PlacementGroup group, int bundleIndex) { @@ -121,5 +113,4 @@ public class BaseActorCreator { protected ActorCreationOptions buildOptions() { return builder.build(); } - } diff --git a/java/api/src/main/java/io/ray/api/call/BaseTaskCaller.java b/java/api/src/main/java/io/ray/api/call/BaseTaskCaller.java index 88c58e053..8b683c7bd 100644 --- a/java/api/src/main/java/io/ray/api/call/BaseTaskCaller.java +++ b/java/api/src/main/java/io/ray/api/call/BaseTaskCaller.java @@ -14,8 +14,7 @@ public class BaseTaskCaller> { /** * Set a name for this task. * - * @param name task name - * @return self + * @param name task name Returns self * @see CallOptions.Builder#setName(java.lang.String) */ public T setName(String name) { @@ -28,8 +27,7 @@ public class BaseTaskCaller> { * times. If the same resource is set multiple times, the latest quantity will be used. * * @param name resource name - * @param value resource capacity - * @return self + * @param value resource capacity Returns self * @see CallOptions.Builder#setResource(java.lang.String, java.lang.Double) */ public T setResource(String name, Double value) { @@ -41,8 +39,7 @@ public class BaseTaskCaller> { * Set custom requirements for multiple resources. This method can be called multiple times. If * the same resource is set multiple times, the latest quantity will be used. * - * @param resources requirements for multiple resources. - * @return self + * @param resources requirements for multiple resources. Returns self * @see CallOptions.Builder#setResources(java.util.Map) */ public T setResources(Map resources) { diff --git a/java/api/src/main/java/io/ray/api/call/PyActorCreator.java b/java/api/src/main/java/io/ray/api/call/PyActorCreator.java index 4242b68de..5add65346 100644 --- a/java/api/src/main/java/io/ray/api/call/PyActorCreator.java +++ b/java/api/src/main/java/io/ray/api/call/PyActorCreator.java @@ -4,9 +4,7 @@ import io.ray.api.PyActorHandle; import io.ray.api.Ray; import io.ray.api.function.PyActorClass; -/** - * A helper to create python actor. - */ +/** A helper to create python actor. */ public class PyActorCreator extends BaseActorCreator { private final PyActorClass pyActorClass; private final Object[] args; @@ -19,7 +17,7 @@ public class PyActorCreator extends BaseActorCreator { /** * Create a python actor remotely and return a handle to the created actor. * - * @return a handle to the created python actor. + *

Returns a handle to the created python actor. */ public PyActorHandle remote() { return Ray.internal().createActor(pyActorClass, args, buildOptions()); diff --git a/java/api/src/main/java/io/ray/api/call/PyActorTaskCaller.java b/java/api/src/main/java/io/ray/api/call/PyActorTaskCaller.java index bbe79d134..c9444548f 100644 --- a/java/api/src/main/java/io/ray/api/call/PyActorTaskCaller.java +++ b/java/api/src/main/java/io/ray/api/call/PyActorTaskCaller.java @@ -25,11 +25,10 @@ public class PyActorTaskCaller { * Execute a python actor method remotely and return an object reference to the result object in * the object store. * - * @return an object reference to an object in the object store. + *

Returns an object reference to an object in the object store. */ @SuppressWarnings("unchecked") public ObjectRef remote() { return Ray.internal().callActor(actor, method, args); } - } diff --git a/java/api/src/main/java/io/ray/api/call/PyTaskCaller.java b/java/api/src/main/java/io/ray/api/call/PyTaskCaller.java index ca5c524a2..8d58e9b30 100644 --- a/java/api/src/main/java/io/ray/api/call/PyTaskCaller.java +++ b/java/api/src/main/java/io/ray/api/call/PyTaskCaller.java @@ -22,11 +22,10 @@ public class PyTaskCaller extends BaseTaskCaller> { * Execute a python function remotely and return an object reference to the result object in the * object store. * - * @return an object reference to an object in the object store. + *

Returns an object reference to an object in the object store. */ @SuppressWarnings("unchecked") public ObjectRef remote() { return Ray.internal().call(func, args, buildOptions()); } - } diff --git a/java/api/src/main/java/io/ray/api/call/TaskCaller.java b/java/api/src/main/java/io/ray/api/call/TaskCaller.java index 80dacec2d..82f72d63e 100644 --- a/java/api/src/main/java/io/ray/api/call/TaskCaller.java +++ b/java/api/src/main/java/io/ray/api/call/TaskCaller.java @@ -22,7 +22,7 @@ public class TaskCaller extends BaseTaskCaller> { * Execute a java function remotely and return an object reference to the result object in the * object store. * - * @return an object reference to an object in the object store. + *

Returns an object reference to an object in the object store. */ @SuppressWarnings("unchecked") public ObjectRef remote() { diff --git a/java/api/src/main/java/io/ray/api/call/VoidActorTaskCaller.java b/java/api/src/main/java/io/ray/api/call/VoidActorTaskCaller.java index 033b37187..5487561c1 100644 --- a/java/api/src/main/java/io/ray/api/call/VoidActorTaskCaller.java +++ b/java/api/src/main/java/io/ray/api/call/VoidActorTaskCaller.java @@ -4,9 +4,7 @@ import io.ray.api.ActorHandle; import io.ray.api.Ray; import io.ray.api.function.RayFuncVoid; -/** - * A helper to call java actor method which doesn't have a return value. - */ +/** A helper to call java actor method which doesn't have a return value. */ public class VoidActorTaskCaller { private final ActorHandle actor; private final RayFuncVoid func; @@ -18,11 +16,8 @@ public class VoidActorTaskCaller { this.args = args; } - /** - * Execute a function remotely. - */ + /** Execute a function remotely. */ public void remote() { Ray.internal().callActor(actor, func, args); } - } diff --git a/java/api/src/main/java/io/ray/api/call/VoidTaskCaller.java b/java/api/src/main/java/io/ray/api/call/VoidTaskCaller.java index feca5f1f1..dee66de9e 100644 --- a/java/api/src/main/java/io/ray/api/call/VoidTaskCaller.java +++ b/java/api/src/main/java/io/ray/api/call/VoidTaskCaller.java @@ -3,9 +3,7 @@ package io.ray.api.call; import io.ray.api.Ray; import io.ray.api.function.RayFuncVoid; -/** - * A helper to call java remote function which doesn't have a return value. - */ +/** A helper to call java remote function which doesn't have a return value. */ public class VoidTaskCaller extends BaseTaskCaller { private final RayFuncVoid func; private final Object[] args; @@ -15,11 +13,8 @@ public class VoidTaskCaller extends BaseTaskCaller { this.args = args; } - /** - * Execute a function remotely. - */ + /** Execute a function remotely. */ public void remote() { Ray.internal().call(func, args, buildOptions()); } - } diff --git a/java/api/src/main/java/io/ray/api/function/PyActorClass.java b/java/api/src/main/java/io/ray/api/function/PyActorClass.java index a207a233b..c753e1f27 100644 --- a/java/api/src/main/java/io/ray/api/function/PyActorClass.java +++ b/java/api/src/main/java/io/ray/api/function/PyActorClass.java @@ -38,11 +38,9 @@ public class PyActorClass { * Create a python actor class. * * @param moduleName The full module name of this actor class - * @param className The name of this actor class - * @return a python actor class + * @param className The name of this actor class Returns a python actor class */ public static PyActorClass of(String moduleName, String className) { return new PyActorClass(moduleName, className); } - } diff --git a/java/api/src/main/java/io/ray/api/function/PyActorMethod.java b/java/api/src/main/java/io/ray/api/function/PyActorMethod.java index fae0d63d6..f91b0c9f9 100644 --- a/java/api/src/main/java/io/ray/api/function/PyActorMethod.java +++ b/java/api/src/main/java/io/ray/api/function/PyActorMethod.java @@ -2,9 +2,9 @@ package io.ray.api.function; /** * A class that represents a method of a Python actor. - *

- * Note, information about the actor will be inferred from the actor handle, - * so it's not specified in this class. + * + *

Note, information about the actor will be inferred from the actor handle, so it's not + * specified in this class. * *

  * there is a Python actor class A.
@@ -43,8 +43,7 @@ public class PyActorMethod {
   /**
    * Create a python actor method.
    *
-   * @param methodName The name of this actor method
-   * @return a python actor method.
+   * @param methodName The name of this actor method Returns a python actor method.
    */
   public static PyActorMethod of(String methodName) {
     return of(methodName, Object.class);
@@ -55,11 +54,9 @@ public class PyActorMethod {
    *
    * @param methodName The name of this actor method
    * @param returnType Class of the return value of this actor method
-   * @param  The type of the return value of this actor method
-   * @return a python actor method.
+   * @param  The type of the return value of this actor method Returns a python actor method.
    */
   public static  PyActorMethod of(String methodName, Class returnType) {
     return new PyActorMethod<>(methodName, returnType);
   }
-
 }
diff --git a/java/api/src/main/java/io/ray/api/function/PyFunction.java b/java/api/src/main/java/io/ray/api/function/PyFunction.java
index 57540c23b..119bba4e5 100644
--- a/java/api/src/main/java/io/ray/api/function/PyFunction.java
+++ b/java/api/src/main/java/io/ray/api/function/PyFunction.java
@@ -49,11 +49,9 @@ public class PyFunction {
    * Create a python function.
    *
    * @param moduleName The full module name of this function
-   * @param functionName The name of this function
-   * @return a python function.
+   * @param functionName The name of this function Returns a python function.
    */
-  public static PyFunction of(
-      String moduleName, String functionName) {
+  public static PyFunction of(String moduleName, String functionName) {
     return of(moduleName, functionName, Object.class);
   }
 
@@ -63,12 +61,9 @@ public class PyFunction {
    * @param moduleName The full module name of this function
    * @param functionName The name of this function
    * @param returnType Class of the return value of this function
-   * @param  Type of the return value of this function
-   * @return a python function.
+   * @param  Type of the return value of this function Returns a python function.
    */
-  public static  PyFunction of(
-      String moduleName, String functionName, Class returnType) {
+  public static  PyFunction of(String moduleName, String functionName, Class returnType) {
     return new PyFunction<>(moduleName, functionName, returnType);
   }
-
 }
diff --git a/java/api/src/main/java/io/ray/api/function/RayFunc.java b/java/api/src/main/java/io/ray/api/function/RayFunc.java
index 0bf6109d1..0721fc1b6 100644
--- a/java/api/src/main/java/io/ray/api/function/RayFunc.java
+++ b/java/api/src/main/java/io/ray/api/function/RayFunc.java
@@ -2,8 +2,5 @@ package io.ray.api.function;
 
 import java.io.Serializable;
 
-/**
- * Base interface of all Ray remote java functions.
- */
-public interface RayFunc extends Serializable {
-}
+/** Base interface of all Ray remote java functions. */
+public interface RayFunc extends Serializable {}
diff --git a/java/api/src/main/java/io/ray/api/function/RayFunc0.java b/java/api/src/main/java/io/ray/api/function/RayFunc0.java
index b1f979739..bfdda1d2a 100644
--- a/java/api/src/main/java/io/ray/api/function/RayFunc0.java
+++ b/java/api/src/main/java/io/ray/api/function/RayFunc0.java
@@ -2,9 +2,7 @@
 
 package io.ray.api.function;
 
-/**
- * Functional interface for a remote function that has 0 parameter.
- */
+/** Functional interface for a remote function that has 0 parameter. */
 @FunctionalInterface
 public interface RayFunc0 extends RayFuncR {
 
diff --git a/java/api/src/main/java/io/ray/api/function/RayFunc1.java b/java/api/src/main/java/io/ray/api/function/RayFunc1.java
index 903a3349c..efc85c20c 100644
--- a/java/api/src/main/java/io/ray/api/function/RayFunc1.java
+++ b/java/api/src/main/java/io/ray/api/function/RayFunc1.java
@@ -2,9 +2,7 @@
 
 package io.ray.api.function;
 
-/**
- * Functional interface for a remote function that has 1 parameter.
- */
+/** Functional interface for a remote function that has 1 parameter. */
 @FunctionalInterface
 public interface RayFunc1 extends RayFuncR {
 
diff --git a/java/api/src/main/java/io/ray/api/function/RayFunc2.java b/java/api/src/main/java/io/ray/api/function/RayFunc2.java
index 8a0cfbf70..90f24126e 100644
--- a/java/api/src/main/java/io/ray/api/function/RayFunc2.java
+++ b/java/api/src/main/java/io/ray/api/function/RayFunc2.java
@@ -2,9 +2,7 @@
 
 package io.ray.api.function;
 
-/**
- * Functional interface for a remote function that has 2 parameters.
- */
+/** Functional interface for a remote function that has 2 parameters. */
 @FunctionalInterface
 public interface RayFunc2 extends RayFuncR {
 
diff --git a/java/api/src/main/java/io/ray/api/function/RayFunc3.java b/java/api/src/main/java/io/ray/api/function/RayFunc3.java
index 4b878f71a..0ebdfe3ee 100644
--- a/java/api/src/main/java/io/ray/api/function/RayFunc3.java
+++ b/java/api/src/main/java/io/ray/api/function/RayFunc3.java
@@ -2,9 +2,7 @@
 
 package io.ray.api.function;
 
-/**
- * Functional interface for a remote function that has 3 parameters.
- */
+/** Functional interface for a remote function that has 3 parameters. */
 @FunctionalInterface
 public interface RayFunc3 extends RayFuncR {
 
diff --git a/java/api/src/main/java/io/ray/api/function/RayFunc4.java b/java/api/src/main/java/io/ray/api/function/RayFunc4.java
index 41c2af3d2..05a2a498a 100644
--- a/java/api/src/main/java/io/ray/api/function/RayFunc4.java
+++ b/java/api/src/main/java/io/ray/api/function/RayFunc4.java
@@ -2,9 +2,7 @@
 
 package io.ray.api.function;
 
-/**
- * Functional interface for a remote function that has 4 parameters.
- */
+/** Functional interface for a remote function that has 4 parameters. */
 @FunctionalInterface
 public interface RayFunc4 extends RayFuncR {
 
diff --git a/java/api/src/main/java/io/ray/api/function/RayFunc5.java b/java/api/src/main/java/io/ray/api/function/RayFunc5.java
index 90ebb7b43..1a4b5371c 100644
--- a/java/api/src/main/java/io/ray/api/function/RayFunc5.java
+++ b/java/api/src/main/java/io/ray/api/function/RayFunc5.java
@@ -2,9 +2,7 @@
 
 package io.ray.api.function;
 
-/**
- * Functional interface for a remote function that has 5 parameters.
- */
+/** Functional interface for a remote function that has 5 parameters. */
 @FunctionalInterface
 public interface RayFunc5 extends RayFuncR {
 
diff --git a/java/api/src/main/java/io/ray/api/function/RayFunc6.java b/java/api/src/main/java/io/ray/api/function/RayFunc6.java
index e4a5b21eb..4533c3b56 100644
--- a/java/api/src/main/java/io/ray/api/function/RayFunc6.java
+++ b/java/api/src/main/java/io/ray/api/function/RayFunc6.java
@@ -2,9 +2,7 @@
 
 package io.ray.api.function;
 
-/**
- * Functional interface for a remote function that has 6 parameters.
- */
+/** Functional interface for a remote function that has 6 parameters. */
 @FunctionalInterface
 public interface RayFunc6 extends RayFuncR {
 
diff --git a/java/api/src/main/java/io/ray/api/function/RayFuncR.java b/java/api/src/main/java/io/ray/api/function/RayFuncR.java
index 7f2bf5436..e4c50687b 100644
--- a/java/api/src/main/java/io/ray/api/function/RayFuncR.java
+++ b/java/api/src/main/java/io/ray/api/function/RayFuncR.java
@@ -1,10 +1,8 @@
 package io.ray.api.function;
 
 /**
- * Interface of all Ray remote functions which have a return value
+ * Interface of all Ray remote functions which have a return value.
  *
  * @param  Type of function return value
  */
-public interface RayFuncR extends RayFunc {
-
-}
+public interface RayFuncR extends RayFunc {}
diff --git a/java/api/src/main/java/io/ray/api/function/RayFuncVoid.java b/java/api/src/main/java/io/ray/api/function/RayFuncVoid.java
index 74ec8182f..463f29906 100644
--- a/java/api/src/main/java/io/ray/api/function/RayFuncVoid.java
+++ b/java/api/src/main/java/io/ray/api/function/RayFuncVoid.java
@@ -1,8 +1,4 @@
 package io.ray.api.function;
 
-/**
- * Interface of all `RayFuncVoidX` classes.
- */
-public interface RayFuncVoid extends RayFunc {
-
-}
+/** Interface of all `RayFuncVoidX` classes. */
+public interface RayFuncVoid extends RayFunc {}
diff --git a/java/api/src/main/java/io/ray/api/function/RayFuncVoid0.java b/java/api/src/main/java/io/ray/api/function/RayFuncVoid0.java
index 5ecce5b72..7137d4e3a 100644
--- a/java/api/src/main/java/io/ray/api/function/RayFuncVoid0.java
+++ b/java/api/src/main/java/io/ray/api/function/RayFuncVoid0.java
@@ -2,9 +2,7 @@
 
 package io.ray.api.function;
 
-/**
- * Functional interface for a remote function that has 0 parameter.
- */
+/** Functional interface for a remote function that has 0 parameter. */
 @FunctionalInterface
 public interface RayFuncVoid0 extends RayFuncVoid {
 
diff --git a/java/api/src/main/java/io/ray/api/function/RayFuncVoid1.java b/java/api/src/main/java/io/ray/api/function/RayFuncVoid1.java
index dc4a4bb1c..08f56807d 100644
--- a/java/api/src/main/java/io/ray/api/function/RayFuncVoid1.java
+++ b/java/api/src/main/java/io/ray/api/function/RayFuncVoid1.java
@@ -2,9 +2,7 @@
 
 package io.ray.api.function;
 
-/**
- * Functional interface for a remote function that has 1 parameter.
- */
+/** Functional interface for a remote function that has 1 parameter. */
 @FunctionalInterface
 public interface RayFuncVoid1 extends RayFuncVoid {
 
diff --git a/java/api/src/main/java/io/ray/api/function/RayFuncVoid2.java b/java/api/src/main/java/io/ray/api/function/RayFuncVoid2.java
index 42f4c102b..85e132d54 100644
--- a/java/api/src/main/java/io/ray/api/function/RayFuncVoid2.java
+++ b/java/api/src/main/java/io/ray/api/function/RayFuncVoid2.java
@@ -2,9 +2,7 @@
 
 package io.ray.api.function;
 
-/**
- * Functional interface for a remote function that has 2 parameters.
- */
+/** Functional interface for a remote function that has 2 parameters. */
 @FunctionalInterface
 public interface RayFuncVoid2 extends RayFuncVoid {
 
diff --git a/java/api/src/main/java/io/ray/api/function/RayFuncVoid3.java b/java/api/src/main/java/io/ray/api/function/RayFuncVoid3.java
index 7d538a028..c1c105ca3 100644
--- a/java/api/src/main/java/io/ray/api/function/RayFuncVoid3.java
+++ b/java/api/src/main/java/io/ray/api/function/RayFuncVoid3.java
@@ -2,9 +2,7 @@
 
 package io.ray.api.function;
 
-/**
- * Functional interface for a remote function that has 3 parameters.
- */
+/** Functional interface for a remote function that has 3 parameters. */
 @FunctionalInterface
 public interface RayFuncVoid3 extends RayFuncVoid {
 
diff --git a/java/api/src/main/java/io/ray/api/function/RayFuncVoid4.java b/java/api/src/main/java/io/ray/api/function/RayFuncVoid4.java
index fd809786b..466131d75 100644
--- a/java/api/src/main/java/io/ray/api/function/RayFuncVoid4.java
+++ b/java/api/src/main/java/io/ray/api/function/RayFuncVoid4.java
@@ -2,9 +2,7 @@
 
 package io.ray.api.function;
 
-/**
- * Functional interface for a remote function that has 4 parameters.
- */
+/** Functional interface for a remote function that has 4 parameters. */
 @FunctionalInterface
 public interface RayFuncVoid4 extends RayFuncVoid {
 
diff --git a/java/api/src/main/java/io/ray/api/function/RayFuncVoid5.java b/java/api/src/main/java/io/ray/api/function/RayFuncVoid5.java
index 10d7309f3..445f94e09 100644
--- a/java/api/src/main/java/io/ray/api/function/RayFuncVoid5.java
+++ b/java/api/src/main/java/io/ray/api/function/RayFuncVoid5.java
@@ -2,9 +2,7 @@
 
 package io.ray.api.function;
 
-/**
- * Functional interface for a remote function that has 5 parameters.
- */
+/** Functional interface for a remote function that has 5 parameters. */
 @FunctionalInterface
 public interface RayFuncVoid5 extends RayFuncVoid {
 
diff --git a/java/api/src/main/java/io/ray/api/function/RayFuncVoid6.java b/java/api/src/main/java/io/ray/api/function/RayFuncVoid6.java
index 51086913f..85082e2b2 100644
--- a/java/api/src/main/java/io/ray/api/function/RayFuncVoid6.java
+++ b/java/api/src/main/java/io/ray/api/function/RayFuncVoid6.java
@@ -2,9 +2,7 @@
 
 package io.ray.api.function;
 
-/**
- * Functional interface for a remote function that has 6 parameters.
- */
+/** Functional interface for a remote function that has 6 parameters. */
 @FunctionalInterface
 public interface RayFuncVoid6 extends RayFuncVoid {
 
diff --git a/java/api/src/main/java/io/ray/api/id/ActorId.java b/java/api/src/main/java/io/ray/api/id/ActorId.java
index a21d4e79f..4983c42fc 100644
--- a/java/api/src/main/java/io/ray/api/id/ActorId.java
+++ b/java/api/src/main/java/io/ray/api/id/ActorId.java
@@ -25,18 +25,14 @@ public class ActorId extends BaseId implements Serializable {
     return new ActorId(bytes);
   }
 
-  /**
-   * Generate a nil ActorId.
-   */
+  /** Generate a nil ActorId. */
   private static ActorId nil() {
     byte[] b = new byte[LENGTH];
     Arrays.fill(b, (byte) 0xFF);
     return new ActorId(b);
   }
 
-  /**
-   * Generate an ActorId with random value. Used for local mode and test only.
-   */
+  /** Generate an ActorId with random value. Used for local mode and test only. */
   public static ActorId fromRandom() {
     byte[] b = new byte[LENGTH];
     new Random().nextBytes(b);
diff --git a/java/api/src/main/java/io/ray/api/id/BaseId.java b/java/api/src/main/java/io/ray/api/id/BaseId.java
index 5c784acea..573f549b2 100644
--- a/java/api/src/main/java/io/ray/api/id/BaseId.java
+++ b/java/api/src/main/java/io/ray/api/id/BaseId.java
@@ -11,34 +11,30 @@ public abstract class BaseId implements Serializable {
   private int hashCodeCache = 0;
   private Boolean isNilCache = null;
 
-  /**
-   * Create a BaseId instance according to the input byte array.
-   */
+  /** Create a BaseId instance according to the input byte array. */
   protected BaseId(byte[] id) {
     if (id.length != size()) {
-      throw new IllegalArgumentException("Failed to construct BaseId, expect " + size()
-              + " bytes, but got " + id.length + " bytes.");
+      throw new IllegalArgumentException(
+          "Failed to construct BaseId, expect "
+              + size()
+              + " bytes, but got "
+              + id.length
+              + " bytes.");
     }
     this.id = id;
   }
 
-  /**
-   * Get the byte data of this id.
-   */
+  /** Get the byte data of this id. */
   public byte[] getBytes() {
     return id;
   }
 
-  /**
-   * Convert the byte data to a ByteBuffer.
-   */
+  /** Convert the byte data to a ByteBuffer. */
   public ByteBuffer toByteBuffer() {
     return ByteBuffer.wrap(id);
   }
 
-  /**
-   * @return True if this id is nil.
-   */
+  /** Returns true if this id is nil. */
   public boolean isNil() {
     if (isNilCache == null) {
       boolean localIsNil = true;
@@ -55,7 +51,8 @@ public abstract class BaseId implements Serializable {
 
   /**
    * Derived class should implement this function.
-   * @return The length of this id in bytes.
+   *
+   * 

Returns The length of this id in bytes. */ public abstract int size(); @@ -96,5 +93,4 @@ public abstract class BaseId implements Serializable { bb.get(id); return id; } - } diff --git a/java/api/src/main/java/io/ray/api/id/JobId.java b/java/api/src/main/java/io/ray/api/id/JobId.java index 48aa61e77..0b5e228ac 100644 --- a/java/api/src/main/java/io/ray/api/id/JobId.java +++ b/java/api/src/main/java/io/ray/api/id/JobId.java @@ -5,32 +5,24 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Arrays; -/** - * Represents the id of a Ray job. - */ +/** Represents the id of a Ray job. */ public class JobId extends BaseId implements Serializable { public static final int LENGTH = 4; public static final JobId NIL = genNil(); - /** - * Create a JobID instance according to the given bytes. - */ + /** Create a JobID instance according to the given bytes. */ private JobId(byte[] id) { super(id); } - /** - * Create a JobId from a given hex string. - */ + /** Create a JobId from a given hex string. */ public static JobId fromHexString(String hex) { return new JobId(hexString2Bytes(hex)); } - /** - * Creates a JobId from the given ByteBuffer. - */ + /** Creates a JobId from the given ByteBuffer. */ public static JobId fromByteBuffer(ByteBuffer bb) { return new JobId(byteBuffer2Bytes(bb)); } @@ -49,9 +41,7 @@ public class JobId extends BaseId implements Serializable { return JobId.fromByteBuffer(wbb); } - /** - * Generate a nil JobId. - */ + /** Generate a nil JobId. */ private static JobId genNil() { byte[] b = new byte[LENGTH]; Arrays.fill(b, (byte) 0xFF); diff --git a/java/api/src/main/java/io/ray/api/id/ObjectId.java b/java/api/src/main/java/io/ray/api/id/ObjectId.java index 78b677ac8..2f53b8f4f 100644 --- a/java/api/src/main/java/io/ray/api/id/ObjectId.java +++ b/java/api/src/main/java/io/ray/api/id/ObjectId.java @@ -5,23 +5,17 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Random; -/** - * Represents the id of a Ray object. - */ +/** Represents the id of a Ray object. */ public class ObjectId extends BaseId implements Serializable { public static final int LENGTH = 28; - /** - * Create an ObjectId from a ByteBuffer. - */ + /** Create an ObjectId from a ByteBuffer. */ public static ObjectId fromByteBuffer(ByteBuffer bb) { return new ObjectId(byteBuffer2Bytes(bb)); } - /** - * Generate an ObjectId with random value. - */ + /** Generate an ObjectId with random value. */ public static ObjectId fromRandom() { // This is tightly coupled with ObjectID definition in C++. If that changes, // this must be changed as well. @@ -41,5 +35,4 @@ public class ObjectId extends BaseId implements Serializable { public int size() { return LENGTH; } - } diff --git a/java/api/src/main/java/io/ray/api/id/PlacementGroupId.java b/java/api/src/main/java/io/ray/api/id/PlacementGroupId.java index dffdcae06..a27ec6f54 100644 --- a/java/api/src/main/java/io/ray/api/id/PlacementGroupId.java +++ b/java/api/src/main/java/io/ray/api/id/PlacementGroupId.java @@ -5,9 +5,7 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Random; -/** - * Represents the id of a placement group. - */ +/** Represents the id of a placement group. */ public class PlacementGroupId extends BaseId implements Serializable { public static final int LENGTH = 16; @@ -18,32 +16,24 @@ public class PlacementGroupId extends BaseId implements Serializable { super(id); } - /** - * Creates a PlacementGroupId from the given ByteBuffer. - */ + /** Creates a PlacementGroupId from the given ByteBuffer. */ public static PlacementGroupId fromByteBuffer(ByteBuffer bb) { return new PlacementGroupId(byteBuffer2Bytes(bb)); } - /** - * Create a PlacementGroupId instance according to the given bytes. - */ + /** Create a PlacementGroupId instance according to the given bytes. */ public static PlacementGroupId fromBytes(byte[] bytes) { return new PlacementGroupId(bytes); } - /** - * Generate a nil PlacementGroupId. - */ + /** Generate a nil PlacementGroupId. */ private static PlacementGroupId nil() { byte[] b = new byte[LENGTH]; Arrays.fill(b, (byte) 0xFF); return new PlacementGroupId(b); } - /** - * Generate an PlacementGroupId with random value. Used for local mode and test only. - */ + /** Generate an PlacementGroupId with random value. Used for local mode and test only. */ public static PlacementGroupId fromRandom() { byte[] b = new byte[LENGTH]; new Random().nextBytes(b); diff --git a/java/api/src/main/java/io/ray/api/id/TaskId.java b/java/api/src/main/java/io/ray/api/id/TaskId.java index 4f5b0c5ff..ed7f06abe 100644 --- a/java/api/src/main/java/io/ray/api/id/TaskId.java +++ b/java/api/src/main/java/io/ray/api/id/TaskId.java @@ -4,9 +4,7 @@ import java.io.Serializable; import java.nio.ByteBuffer; import java.util.Arrays; -/** - * Represents the id of a Ray task. - */ +/** Represents the id of a Ray task. */ public class TaskId extends BaseId implements Serializable { public static final int UNIQUE_BYTES_LENGTH = 8; @@ -15,30 +13,22 @@ public class TaskId extends BaseId implements Serializable { public static final TaskId NIL = genNil(); - /** - * Create a TaskId from a hex string. - */ + /** Create a TaskId from a hex string. */ public static TaskId fromHexString(String hex) { return new TaskId(hexString2Bytes(hex)); } - /** - * Creates a TaskId from a ByteBuffer. - */ + /** Creates a TaskId from a ByteBuffer. */ public static TaskId fromByteBuffer(ByteBuffer bb) { return new TaskId(byteBuffer2Bytes(bb)); } - /** - * Creates a TaskId from given bytes. - */ + /** Creates a TaskId from given bytes. */ public static TaskId fromBytes(byte[] bytes) { return new TaskId(bytes); } - /** - * Generate a nil TaskId. - */ + /** Generate a nil TaskId. */ private static TaskId genNil() { byte[] b = new byte[LENGTH]; Arrays.fill(b, (byte) 0xFF); diff --git a/java/api/src/main/java/io/ray/api/id/UniqueId.java b/java/api/src/main/java/io/ray/api/id/UniqueId.java index 44b19f6a7..18697f8c8 100644 --- a/java/api/src/main/java/io/ray/api/id/UniqueId.java +++ b/java/api/src/main/java/io/ray/api/id/UniqueId.java @@ -5,41 +5,30 @@ import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Random; -/** - * Represents a unique id of all Ray concepts, including - * workers, actors, checkpoints, etc. - */ +/** Represents a unique id of all Ray concepts, including workers, actors, checkpoints, etc. */ public class UniqueId extends BaseId implements Serializable { public static final int LENGTH = 28; public static final UniqueId NIL = genNil(); - /** - * Create a UniqueId from a hex string. - */ + /** Create a UniqueId from a hex string. */ public static UniqueId fromHexString(String hex) { return new UniqueId(hexString2Bytes(hex)); } - /** - * Creates a UniqueId from a ByteBuffer. - */ + /** Creates a UniqueId from a ByteBuffer. */ public static UniqueId fromByteBuffer(ByteBuffer bb) { return new UniqueId(byteBuffer2Bytes(bb)); } - /** - * Generate a nil UniqueId. - */ + /** Generate a nil UniqueId. */ private static UniqueId genNil() { byte[] b = new byte[LENGTH]; Arrays.fill(b, (byte) 0xFF); return new UniqueId(b); } - /** - * Generate an UniqueId with random value. - */ + /** Generate an UniqueId with random value. */ public static UniqueId randomId() { byte[] b = new byte[LENGTH]; new Random().nextBytes(b); diff --git a/java/api/src/main/java/io/ray/api/options/ActorCreationOptions.java b/java/api/src/main/java/io/ray/api/options/ActorCreationOptions.java index 08fd77c45..29a13c115 100644 --- a/java/api/src/main/java/io/ray/api/options/ActorCreationOptions.java +++ b/java/api/src/main/java/io/ray/api/options/ActorCreationOptions.java @@ -5,9 +5,7 @@ import io.ray.api.placementgroup.PlacementGroup; import java.util.HashMap; import java.util.Map; -/** - * The options for creating actor. - */ +/** The options for creating actor. */ public class ActorCreationOptions extends BaseTaskOptions { public final boolean global; public final String name; @@ -17,9 +15,15 @@ public class ActorCreationOptions extends BaseTaskOptions { public final PlacementGroup group; public final int bundleIndex; - private ActorCreationOptions(boolean global, String name, Map resources, - int maxRestarts, String jvmOptions, int maxConcurrency, - PlacementGroup group, int bundleIndex) { + private ActorCreationOptions( + boolean global, + String name, + Map resources, + int maxRestarts, + String jvmOptions, + int maxConcurrency, + PlacementGroup group, + int bundleIndex) { super(resources); this.global = global; this.name = name; @@ -30,9 +34,7 @@ public class ActorCreationOptions extends BaseTaskOptions { this.bundleIndex = bundleIndex; } - /** - * The inner class for building ActorCreationOptions. - */ + /** The inner class for building ActorCreationOptions. */ public static class Builder { private boolean global; private String name; @@ -44,13 +46,11 @@ public class ActorCreationOptions extends BaseTaskOptions { private int bundleIndex; /** - * Set the actor name of a named actor. - * This named actor is only accessible from this job by this name via - * {@link Ray#getActor(java.lang.String)}. If you want create a named actor that is accessible - * from all jobs, use {@link Builder#setGlobalName(java.lang.String)} instead. + * Set the actor name of a named actor. This named actor is only accessible from this job by + * this name via {@link Ray#getActor(java.lang.String)}. If you want create a named actor that + * is accessible from all jobs, use {@link Builder#setGlobalName(java.lang.String)} instead. * - * @param name The name of the named actor. - * @return self + * @param name The name of the named actor. Returns self */ public Builder setName(String name) { this.name = name; @@ -63,8 +63,7 @@ public class ActorCreationOptions extends BaseTaskOptions { * {@link Ray#getGlobalActor(java.lang.String)}. If you want to create a named actor that is * only accessible from this job, use {@link Builder#setName(java.lang.String)} instead. * - * @param name The name of the named actor. - * @return self + * @param name The name of the named actor. Returns self */ public Builder setGlobalName(String name) { this.name = name; @@ -73,13 +72,12 @@ public class ActorCreationOptions extends BaseTaskOptions { } /** - * Set a custom resource requirement to reserve for the lifetime of this actor. - * This method can be called multiple times. If the same resource is set multiple times, - * the latest quantity will be used. + * Set a custom resource requirement to reserve for the lifetime of this actor. This method can + * be called multiple times. If the same resource is set multiple times, the latest quantity + * will be used. * * @param resourceName resource name - * @param resourceQuantity resource quantity - * @return self + * @param resourceQuantity resource quantity Returns self */ public Builder setResource(String resourceName, Double resourceQuantity) { this.resources.put(resourceName, resourceQuantity); @@ -87,12 +85,11 @@ public class ActorCreationOptions extends BaseTaskOptions { } /** - * Set custom resource requirements to reserve for the lifetime of this actor. - * This method can be called multiple times. If the same resource is set multiple times, - * the latest quantity will be used. + * Set custom resource requirements to reserve for the lifetime of this actor. This method can + * be called multiple times. If the same resource is set multiple times, the latest quantity + * will be used. * - * @param resources requirements for multiple resources. - * @return self + * @param resources requirements for multiple resources. Returns self */ public Builder setResources(Map resources) { this.resources.putAll(resources); @@ -104,8 +101,7 @@ public class ActorCreationOptions extends BaseTaskOptions { * unexpectedly. The minimum valid value is 0 (default), which indicates that the actor doesn't * need to be restarted. A value of -1 indicates that an actor should be restarted indefinitely. * - * @param maxRestarts max number of actor restarts - * @return self + * @param maxRestarts max number of actor restarts Returns self */ public Builder setMaxRestarts(int maxRestarts) { this.maxRestarts = maxRestarts; @@ -114,11 +110,10 @@ public class ActorCreationOptions extends BaseTaskOptions { /** * Set the JVM options for the Java worker that this actor is running in. - *

- * Note, if this is set, this actor won't share Java worker with other actors or tasks. * - * @param jvmOptions JVM options for the Java worker that this actor is running in. - * @return self + *

Note, if this is set, this actor won't share Java worker with other actors or tasks. + * + * @param jvmOptions JVM options for the Java worker that this actor is running in. Returns self */ public Builder setJvmOptions(String jvmOptions) { this.jvmOptions = jvmOptions; @@ -127,12 +122,12 @@ public class ActorCreationOptions extends BaseTaskOptions { /** * Set the max number of concurrent calls to allow for this actor. - *

- * The max concurrency defaults to 1 for threaded execution. - * Note that the execution order is not guaranteed when {@code max_concurrency > 1}. * - * @param maxConcurrency The max number of concurrent calls to allow for this actor. - * @return self + *

The max concurrency defaults to 1 for threaded execution. Note that the execution order is + * not guaranteed when {@code max_concurrency > 1}. + * + * @param maxConcurrency The max number of concurrent calls to allow for this actor. Returns + * self */ public Builder setMaxConcurrency(int maxConcurrency) { if (maxConcurrency <= 0) { @@ -147,8 +142,7 @@ public class ActorCreationOptions extends BaseTaskOptions { * Set the placement group to place this actor in. * * @param group The placement group of the actor. - * @param bundleIndex The index of the bundle to place this actor in. - * @return self + * @param bundleIndex The index of the bundle to place this actor in. Returns self */ public Builder setPlacementGroup(PlacementGroup group, int bundleIndex) { this.group = group; @@ -161,5 +155,4 @@ public class ActorCreationOptions extends BaseTaskOptions { global, name, resources, maxRestarts, jvmOptions, maxConcurrency, group, bundleIndex); } } - } diff --git a/java/api/src/main/java/io/ray/api/options/BaseTaskOptions.java b/java/api/src/main/java/io/ray/api/options/BaseTaskOptions.java index 943e2608d..21c67dde5 100644 --- a/java/api/src/main/java/io/ray/api/options/BaseTaskOptions.java +++ b/java/api/src/main/java/io/ray/api/options/BaseTaskOptions.java @@ -4,9 +4,7 @@ import java.io.Serializable; import java.util.HashMap; import java.util.Map; -/** - * The options class for RayCall or ActorCreation. - */ +/** The options class for RayCall or ActorCreation. */ public abstract class BaseTaskOptions implements Serializable { public final Map resources; @@ -18,19 +16,22 @@ public abstract class BaseTaskOptions implements Serializable { public BaseTaskOptions(Map resources) { for (Map.Entry entry : resources.entrySet()) { if (entry.getValue() == null || entry.getValue().compareTo(0.0) <= 0) { - throw new IllegalArgumentException(String.format("Resource values should be " - + "positive. Specified resource: %s = %s.", entry.getKey(), entry.getValue())); + throw new IllegalArgumentException( + String.format( + "Resource values should be " + "positive. Specified resource: %s = %s.", + entry.getKey(), entry.getValue())); } // Note: A resource value should be an integer if it is greater than 1.0. // e.g. 3.0 is a valid resource value, but 3.5 is not. if (entry.getValue().compareTo(1.0) >= 0 && entry.getValue().compareTo(Math.floor(entry.getValue())) != 0) { - throw new IllegalArgumentException(String.format("A resource value should be " - + "an integer if it is greater than 1.0. Specified resource: %s = %s.", - entry.getKey(), entry.getValue())); + throw new IllegalArgumentException( + String.format( + "A resource value should be " + + "an integer if it is greater than 1.0. Specified resource: %s = %s.", + entry.getKey(), entry.getValue())); } } this.resources = resources; } - } diff --git a/java/api/src/main/java/io/ray/api/options/CallOptions.java b/java/api/src/main/java/io/ray/api/options/CallOptions.java index 37e474d55..233c30aa3 100644 --- a/java/api/src/main/java/io/ray/api/options/CallOptions.java +++ b/java/api/src/main/java/io/ray/api/options/CallOptions.java @@ -22,8 +22,7 @@ public class CallOptions extends BaseTaskOptions { /** * Set a name for this task. * - * @param name task name - * @return self + * @param name task name Returns self */ public Builder setName(String name) { this.name = name; @@ -35,8 +34,7 @@ public class CallOptions extends BaseTaskOptions { * multiple times. If the same resource is set multiple times, the latest quantity will be used. * * @param name resource name - * @param value resource capacity - * @return self + * @param value resource capacity Returns self */ public Builder setResource(String name, Double value) { this.resources.put(name, value); @@ -47,8 +45,7 @@ public class CallOptions extends BaseTaskOptions { * Set custom requirements for multiple resources. This method can be called multiple times. If * the same resource is set multiple times, the latest quantity will be used. * - * @param resources requirements for multiple resources. - * @return self + * @param resources requirements for multiple resources. Returns self */ public Builder setResources(Map resources) { this.resources.putAll(resources); diff --git a/java/api/src/main/java/io/ray/api/placementgroup/PlacementGroup.java b/java/api/src/main/java/io/ray/api/placementgroup/PlacementGroup.java index 36531680c..9b4080deb 100644 --- a/java/api/src/main/java/io/ray/api/placementgroup/PlacementGroup.java +++ b/java/api/src/main/java/io/ray/api/placementgroup/PlacementGroup.java @@ -1,10 +1,9 @@ package io.ray.api.placementgroup; /** - * A placement group is used to place interdependent actors according to a specific strategy - * {@link PlacementStrategy}. - * When a placement group is created, the corresponding actor slots and resources are preallocated. - * A placement group consists of one or more bundles plus a specific placement strategy. + * A placement group is used to place interdependent actors according to a specific strategy {@link + * PlacementStrategy}. When a placement group is created, the corresponding actor slots and + * resources are preallocated. A placement group consists of one or more bundles plus a specific + * placement strategy. */ -public interface PlacementGroup { -} +public interface PlacementGroup {} diff --git a/java/api/src/main/java/io/ray/api/placementgroup/PlacementGroupState.java b/java/api/src/main/java/io/ray/api/placementgroup/PlacementGroupState.java index c2aeae589..684c69ecc 100644 --- a/java/api/src/main/java/io/ray/api/placementgroup/PlacementGroupState.java +++ b/java/api/src/main/java/io/ray/api/placementgroup/PlacementGroupState.java @@ -1,33 +1,21 @@ package io.ray.api.placementgroup; -/** - * State of placement group. - */ +/** State of placement group. */ public enum PlacementGroupState { - /** - * Wait for resource to schedule. - */ + /** Wait for resource to schedule. */ PENDING(0), - /** - * The placement group has created on some node. - */ + /** The placement group has created on some node. */ CREATED(1), - /** - * The placement group has removed. - */ + /** The placement group has removed. */ REMOVED(2), - /** - * The placement group is rescheduling. - */ + /** The placement group is rescheduling. */ RESCHEDULING(3), - /** - * Unrecognized state. - */ + /** Unrecognized state. */ UNRECOGNIZED(-1); private int value = 0; diff --git a/java/api/src/main/java/io/ray/api/placementgroup/PlacementStrategy.java b/java/api/src/main/java/io/ray/api/placementgroup/PlacementStrategy.java index 1fc4036f5..8a3068ec5 100644 --- a/java/api/src/main/java/io/ray/api/placementgroup/PlacementStrategy.java +++ b/java/api/src/main/java/io/ray/api/placementgroup/PlacementStrategy.java @@ -1,33 +1,23 @@ package io.ray.api.placementgroup; -/** - * The actor placement strategy. - */ +/** The actor placement strategy. */ public enum PlacementStrategy { - /** - * Packs Bundles close together inside nodes as tight as possible. - */ + /** Packs Bundles close together inside nodes as tight as possible. */ PACK(0), - /** - * Places Bundles across distinct nodes as even as possible. - */ + /** Places Bundles across distinct nodes as even as possible. */ SPREAD(1), - /** - * Packs Bundles into one node. The group is not allowed to span multiple nodes. - */ + /** Packs Bundles into one node. The group is not allowed to span multiple nodes. */ STRICT_PACK(2), /** - * Places Bundles across distinct nodes. - * The group is not allowed to deploy more than one bundle on a node. + * Places Bundles across distinct nodes. The group is not allowed to deploy more than one bundle + * on a node. */ STRICT_SPREAD(3), - /** - * Unrecognized strategy. - */ + /** Unrecognized strategy. */ UNRECOGNIZED(-1); private int value = 0; diff --git a/java/api/src/main/java/io/ray/api/runtime/RayRuntime.java b/java/api/src/main/java/io/ray/api/runtime/RayRuntime.java index 8817d5b1b..2f3eeb2a7 100644 --- a/java/api/src/main/java/io/ray/api/runtime/RayRuntime.java +++ b/java/api/src/main/java/io/ray/api/runtime/RayRuntime.java @@ -22,37 +22,31 @@ import java.util.Map; import java.util.Optional; import java.util.concurrent.Callable; -/** - * Base interface of a Ray runtime. - */ +/** Base interface of a Ray runtime. */ public interface RayRuntime { - /** - * Shutdown the runtime. - */ + /** Shutdown the runtime. */ void shutdown(); /** * Store an object in the object store. * - * @param obj The Java object to be stored. - * @return A ObjectRef instance that represents the in-store object. + * @param obj The Java object to be stored. Returns A ObjectRef instance that represents the + * in-store object. */ ObjectRef put(T obj); /** * Get an object from the object store. * - * @param objectRef The reference of the object to get. - * @return The Java object. + * @param objectRef The reference of the object to get. Returns The Java object. */ T get(ObjectRef objectRef); /** * Get a list of objects from the object store. * - * @param objectRefs The list of object references. - * @return A list of Java objects. + * @param objectRefs The list of object references. Returns A list of Java objects. */ List get(List> objectRefs); @@ -62,8 +56,8 @@ public interface RayRuntime { * * @param waitList A list of ObjectRef to wait for. * @param numReturns The number of objects that should be returned. - * @param timeoutMs The maximum time in milliseconds to wait before returning. - * @return Two lists, one containing locally available objects, one containing the rest. + * @param timeoutMs The maximum time in milliseconds to wait before returning. Returns Two lists, + * one containing locally available objects, one containing the rest. */ WaitResult wait(List> waitList, int numReturns, int timeoutMs); @@ -88,13 +82,12 @@ public interface RayRuntime { /** * Get a handle to a named actor. - *

- * Gets a handle to a named actor with the given name. The actor must - * have been created with name specified. + * + *

Gets a handle to a named actor with the given name. The actor must have been created with + * name specified. * * @param name The name of the named actor. - * @param global Whether the named actor is global. - * @return ActorHandle to the actor. + * @param global Whether the named actor is global. Returns ActorHandle to the actor. */ Optional getActor(String name, boolean global); @@ -111,8 +104,7 @@ public interface RayRuntime { * * @param func The remote function to run. * @param args The arguments of the remote function. - * @param options The options for this call. - * @return The result object. + * @param options The options for this call. Returns The result object. */ ObjectRef call(RayFunc func, Object[] args, CallOptions options); @@ -121,8 +113,7 @@ public interface RayRuntime { * * @param pyFunction The Python function. * @param args Arguments of the function. - * @param options The options for this call. - * @return The result object. + * @param options The options for this call. Returns The result object. */ ObjectRef call(PyFunction pyFunction, Object[] args, CallOptions options); @@ -131,8 +122,7 @@ public interface RayRuntime { * * @param actor A handle to the actor. * @param func The remote function to run, it must be a method of the given actor. - * @param args The arguments of the remote function. - * @return The result object. + * @param args The arguments of the remote function. Returns The result object. */ ObjectRef callActor(ActorHandle actor, RayFunc func, Object[] args); @@ -141,8 +131,7 @@ public interface RayRuntime { * * @param pyActor A handle to the actor. * @param pyActorMethod The actor method. - * @param args Arguments of the function. - * @return The result object. + * @param args Arguments of the function. Returns The result object. */ ObjectRef callActor(PyActorHandle pyActor, PyActorMethod pyActorMethod, Object[] args); @@ -152,28 +141,25 @@ public interface RayRuntime { * @param actorFactoryFunc A remote function whose return value is the actor object. * @param args The arguments for the remote function. * @param The type of the actor object. - * @param options The options for creating actor. - * @return A handle to the actor. + * @param options The options for creating actor. Returns A handle to the actor. */ - ActorHandle createActor(RayFunc actorFactoryFunc, Object[] args, - ActorCreationOptions options); + ActorHandle createActor( + RayFunc actorFactoryFunc, Object[] args, ActorCreationOptions options); /** * Create a Python actor on a remote node. * * @param pyActorClass The Python actor class. * @param args Arguments of the actor constructor. - * @param options The options for creating actor. - * @return A handle to the actor. + * @param options The options for creating actor. Returns A handle to the actor. */ - PyActorHandle createActor(PyActorClass pyActorClass, Object[] args, - ActorCreationOptions options); + PyActorHandle createActor(PyActorClass pyActorClass, Object[] args, ActorCreationOptions options); - PlacementGroup createPlacementGroup(String name, List> bundles, - PlacementStrategy strategy); + PlacementGroup createPlacementGroup( + String name, List> bundles, PlacementStrategy strategy); - PlacementGroup createPlacementGroup(List> bundles, - PlacementStrategy strategy); + PlacementGroup createPlacementGroup( + List> bundles, PlacementStrategy strategy); RuntimeContext getRuntimeContext(); @@ -184,48 +170,47 @@ public interface RayRuntime { /** * Wrap a {@link Runnable} with necessary context capture. * - * @param runnable The runnable to wrap. - * @return The wrapped runnable. + * @param runnable The runnable to wrap. Returns The wrapped runnable. */ Runnable wrapRunnable(Runnable runnable); /** * Wrap a {@link Callable} with necessary context capture. * - * @param callable The callable to wrap. - * @return The wrapped callable. + * @param callable The callable to wrap. Returns The wrapped callable. */ Callable wrapCallable(Callable callable); - /** - * Intentionally exit the current actor. - */ + /** Intentionally exit the current actor. */ void exitActor(); /** * Get a placement group by id. - * @param id placement group id. - * @return The placement group. + * + * @param id placement group id. Returns The placement group. */ PlacementGroup getPlacementGroup(PlacementGroupId id); /** * Get all placement groups in this cluster. - * @return All placement groups. + * + *

Returns All placement groups. */ List getAllPlacementGroups(); /** * Remove a placement group by id. + * * @param id Id of the placement group. */ void removePlacementGroup(PlacementGroupId id); /** * Wait for the placement group to be ready within the specified time. + * * @param id Id of placement group. - * @param timeoutMs Timeout in milliseconds. - * @return True if the placement group is created. False otherwise. + * @param timeoutMs Timeout in milliseconds. Returns True if the placement group is created. False + * otherwise. */ boolean waitPlacementGroupReady(PlacementGroupId id, int timeoutMs); } diff --git a/java/api/src/main/java/io/ray/api/runtime/RayRuntimeFactory.java b/java/api/src/main/java/io/ray/api/runtime/RayRuntimeFactory.java index da758c07b..fae194c92 100644 --- a/java/api/src/main/java/io/ray/api/runtime/RayRuntimeFactory.java +++ b/java/api/src/main/java/io/ray/api/runtime/RayRuntimeFactory.java @@ -1,8 +1,6 @@ package io.ray.api.runtime; -/** - * A factory that produces a RayRuntime instance. - */ +/** A factory that produces a RayRuntime instance. */ public interface RayRuntimeFactory { RayRuntime createRayRuntime(); diff --git a/java/api/src/main/java/io/ray/api/runtimecontext/NodeInfo.java b/java/api/src/main/java/io/ray/api/runtimecontext/NodeInfo.java index dcf5f3e0c..44e934e7a 100644 --- a/java/api/src/main/java/io/ray/api/runtimecontext/NodeInfo.java +++ b/java/api/src/main/java/io/ray/api/runtimecontext/NodeInfo.java @@ -3,9 +3,7 @@ package io.ray.api.runtimecontext; import io.ray.api.id.UniqueId; import java.util.Map; -/** - * A class that represents the information of a node. - */ +/** A class that represents the information of a node. */ public class NodeInfo { public final UniqueId nodeId; @@ -24,9 +22,15 @@ public class NodeInfo { public final Map resources; - public NodeInfo(UniqueId nodeId, String nodeAddress, String nodeHostname, int nodeManagerPort, - String objectStoreSocketName, String rayletSocketName, - boolean isAlive, Map resources) { + public NodeInfo( + UniqueId nodeId, + String nodeAddress, + String nodeHostname, + int nodeManagerPort, + String objectStoreSocketName, + String rayletSocketName, + boolean isAlive, + Map resources) { this.nodeId = nodeId; this.nodeAddress = nodeAddress; this.nodeHostname = nodeHostname; @@ -39,12 +43,19 @@ public class NodeInfo { public String toString() { return "NodeInfo{" - + "nodeId='" + nodeId + '\'' - + ", nodeAddress='" + nodeAddress + "\'" - + ", nodeHostname'" + nodeHostname + "\'" - + ", isAlive=" + isAlive - + ", resources=" + resources + + "nodeId='" + + nodeId + + '\'' + + ", nodeAddress='" + + nodeAddress + + "\'" + + ", nodeHostname'" + + nodeHostname + + "\'" + + ", isAlive=" + + isAlive + + ", resources=" + + resources + "}"; } - } diff --git a/java/api/src/main/java/io/ray/api/runtimecontext/RuntimeContext.java b/java/api/src/main/java/io/ray/api/runtimecontext/RuntimeContext.java index bf4f733ad..705497ca0 100644 --- a/java/api/src/main/java/io/ray/api/runtimecontext/RuntimeContext.java +++ b/java/api/src/main/java/io/ray/api/runtimecontext/RuntimeContext.java @@ -4,27 +4,23 @@ import io.ray.api.id.ActorId; import io.ray.api.id.JobId; import java.util.List; -/** - * A class used for getting information of Ray runtime. - */ +/** A class used for getting information of Ray runtime. */ public interface RuntimeContext { - /** - * Get the current Job ID. - */ + /** Get the current Job ID. */ JobId getCurrentJobId(); /** * Get the current actor ID. * - * Note, this can only be called in actors. + *

Note, this can only be called in actors. */ ActorId getCurrentActorId(); /** * Returns true if the current actor was restarted, false if it's created for the first time. * - * Note, this method should only be called from an actor creation task. + *

Note, this method should only be called from an actor creation task. */ boolean wasCurrentActorRestarted(); @@ -33,8 +29,6 @@ public interface RuntimeContext { */ boolean isSingleProcess(); - /** - * Get all node information in Ray cluster. - */ + /** Get all node information in Ray cluster. */ List getAllNodeInfo(); } diff --git a/java/checkstyle-suppressions.xml b/java/checkstyle-suppressions.xml index 88233117b..804609cfa 100644 --- a/java/checkstyle-suppressions.xml +++ b/java/checkstyle-suppressions.xml @@ -4,13 +4,4 @@ - - - - - - - - - diff --git a/java/checkstyle.xml b/java/checkstyle.xml index ef13e373c..e6ae2f896 100644 --- a/java/checkstyle.xml +++ b/java/checkstyle.xml @@ -26,6 +26,11 @@ + + + + @@ -58,20 +63,6 @@ value="LITERAL_TRY, LITERAL_FINALLY, LITERAL_IF, LITERAL_ELSE, LITERAL_SWITCH"/> - - - - - - - - - - @@ -164,14 +155,6 @@ - - - - - - - - @@ -208,8 +191,8 @@ - + @@ -232,9 +215,6 @@ - - - diff --git a/java/pom.xml b/java/pom.xml index 70026a5ac..4cf4f50bc 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -63,6 +63,7 @@ 1.8 UTF-8 1.1.0-SNAPSHOT + 1.1.0-SNAPSHOT @@ -176,7 +177,14 @@ org.apache.maven.plugins maven-checkstyle-plugin - 3.0.0 + 3.1.0 + + + com.puppycrawl.tools + checkstyle + 8.19 + + validate @@ -218,6 +226,26 @@ + + + com.diffplug.spotless + spotless-maven-plugin + 2.5.0 + + + + **/RayCall.java + **/ActorCall.java + **/PyActorCall.java + **/runtime/generated/**/*.* + + + 1.7 + + + + + @@ -229,6 +257,10 @@ org.apache.maven.plugins maven-source-plugin + + com.diffplug.spotless + spotless-maven-plugin + diff --git a/java/runtime/src/main/java/io/ray/runtime/AbstractRayRuntime.java b/java/runtime/src/main/java/io/ray/runtime/AbstractRayRuntime.java index 6dabe3c3c..f3478e4c6 100644 --- a/java/runtime/src/main/java/io/ray/runtime/AbstractRayRuntime.java +++ b/java/runtime/src/main/java/io/ray/runtime/AbstractRayRuntime.java @@ -45,9 +45,7 @@ import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Core functionality to implement Ray APIs. - */ +/** Core functionality to implement Ray APIs. */ public abstract class AbstractRayRuntime implements RayRuntimeInternal { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractRayRuntime.class); @@ -63,9 +61,7 @@ public abstract class AbstractRayRuntime implements RayRuntimeInternal { protected TaskSubmitter taskSubmitter; protected WorkerContext workerContext; - /** - * Whether the required thread context is set on the current thread. - */ + /** Whether the required thread context is set on the current thread. */ final ThreadLocal isContextSet = ThreadLocal.withInitial(() -> false); public AbstractRayRuntime(RayConfig rayConfig) { @@ -78,7 +74,7 @@ public abstract class AbstractRayRuntime implements RayRuntimeInternal { @Override public ObjectRef put(T obj) { ObjectId objectId = objectStore.put(obj); - return new ObjectRefImpl(objectId, (Class)(obj == null ? Object.class : obj.getClass())); + return new ObjectRefImpl(objectId, (Class) (obj == null ? Object.class : obj.getClass())); } @Override @@ -101,8 +97,11 @@ public abstract class AbstractRayRuntime implements RayRuntimeInternal { @Override public void free(List> objectRefs, boolean localOnly) { - objectStore.delete(objectRefs.stream().map(ref -> ((ObjectRefImpl) ref).getId()).collect( - Collectors.toList()), localOnly); + objectStore.delete( + objectRefs.stream() + .map(ref -> ((ObjectRefImpl) ref).getId()) + .collect(Collectors.toList()), + localOnly); } @Override @@ -120,13 +119,11 @@ public abstract class AbstractRayRuntime implements RayRuntimeInternal { @Override public ObjectRef call(PyFunction pyFunction, Object[] args, CallOptions options) { - PyFunctionDescriptor functionDescriptor = new PyFunctionDescriptor( - pyFunction.moduleName, - "", - pyFunction.functionName); + PyFunctionDescriptor functionDescriptor = + new PyFunctionDescriptor(pyFunction.moduleName, "", pyFunction.functionName); // Python functions always have a return value, even if it's `None`. - return callNormalFunction(functionDescriptor, args, - /*returnType=*/Optional.of(pyFunction.returnType), options); + return callNormalFunction( + functionDescriptor, args, /*returnType=*/ Optional.of(pyFunction.returnType), options); } @Override @@ -139,17 +136,18 @@ public abstract class AbstractRayRuntime implements RayRuntimeInternal { @Override public ObjectRef callActor(PyActorHandle pyActor, PyActorMethod pyActorMethod, Object... args) { - PyFunctionDescriptor functionDescriptor = new PyFunctionDescriptor(pyActor.getModuleName(), - pyActor.getClassName(), pyActorMethod.methodName); + PyFunctionDescriptor functionDescriptor = + new PyFunctionDescriptor( + pyActor.getModuleName(), pyActor.getClassName(), pyActorMethod.methodName); // Python functions always have a return value, even if it's `None`. - return callActorFunction(pyActor, functionDescriptor, args, - /*returnType=*/Optional.of(pyActorMethod.returnType)); + return callActorFunction( + pyActor, functionDescriptor, args, /*returnType=*/ Optional.of(pyActorMethod.returnType)); } @Override @SuppressWarnings("unchecked") - public ActorHandle createActor(RayFunc actorFactoryFunc, - Object[] args, ActorCreationOptions options) { + public ActorHandle createActor( + RayFunc actorFactoryFunc, Object[] args, ActorCreationOptions options) { FunctionDescriptor functionDescriptor = functionManager.getFunction(workerContext.getCurrentJobId(), actorFactoryFunc) .functionDescriptor; @@ -157,24 +155,24 @@ public abstract class AbstractRayRuntime implements RayRuntimeInternal { } @Override - public PyActorHandle createActor(PyActorClass pyActorClass, Object[] args, - ActorCreationOptions options) { - PyFunctionDescriptor functionDescriptor = new PyFunctionDescriptor( - pyActorClass.moduleName, - pyActorClass.className, - PYTHON_INIT_METHOD_NAME); + public PyActorHandle createActor( + PyActorClass pyActorClass, Object[] args, ActorCreationOptions options) { + PyFunctionDescriptor functionDescriptor = + new PyFunctionDescriptor( + pyActorClass.moduleName, pyActorClass.className, PYTHON_INIT_METHOD_NAME); return (PyActorHandle) createActorImpl(functionDescriptor, args, options); } @Override - public PlacementGroup createPlacementGroup(String name, - List> bundles, PlacementStrategy strategy) { - boolean bundleResourceValid = bundles.stream().allMatch( - bundle -> bundle.values().stream().allMatch(resource -> resource > 0)); + public PlacementGroup createPlacementGroup( + String name, List> bundles, PlacementStrategy strategy) { + boolean bundleResourceValid = + bundles.stream() + .allMatch(bundle -> bundle.values().stream().allMatch(resource -> resource > 0)); if (bundles.isEmpty() || !bundleResourceValid) { throw new IllegalArgumentException( - "Bundles cannot be empty or bundle's resource must be positive."); + "Bundles cannot be empty or bundle's resource must be positive."); } return taskSubmitter.createPlacementGroup(name, bundles, strategy); } @@ -264,12 +262,13 @@ public abstract class AbstractRayRuntime implements RayRuntimeInternal { private ObjectRef callNormalFunction( FunctionDescriptor functionDescriptor, - Object[] args, Optional> returnType, CallOptions options) { + Object[] args, + Optional> returnType, + CallOptions options) { int numReturns = returnType.isPresent() ? 1 : 0; - List functionArgs = ArgumentsBuilder - .wrap(args, functionDescriptor.getLanguage()); - List returnIds = taskSubmitter.submitTask(functionDescriptor, - functionArgs, numReturns, options); + List functionArgs = ArgumentsBuilder.wrap(args, functionDescriptor.getLanguage()); + List returnIds = + taskSubmitter.submitTask(functionDescriptor, functionArgs, numReturns, options); Preconditions.checkState(returnIds.size() == numReturns); if (returnIds.isEmpty()) { return null; @@ -284,10 +283,9 @@ public abstract class AbstractRayRuntime implements RayRuntimeInternal { Object[] args, Optional> returnType) { int numReturns = returnType.isPresent() ? 1 : 0; - List functionArgs = ArgumentsBuilder - .wrap(args, functionDescriptor.getLanguage()); - List returnIds = taskSubmitter.submitActorTask(rayActor, - functionDescriptor, functionArgs, numReturns, null); + List functionArgs = ArgumentsBuilder.wrap(args, functionDescriptor.getLanguage()); + List returnIds = + taskSubmitter.submitActorTask(rayActor, functionDescriptor, functionArgs, numReturns, null); Preconditions.checkState(returnIds.size() == numReturns); if (returnIds.isEmpty()) { return null; @@ -296,10 +294,9 @@ public abstract class AbstractRayRuntime implements RayRuntimeInternal { } } - private BaseActorHandle createActorImpl(FunctionDescriptor functionDescriptor, - Object[] args, ActorCreationOptions options) { - List functionArgs = ArgumentsBuilder - .wrap(args, functionDescriptor.getLanguage()); + private BaseActorHandle createActorImpl( + FunctionDescriptor functionDescriptor, Object[] args, ActorCreationOptions options) { + List functionArgs = ArgumentsBuilder.wrap(args, functionDescriptor.getLanguage()); if (functionDescriptor.getLanguage() != Language.JAVA && options != null) { Preconditions.checkState(Strings.isNullOrEmpty(options.jvmOptions)); } diff --git a/java/runtime/src/main/java/io/ray/runtime/DefaultRayRuntimeFactory.java b/java/runtime/src/main/java/io/ray/runtime/DefaultRayRuntimeFactory.java index 3e4825458..2356a70c7 100644 --- a/java/runtime/src/main/java/io/ray/runtime/DefaultRayRuntimeFactory.java +++ b/java/runtime/src/main/java/io/ray/runtime/DefaultRayRuntimeFactory.java @@ -9,9 +9,7 @@ import io.ray.runtime.util.LoggingUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * The default Ray runtime factory. It produces an instance of RayRuntime. - */ +/** The default Ray runtime factory. It produces an instance of RayRuntime. */ public class DefaultRayRuntimeFactory implements RayRuntimeFactory { @Override @@ -22,19 +20,22 @@ public class DefaultRayRuntimeFactory implements RayRuntimeFactory { if (rayConfig.workerMode == WorkerType.WORKER) { // Handle the uncaught exceptions thrown from user-spawned threads. - Thread.setDefaultUncaughtExceptionHandler((Thread t, Throwable e) -> { - logger.error(String.format("Uncaught worker exception in thread %s", t), e); - }); + Thread.setDefaultUncaughtExceptionHandler( + (Thread t, Throwable e) -> { + logger.error(String.format("Uncaught worker exception in thread %s", t), e); + }); } try { logger.debug("Initializing runtime with config: {}", rayConfig); - AbstractRayRuntime innerRuntime = rayConfig.runMode == RunMode.SINGLE_PROCESS - ? new RayDevRuntime(rayConfig) - : new RayNativeRuntime(rayConfig); - RayRuntimeInternal runtime = rayConfig.numWorkersPerProcess > 1 - ? RayRuntimeProxy.newInstance(innerRuntime) - : innerRuntime; + AbstractRayRuntime innerRuntime = + rayConfig.runMode == RunMode.SINGLE_PROCESS + ? new RayDevRuntime(rayConfig) + : new RayNativeRuntime(rayConfig); + RayRuntimeInternal runtime = + rayConfig.numWorkersPerProcess > 1 + ? RayRuntimeProxy.newInstance(innerRuntime) + : innerRuntime; runtime.start(); return runtime; } catch (Exception e) { diff --git a/java/runtime/src/main/java/io/ray/runtime/RayDevRuntime.java b/java/runtime/src/main/java/io/ray/runtime/RayDevRuntime.java index d7a686db8..9fa2ccc52 100644 --- a/java/runtime/src/main/java/io/ray/runtime/RayDevRuntime.java +++ b/java/runtime/src/main/java/io/ray/runtime/RayDevRuntime.java @@ -35,14 +35,15 @@ public class RayDevRuntime extends AbstractRayRuntime { taskExecutor = new LocalModeTaskExecutor(this); workerContext = new LocalModeWorkerContext(rayConfig.getJobId()); objectStore = new LocalModeObjectStore(workerContext); - taskSubmitter = new LocalModeTaskSubmitter(this, taskExecutor, - (LocalModeObjectStore) objectStore); - ((LocalModeObjectStore) objectStore).addObjectPutCallback( - objectId -> { - if (taskSubmitter != null) { - ((LocalModeTaskSubmitter) taskSubmitter).onObjectPut(objectId); - } - }); + taskSubmitter = + new LocalModeTaskSubmitter(this, taskExecutor, (LocalModeObjectStore) objectStore); + ((LocalModeObjectStore) objectStore) + .addObjectPutCallback( + objectId -> { + if (taskSubmitter != null) { + ((LocalModeTaskSubmitter) taskSubmitter).onObjectPut(objectId); + } + }); } @Override @@ -72,7 +73,7 @@ public class RayDevRuntime extends AbstractRayRuntime { @SuppressWarnings("unchecked") @Override public Optional getActor(String name, boolean global) { - return (Optional) ((LocalModeTaskSubmitter)taskSubmitter).getActor(name, global); + return (Optional) ((LocalModeTaskSubmitter) taskSubmitter).getActor(name, global); } @Override @@ -87,24 +88,21 @@ public class RayDevRuntime extends AbstractRayRuntime { } @Override - public PlacementGroup getPlacementGroup( - PlacementGroupId id) { - //@TODO(clay4444): We need a LocalGcsClient before implements this. + public PlacementGroup getPlacementGroup(PlacementGroupId id) { + // @TODO(clay4444): We need a LocalGcsClient before implements this. throw new UnsupportedOperationException( - "Ray doesn't support placement group operations in local mode."); + "Ray doesn't support placement group operations in local mode."); } @Override public List getAllPlacementGroups() { - //@TODO(clay4444): We need a LocalGcsClient before implements this. + // @TODO(clay4444): We need a LocalGcsClient before implements this. throw new UnsupportedOperationException( - "Ray doesn't support placement group operations in local mode."); + "Ray doesn't support placement group operations in local mode."); } @Override - public void exitActor() { - - } + public void exitActor() {} private JobId nextJobId() { return JobId.fromInt(jobCounter.getAndIncrement()); diff --git a/java/runtime/src/main/java/io/ray/runtime/RayNativeRuntime.java b/java/runtime/src/main/java/io/ray/runtime/RayNativeRuntime.java index a9dbeeea5..f9f12b4d7 100644 --- a/java/runtime/src/main/java/io/ray/runtime/RayNativeRuntime.java +++ b/java/runtime/src/main/java/io/ray/runtime/RayNativeRuntime.java @@ -29,9 +29,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Native runtime for cluster mode. - */ +/** Native runtime for cluster mode. */ public final class RayNativeRuntime extends AbstractRayRuntime { private static final Logger LOGGER = LoggerFactory.getLogger(RayNativeRuntime.class); @@ -39,10 +37,9 @@ public final class RayNativeRuntime extends AbstractRayRuntime { private boolean startRayHead = false; /** - * In Java, GC runs in a standalone thread, and we can't control the exact - * timing of garbage collection. By using this lock, when - * {@link NativeObjectStore#nativeRemoveLocalReference} is executing, the core - * worker will not be shut down, therefore it guarantees some kind of + * In Java, GC runs in a standalone thread, and we can't control the exact timing of garbage + * collection. By using this lock, when {@link NativeObjectStore#nativeRemoveLocalReference} is + * executing, the core worker will not be shut down, therefore it guarantees some kind of * thread-safety. Note that this guarantee only works for driver. */ private final ReadWriteLock shutdownLock = new ReentrantReadWriteLock(); @@ -117,21 +114,29 @@ public final class RayNativeRuntime extends AbstractRayRuntime { rayletConfigStringMap.put(entry.getKey(), entry.getValue().toString()); } - nativeInitialize(rayConfig.workerMode.getNumber(), - rayConfig.nodeIp, rayConfig.getNodeManagerPort(), + nativeInitialize( + rayConfig.workerMode.getNumber(), + rayConfig.nodeIp, + rayConfig.getNodeManagerPort(), rayConfig.workerMode == WorkerType.DRIVER ? System.getProperty("user.dir") : "", - rayConfig.objectStoreSocketName, rayConfig.rayletSocketName, + rayConfig.objectStoreSocketName, + rayConfig.rayletSocketName, (rayConfig.workerMode == WorkerType.DRIVER ? rayConfig.getJobId() : JobId.NIL).getBytes(), - new GcsClientOptions(rayConfig), numWorkersPerProcess, - rayConfig.logDir, rayletConfigStringMap, serializedJobConfig); + new GcsClientOptions(rayConfig), + numWorkersPerProcess, + rayConfig.logDir, + rayletConfigStringMap, + serializedJobConfig); taskExecutor = new NativeTaskExecutor(this); workerContext = new NativeWorkerContext(); objectStore = new NativeObjectStore(workerContext, shutdownLock); taskSubmitter = new NativeTaskSubmitter(); - LOGGER.debug("RayNativeRuntime started with store {}, raylet {}", - rayConfig.objectStoreSocketName, rayConfig.rayletSocketName); + LOGGER.debug( + "RayNativeRuntime started with store {}, raylet {}", + rayConfig.objectStoreSocketName, + rayConfig.rayletSocketName); } catch (Exception e) { if (startRayHead) { try { @@ -201,8 +206,8 @@ public final class RayNativeRuntime extends AbstractRayRuntime { @Override public Object getAsyncContext() { - return new AsyncContext(workerContext.getCurrentWorkerId(), - workerContext.getCurrentClassLoader()); + return new AsyncContext( + workerContext.getCurrentWorkerId(), workerContext.getCurrentClassLoader()); } @Override @@ -229,10 +234,18 @@ public final class RayNativeRuntime extends AbstractRayRuntime { } private static native void nativeInitialize( - int workerMode, String ndoeIpAddress, - int nodeManagerPort, String driverName, String storeSocket, String rayletSocket, - byte[] jobId, GcsClientOptions gcsClientOptions, int numWorkersPerProcess, - String logDir, Map rayletConfigParameters, byte[] serializedJobConfig); + int workerMode, + String ndoeIpAddress, + int nodeManagerPort, + String driverName, + String storeSocket, + String rayletSocket, + byte[] jobId, + GcsClientOptions gcsClientOptions, + int numWorkersPerProcess, + String logDir, + Map rayletConfigParameters, + byte[] serializedJobConfig); private static native void nativeRunTaskExecutor(TaskExecutor taskExecutor); diff --git a/java/runtime/src/main/java/io/ray/runtime/RayRuntimeInternal.java b/java/runtime/src/main/java/io/ray/runtime/RayRuntimeInternal.java index d70e4ed2d..8fc14caa3 100644 --- a/java/runtime/src/main/java/io/ray/runtime/RayRuntimeInternal.java +++ b/java/runtime/src/main/java/io/ray/runtime/RayRuntimeInternal.java @@ -8,14 +8,10 @@ import io.ray.runtime.gcs.GcsClient; import io.ray.runtime.object.ObjectStore; import io.ray.runtime.task.TaskExecutor; -/** - * This interface is required to make {@link RayRuntimeProxy} work. - */ +/** This interface is required to make {@link RayRuntimeProxy} work. */ public interface RayRuntimeInternal extends RayRuntime { - /** - * Start runtime. - */ + /** Start runtime. */ void start(); WorkerContext getWorkerContext(); diff --git a/java/runtime/src/main/java/io/ray/runtime/RayRuntimeProxy.java b/java/runtime/src/main/java/io/ray/runtime/RayRuntimeProxy.java index 6220e1a8f..4a427295c 100644 --- a/java/runtime/src/main/java/io/ray/runtime/RayRuntimeProxy.java +++ b/java/runtime/src/main/java/io/ray/runtime/RayRuntimeProxy.java @@ -13,9 +13,7 @@ import java.lang.reflect.Method; */ public class RayRuntimeProxy implements InvocationHandler { - /** - * The original runtime. - */ + /** The original runtime. */ private AbstractRayRuntime obj; private RayRuntimeProxy(AbstractRayRuntime obj) { @@ -26,19 +24,20 @@ public class RayRuntimeProxy implements InvocationHandler { return obj; } - /** - * Generate a new instance of {@link RayRuntimeInternal} with additional context check. - */ + /** Generate a new instance of {@link RayRuntimeInternal} with additional context check. */ static RayRuntimeInternal newInstance(AbstractRayRuntime obj) { - return (RayRuntimeInternal) java.lang.reflect.Proxy - .newProxyInstance(obj.getClass().getClassLoader(), new Class[]{RayRuntimeInternal.class}, + return (RayRuntimeInternal) + java.lang.reflect.Proxy.newProxyInstance( + obj.getClass().getClassLoader(), + new Class[] {RayRuntimeInternal.class}, new RayRuntimeProxy(obj)); } @Override public Object invoke(Object proxy, Method method, Object[] args) throws Throwable { - if (isInterfaceMethod(method) && !method.getName().equals("shutdown") && !method.getName() - .equals("setAsyncContext")) { + if (isInterfaceMethod(method) + && !method.getName().equals("shutdown") + && !method.getName().equals("setAsyncContext")) { checkIsContextSet(); } try { @@ -52,9 +51,7 @@ public class RayRuntimeProxy implements InvocationHandler { } } - /** - * Whether the method is defined in the {@link RayRuntime} interface. - */ + /** Whether the method is defined in the {@link RayRuntime} interface. */ private boolean isInterfaceMethod(Method method) { try { RayRuntime.class.getMethod(method.getName(), method.getParameterTypes()); @@ -66,8 +63,8 @@ public class RayRuntimeProxy implements InvocationHandler { /** * Check if thread context is set. - *

- * This method should be invoked at the beginning of most public methods of {@link RayRuntime}, + * + *

This method should be invoked at the beginning of most public methods of {@link RayRuntime}, * otherwise the native code might crash due to thread local core worker was not set. We check it * for {@link AbstractRayRuntime} instead of {@link RayNativeRuntime} because we want to catch the * error even if the application runs in {@link RunMode#SINGLE_PROCESS} mode. diff --git a/java/runtime/src/main/java/io/ray/runtime/actor/LocalModeActorHandle.java b/java/runtime/src/main/java/io/ray/runtime/actor/LocalModeActorHandle.java index ad48590ca..6ee6b5361 100644 --- a/java/runtime/src/main/java/io/ray/runtime/actor/LocalModeActorHandle.java +++ b/java/runtime/src/main/java/io/ray/runtime/actor/LocalModeActorHandle.java @@ -9,9 +9,7 @@ import java.io.ObjectInput; import java.io.ObjectOutput; import java.util.concurrent.atomic.AtomicReference; -/** - * Implementation of actor handle for local mode. - */ +/** Implementation of actor handle for local mode. */ public class LocalModeActorHandle implements ActorHandle, Externalizable { private ActorId actorId; @@ -23,11 +21,8 @@ public class LocalModeActorHandle implements ActorHandle, Externalizable { this.previousActorTaskDummyObjectId.set(previousActorTaskDummyObjectId); } - /** - * Required by FST - */ - public LocalModeActorHandle() { - } + /** Required by FST. */ + public LocalModeActorHandle() {} @Override public ActorId getId() { diff --git a/java/runtime/src/main/java/io/ray/runtime/actor/NativeActorHandle.java b/java/runtime/src/main/java/io/ray/runtime/actor/NativeActorHandle.java index 73c88dc93..1dd4b84f5 100644 --- a/java/runtime/src/main/java/io/ray/runtime/actor/NativeActorHandle.java +++ b/java/runtime/src/main/java/io/ray/runtime/actor/NativeActorHandle.java @@ -16,9 +16,7 @@ import java.util.List; */ public abstract class NativeActorHandle implements BaseActorHandle, Externalizable { - /** - * ID of the actor. - */ + /** ID of the actor. */ byte[] actorId; private Language language; @@ -29,11 +27,8 @@ public abstract class NativeActorHandle implements BaseActorHandle, Externalizab this.language = language; } - /** - * Required by FST - */ - NativeActorHandle() { - } + /** Required by FST. */ + NativeActorHandle() {} public static NativeActorHandle create(byte[] actorId) { Language language = Language.forNumber(nativeGetLanguage(actorId)); @@ -76,7 +71,7 @@ public abstract class NativeActorHandle implements BaseActorHandle, Externalizab /** * Serialize this actor handle to bytes. * - * @return the bytes of the actor handle + *

Returns the bytes of the actor handle */ public byte[] toBytes() { return nativeSerialize(actorId); @@ -85,7 +80,7 @@ public abstract class NativeActorHandle implements BaseActorHandle, Externalizab /** * Deserialize an actor handle from bytes. * - * @return the bytes of an actor handle + *

Returns the bytes of an actor handle */ public static NativeActorHandle fromBytes(byte[] bytes) { byte[] actorId = nativeDeserialize(bytes); diff --git a/java/runtime/src/main/java/io/ray/runtime/actor/NativeActorHandleSerializer.java b/java/runtime/src/main/java/io/ray/runtime/actor/NativeActorHandleSerializer.java index 3e2723705..6c3704db9 100644 --- a/java/runtime/src/main/java/io/ray/runtime/actor/NativeActorHandleSerializer.java +++ b/java/runtime/src/main/java/io/ray/runtime/actor/NativeActorHandleSerializer.java @@ -7,20 +7,24 @@ import org.nustaq.serialization.FSTClazzInfo.FSTFieldInfo; import org.nustaq.serialization.FSTObjectInput; import org.nustaq.serialization.FSTObjectOutput; -/** - * To deal with serialization about {@link NativeActorHandle}. - */ +/** To deal with serialization about {@link NativeActorHandle}. */ public class NativeActorHandleSerializer extends FSTBasicObjectSerializer { @Override - public void writeObject(FSTObjectOutput out, Object toWrite, FSTClazzInfo clzInfo, - FSTClazzInfo.FSTFieldInfo referencedBy, int streamPosition) throws IOException { + public void writeObject( + FSTObjectOutput out, + Object toWrite, + FSTClazzInfo clzInfo, + FSTClazzInfo.FSTFieldInfo referencedBy, + int streamPosition) + throws IOException { ((NativeActorHandle) toWrite).writeExternal(out); } @Override - public void readObject(FSTObjectInput in, Object toRead, FSTClazzInfo clzInfo, - FSTFieldInfo referencedBy) throws Exception { + public void readObject( + FSTObjectInput in, Object toRead, FSTClazzInfo clzInfo, FSTFieldInfo referencedBy) + throws Exception { super.readObject(in, toRead, clzInfo, referencedBy); ((NativeActorHandle) toRead).readExternal(in); } diff --git a/java/runtime/src/main/java/io/ray/runtime/actor/NativeJavaActorHandle.java b/java/runtime/src/main/java/io/ray/runtime/actor/NativeJavaActorHandle.java index 8feca0f76..a95e1863e 100644 --- a/java/runtime/src/main/java/io/ray/runtime/actor/NativeJavaActorHandle.java +++ b/java/runtime/src/main/java/io/ray/runtime/actor/NativeJavaActorHandle.java @@ -6,18 +6,14 @@ import io.ray.runtime.generated.Common.Language; import java.io.IOException; import java.io.ObjectInput; -/** - * Java implementation of actor handle for cluster mode. - */ +/** Java implementation of actor handle for cluster mode. */ public class NativeJavaActorHandle extends NativeActorHandle implements ActorHandle { NativeJavaActorHandle(byte[] actorId) { super(actorId, Language.JAVA); } - /** - * Required by FST - */ + /** Required by FST. */ public NativeJavaActorHandle() { super(); } diff --git a/java/runtime/src/main/java/io/ray/runtime/actor/NativePyActorHandle.java b/java/runtime/src/main/java/io/ray/runtime/actor/NativePyActorHandle.java index e863cf689..97a1fa727 100644 --- a/java/runtime/src/main/java/io/ray/runtime/actor/NativePyActorHandle.java +++ b/java/runtime/src/main/java/io/ray/runtime/actor/NativePyActorHandle.java @@ -6,18 +6,14 @@ import io.ray.runtime.generated.Common.Language; import java.io.IOException; import java.io.ObjectInput; -/** - * Python actor handle implementation for cluster mode. - */ +/** Python actor handle implementation for cluster mode. */ public class NativePyActorHandle extends NativeActorHandle implements PyActorHandle { NativePyActorHandle(byte[] actorId) { super(actorId, Language.PYTHON); } - /** - * Required by FST - */ + /** Required by FST. */ public NativePyActorHandle() { super(); } diff --git a/java/runtime/src/main/java/io/ray/runtime/config/RayConfig.java b/java/runtime/src/main/java/io/ray/runtime/config/RayConfig.java index f82a7a124..2a4fbb491 100644 --- a/java/runtime/src/main/java/io/ray/runtime/config/RayConfig.java +++ b/java/runtime/src/main/java/io/ray/runtime/config/RayConfig.java @@ -20,10 +20,7 @@ import org.apache.commons.lang3.BooleanUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; -/** - * Configurations of Ray runtime. - * See `ray.default.conf` for the meaning of each field. - */ +/** Configurations of Ray runtime. See `ray.default.conf` for the meaning of each field. */ public class RayConfig { public static final String DEFAULT_CONFIG_FILE = "ray.default.conf"; @@ -31,10 +28,9 @@ public class RayConfig { private Config config; - /** - * IP of this node. if not provided, IP will be automatically detected. - */ + /** IP of this node. if not provided, IP will be automatically detected. */ public final String nodeIp; + public final WorkerType workerMode; public final RunMode runMode; private JobId jobId; @@ -64,8 +60,8 @@ public class RayConfig { private void validate() { if (workerMode == WorkerType.WORKER) { - Preconditions.checkArgument(redisAddress != null, - "Redis address must be set in worker mode."); + Preconditions.checkArgument( + redisAddress != null, "Redis address must be set in worker mode."); } } @@ -145,7 +141,8 @@ public class RayConfig { if (config.hasPath("ray.raylet.node-manager-port")) { nodeManagerPort = config.getInt("ray.raylet.node-manager-port"); } else { - Preconditions.checkState(workerMode != WorkerType.WORKER, + Preconditions.checkState( + workerMode != WorkerType.WORKER, "Worker started by raylet should accept the node manager port from raylet."); } @@ -217,9 +214,7 @@ public class RayConfig { return config; } - /** - * Renders the config value as a HOCON string. - */ + /** Renders the config value as a HOCON string. */ @Override public String toString() { // These items might be dynamically generated or mutated at runtime. @@ -257,10 +252,8 @@ public class RayConfig { } /** - * Create a RayConfig by reading configuration in the following order: - * 1. System properties. - * 2. `ray.conf` file. - * 3. `ray.default.conf` file. + * Create a RayConfig by reading configuration in the following order: 1. System properties. 2. + * `ray.conf` file. 3. `ray.default.conf` file. */ public static RayConfig create() { ConfigFactory.invalidateCaches(); @@ -274,5 +267,4 @@ public class RayConfig { config = config.withFallback(ConfigFactory.load(DEFAULT_CONFIG_FILE)); return new RayConfig(config.withOnlyPath("ray")); } - } diff --git a/java/runtime/src/main/java/io/ray/runtime/config/RunMode.java b/java/runtime/src/main/java/io/ray/runtime/config/RunMode.java index 39346804d..c86b0b3d5 100644 --- a/java/runtime/src/main/java/io/ray/runtime/config/RunMode.java +++ b/java/runtime/src/main/java/io/ray/runtime/config/RunMode.java @@ -3,13 +3,11 @@ package io.ray.runtime.config; public enum RunMode { /** - * Ray is running in one single Java process, without Raylet backend, object store, and GCS. - * It's useful for debug. + * Ray is running in one single Java process, without Raylet backend, object store, and GCS. It's + * useful for debug. */ SINGLE_PROCESS, - /** - * Ray is running on one or more nodes, with multiple processes. - */ + /** Ray is running on one or more nodes, with multiple processes. */ CLUSTER, } diff --git a/java/runtime/src/main/java/io/ray/runtime/context/LocalModeWorkerContext.java b/java/runtime/src/main/java/io/ray/runtime/context/LocalModeWorkerContext.java index 406c3d651..58d217e7a 100644 --- a/java/runtime/src/main/java/io/ray/runtime/context/LocalModeWorkerContext.java +++ b/java/runtime/src/main/java/io/ray/runtime/context/LocalModeWorkerContext.java @@ -10,9 +10,7 @@ import io.ray.runtime.generated.Common.TaskSpec; import io.ray.runtime.generated.Common.TaskType; import io.ray.runtime.task.LocalModeTaskSubmitter; -/** - * Worker context for local mode. - */ +/** Worker context for local mode. */ public class LocalModeWorkerContext implements WorkerContext { private final JobId jobId; @@ -52,8 +50,7 @@ public class LocalModeWorkerContext implements WorkerContext { } @Override - public void setCurrentClassLoader(ClassLoader currentClassLoader) { - } + public void setCurrentClassLoader(ClassLoader currentClassLoader) {} @Override public TaskType getCurrentTaskType() { diff --git a/java/runtime/src/main/java/io/ray/runtime/context/NativeWorkerContext.java b/java/runtime/src/main/java/io/ray/runtime/context/NativeWorkerContext.java index 76978d0f5..d20d48d0a 100644 --- a/java/runtime/src/main/java/io/ray/runtime/context/NativeWorkerContext.java +++ b/java/runtime/src/main/java/io/ray/runtime/context/NativeWorkerContext.java @@ -9,9 +9,7 @@ import io.ray.runtime.generated.Common.Address; import io.ray.runtime.generated.Common.TaskType; import java.nio.ByteBuffer; -/** - * Worker context for cluster mode. This is a wrapper class for worker context of core worker. - */ +/** Worker context for cluster mode. This is a wrapper class for worker context of core worker. */ public class NativeWorkerContext implements WorkerContext { private final ThreadLocal currentClassLoader = new ThreadLocal<>(); diff --git a/java/runtime/src/main/java/io/ray/runtime/context/RuntimeContextImpl.java b/java/runtime/src/main/java/io/ray/runtime/context/RuntimeContextImpl.java index fc36fac3d..2fec99d49 100644 --- a/java/runtime/src/main/java/io/ray/runtime/context/RuntimeContextImpl.java +++ b/java/runtime/src/main/java/io/ray/runtime/context/RuntimeContextImpl.java @@ -26,15 +26,16 @@ public class RuntimeContextImpl implements RuntimeContext { @Override public ActorId getCurrentActorId() { ActorId actorId = runtime.getWorkerContext().getCurrentActorId(); - Preconditions.checkState(actorId != null && !actorId.isNil(), - "This method should only be called from an actor."); + Preconditions.checkState( + actorId != null && !actorId.isNil(), "This method should only be called from an actor."); return actorId; } @Override public boolean wasCurrentActorRestarted() { TaskType currentTaskType = runtime.getWorkerContext().getCurrentTaskType(); - Preconditions.checkState(currentTaskType == TaskType.ACTOR_CREATION_TASK, + Preconditions.checkState( + currentTaskType == TaskType.ACTOR_CREATION_TASK, "This method can only be called from an actor creation task."); if (isSingleProcess()) { return false; diff --git a/java/runtime/src/main/java/io/ray/runtime/context/WorkerContext.java b/java/runtime/src/main/java/io/ray/runtime/context/WorkerContext.java index 50a5c5ce5..ff750f651 100644 --- a/java/runtime/src/main/java/io/ray/runtime/context/WorkerContext.java +++ b/java/runtime/src/main/java/io/ray/runtime/context/WorkerContext.java @@ -7,24 +7,16 @@ import io.ray.api.id.UniqueId; import io.ray.runtime.generated.Common.Address; import io.ray.runtime.generated.Common.TaskType; -/** - * The context of worker. - */ +/** The context of worker. */ public interface WorkerContext { - /** - * ID of the current worker. - */ + /** ID of the current worker. */ UniqueId getCurrentWorkerId(); - /** - * ID of the current job. - */ + /** ID of the current job. */ JobId getCurrentJobId(); - /** - * ID of the current actor. - */ + /** ID of the current actor. */ ActorId getCurrentActorId(); /** @@ -33,19 +25,13 @@ public interface WorkerContext { */ ClassLoader getCurrentClassLoader(); - /** - * Set the current class loader. - */ + /** Set the current class loader. */ void setCurrentClassLoader(ClassLoader currentClassLoader); - /** - * Type of the current task. - */ + /** Type of the current task. */ TaskType getCurrentTaskType(); - /** - * ID of the current task. - */ + /** ID of the current task. */ TaskId getCurrentTaskId(); Address getRpcAddress(); diff --git a/java/runtime/src/main/java/io/ray/runtime/exception/CrossLanguageException.java b/java/runtime/src/main/java/io/ray/runtime/exception/CrossLanguageException.java index 74b0e2076..a96ec6add 100644 --- a/java/runtime/src/main/java/io/ray/runtime/exception/CrossLanguageException.java +++ b/java/runtime/src/main/java/io/ray/runtime/exception/CrossLanguageException.java @@ -7,8 +7,10 @@ public class CrossLanguageException extends RayException { private Language language; public CrossLanguageException(io.ray.runtime.generated.Common.RayException exception) { - super(String.format("An exception raised from %s:\n%s", exception.getLanguage().name(), - exception.getFormattedExceptionString())); + super( + String.format( + "An exception raised from %s:\n%s", + exception.getLanguage().name(), exception.getFormattedExceptionString())); this.language = exception.getLanguage(); } diff --git a/java/runtime/src/main/java/io/ray/runtime/exception/RayActorException.java b/java/runtime/src/main/java/io/ray/runtime/exception/RayActorException.java index 1bb4b67cd..862e2de9f 100644 --- a/java/runtime/src/main/java/io/ray/runtime/exception/RayActorException.java +++ b/java/runtime/src/main/java/io/ray/runtime/exception/RayActorException.java @@ -5,7 +5,7 @@ import io.ray.api.id.ActorId; /** * Indicates that the actor died unexpectedly before finishing a task. * - * This exception could happen either because the actor process dies while executing a task, or + *

This exception could happen either because the actor process dies while executing a task, or * because a task is submitted to a dead actor. */ public class RayActorException extends RayException { @@ -17,8 +17,7 @@ public class RayActorException extends RayException { } public RayActorException(ActorId actorId) { - super(String.format( - "The actor %s died unexpectedly before finishing this task.", actorId)); + super(String.format("The actor %s died unexpectedly before finishing this task.", actorId)); this.actorId = actorId; } @@ -29,5 +28,4 @@ public class RayActorException extends RayException { public RayActorException(String message, Throwable cause) { super(message, cause); } - } diff --git a/java/runtime/src/main/java/io/ray/runtime/exception/RayException.java b/java/runtime/src/main/java/io/ray/runtime/exception/RayException.java index 10c00aa86..d86ebb863 100644 --- a/java/runtime/src/main/java/io/ray/runtime/exception/RayException.java +++ b/java/runtime/src/main/java/io/ray/runtime/exception/RayException.java @@ -16,8 +16,8 @@ public class RayException extends RuntimeException { } public byte[] toBytes() { - String formattedException = org.apache.commons.lang3.exception.ExceptionUtils - .getStackTrace(this); + String formattedException = + org.apache.commons.lang3.exception.ExceptionUtils.getStackTrace(this); io.ray.runtime.generated.Common.RayException.Builder builder = io.ray.runtime.generated.Common.RayException.newBuilder(); builder.setLanguage(Language.JAVA); @@ -26,13 +26,12 @@ public class RayException extends RuntimeException { return builder.build().toByteArray(); } - public static RayException fromBytes(byte[] serialized) - throws InvalidProtocolBufferException { + public static RayException fromBytes(byte[] serialized) throws InvalidProtocolBufferException { io.ray.runtime.generated.Common.RayException exception = io.ray.runtime.generated.Common.RayException.parseFrom(serialized); if (exception.getLanguage() == Language.JAVA) { - return Serializer - .decode(exception.getSerializedException().toByteArray(), RayException.class); + return Serializer.decode( + exception.getSerializedException().toByteArray(), RayException.class); } else { return new CrossLanguageException(exception); } diff --git a/java/runtime/src/main/java/io/ray/runtime/exception/RayIntentionalSystemExitException.java b/java/runtime/src/main/java/io/ray/runtime/exception/RayIntentionalSystemExitException.java index b2c168c70..cface7eac 100644 --- a/java/runtime/src/main/java/io/ray/runtime/exception/RayIntentionalSystemExitException.java +++ b/java/runtime/src/main/java/io/ray/runtime/exception/RayIntentionalSystemExitException.java @@ -1,8 +1,6 @@ package io.ray.runtime.exception; -/** - * The exception represents that there is an intentional system exit. - */ +/** The exception represents that there is an intentional system exit. */ public class RayIntentionalSystemExitException extends RuntimeException { public RayIntentionalSystemExitException(String message) { diff --git a/java/runtime/src/main/java/io/ray/runtime/exception/RayTaskException.java b/java/runtime/src/main/java/io/ray/runtime/exception/RayTaskException.java index 3b0d5327d..3d382d98b 100644 --- a/java/runtime/src/main/java/io/ray/runtime/exception/RayTaskException.java +++ b/java/runtime/src/main/java/io/ray/runtime/exception/RayTaskException.java @@ -6,8 +6,9 @@ import io.ray.runtime.util.SystemUtil; public class RayTaskException extends RayException { public RayTaskException(String message, Throwable cause) { - super(String.format("(pid=%d, ip=%s) %s", - SystemUtil.pid(), NetworkUtil.getIpAddress(null), message), cause); + super( + String.format( + "(pid=%d, ip=%s) %s", SystemUtil.pid(), NetworkUtil.getIpAddress(null), message), + cause); } - } diff --git a/java/runtime/src/main/java/io/ray/runtime/exception/RayWorkerException.java b/java/runtime/src/main/java/io/ray/runtime/exception/RayWorkerException.java index acb47c806..92504317c 100644 --- a/java/runtime/src/main/java/io/ray/runtime/exception/RayWorkerException.java +++ b/java/runtime/src/main/java/io/ray/runtime/exception/RayWorkerException.java @@ -1,8 +1,6 @@ package io.ray.runtime.exception; -/** - * Indicates that the worker died unexpectedly while executing a task. - */ +/** Indicates that the worker died unexpectedly while executing a task. */ public class RayWorkerException extends RayException { public RayWorkerException() { @@ -16,5 +14,4 @@ public class RayWorkerException extends RayException { public RayWorkerException(String message, Throwable cause) { super(message, cause); } - } diff --git a/java/runtime/src/main/java/io/ray/runtime/exception/UnreconstructableException.java b/java/runtime/src/main/java/io/ray/runtime/exception/UnreconstructableException.java index 719b22141..cbde03746 100644 --- a/java/runtime/src/main/java/io/ray/runtime/exception/UnreconstructableException.java +++ b/java/runtime/src/main/java/io/ray/runtime/exception/UnreconstructableException.java @@ -3,20 +3,20 @@ package io.ray.runtime.exception; import io.ray.api.id.ObjectId; /** - * Indicates that an object is lost (either evicted or explicitly deleted) and cannot be - * restarted. + * Indicates that an object is lost (either evicted or explicitly deleted) and cannot be restarted. * - * Note, this exception only happens for actor objects. If actor's current state is after object's - * creating task, the actor cannot re-run the task to reconstruct the object. + *

Note, this exception only happens for actor objects. If actor's current state is after + * object's creating task, the actor cannot re-run the task to reconstruct the object. */ public class UnreconstructableException extends RayException { public ObjectId objectId; public UnreconstructableException(ObjectId objectId) { - super(String.format( - "Object %s is lost (either evicted or explicitly deleted) and cannot be reconstructed.", - objectId)); + super( + String.format( + "Object %s is lost (either evicted or explicitly deleted) and cannot be reconstructed.", + objectId)); this.objectId = objectId; } @@ -27,5 +27,4 @@ public class UnreconstructableException extends RayException { public UnreconstructableException(String message, Throwable cause) { super(message, cause); } - } diff --git a/java/runtime/src/main/java/io/ray/runtime/functionmanager/FunctionDescriptor.java b/java/runtime/src/main/java/io/ray/runtime/functionmanager/FunctionDescriptor.java index 4e0711d2d..6d2e605f7 100644 --- a/java/runtime/src/main/java/io/ray/runtime/functionmanager/FunctionDescriptor.java +++ b/java/runtime/src/main/java/io/ray/runtime/functionmanager/FunctionDescriptor.java @@ -6,18 +6,14 @@ import java.util.List; /** * Base interface of a Ray task's function descriptor. * - * A function descriptor is a list of strings that can uniquely describe a function. It's used to + *

A function descriptor is a list of strings that can uniquely describe a function. It's used to * load a function in workers. */ public interface FunctionDescriptor { - /** - * @return A list of strings represents the functions. - */ + /** Returns A list of strings represents the functions. */ List toList(); - /** - * @return The language of the function. - */ + /** Returns The language of the function. */ Language getLanguage(); } diff --git a/java/runtime/src/main/java/io/ray/runtime/functionmanager/FunctionManager.java b/java/runtime/src/main/java/io/ray/runtime/functionmanager/FunctionManager.java index c108139ac..d26a13dca 100644 --- a/java/runtime/src/main/java/io/ray/runtime/functionmanager/FunctionManager.java +++ b/java/runtime/src/main/java/io/ray/runtime/functionmanager/FunctionManager.java @@ -34,9 +34,7 @@ import org.objectweb.asm.Type; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Manages functions by job id. - */ +/** Manages functions by job id. */ public class FunctionManager { private static final Logger LOGGER = LoggerFactory.getLogger(FunctionManager.class); @@ -52,21 +50,16 @@ public class FunctionManager { private static final ThreadLocal, JavaFunctionDescriptor>> RAY_FUNC_CACHE = ThreadLocal.withInitial(WeakHashMap::new); - /** - * Mapping from the job id to the functions that belong to this job. - */ + /** Mapping from the job id to the functions that belong to this job. */ private ConcurrentMap jobFunctionTables = new ConcurrentHashMap<>(); - /** - * The resource path which we can load the job's jar resources. - */ + /** The resource path which we can load the job's jar resources. */ private final List codeSearchPath; /** * Construct a FunctionManager with the specified code search path. * - * @param codeSearchPath The specified job resource that can store the job's - * resources. + * @param codeSearchPath The specified job resource that can store the job's resources. */ public FunctionManager(List codeSearchPath) { this.codeSearchPath = codeSearchPath; @@ -76,8 +69,7 @@ public class FunctionManager { * Get the RayFunction from a RayFunc instance (a lambda). * * @param jobId current job id. - * @param func The lambda. - * @return A RayFunction object. + * @param func The lambda. Returns A RayFunction object. */ public RayFunction getFunction(JobId jobId, RayFunc func) { JavaFunctionDescriptor functionDescriptor = RAY_FUNC_CACHE.get().get(func.getClass()); @@ -97,12 +89,10 @@ public class FunctionManager { /** * Get the RayFunction from a function descriptor. * - * @param jobId Current job id. - * @param functionDescriptor The function descriptor. - * @return A RayFunction object. + * @param jobId Current job id. + * @param functionDescriptor The function descriptor. Returns A RayFunction object. */ - public RayFunction getFunction(JobId jobId, - JavaFunctionDescriptor functionDescriptor) { + public RayFunction getFunction(JobId jobId, JavaFunctionDescriptor functionDescriptor) { JobFunctionTable jobFunctionTable = jobFunctionTables.get(jobId); if (jobFunctionTable == null) { synchronized (this) { @@ -121,30 +111,37 @@ public class FunctionManager { if (codeSearchPath == null || codeSearchPath.isEmpty()) { classLoader = getClass().getClassLoader(); } else { - URL[] urls = codeSearchPath.stream() - .filter(p -> StringUtils.isNotBlank(p) && Files.exists(Paths.get(p))) - .flatMap(p -> { - try { - if (!Files.isDirectory(Paths.get(p))) { - if (!p.endsWith(".jar")) { - return Stream.of(Paths.get(p).getParent().toAbsolutePath().toUri().toURL()); - } else { - return Stream.of(Paths.get(p).toAbsolutePath().toUri().toURL()); - } - } else { - List subUrls = new ArrayList<>(); - subUrls.add(Paths.get(p).toAbsolutePath().toUri().toURL()); - Collection jars = FileUtils.listFiles(new File(p), - new RegexFileFilter(".*\\.jar"), DirectoryFileFilter.DIRECTORY); - for (File jar : jars) { - subUrls.add(jar.toPath().toUri().toURL()); - } - return subUrls.stream(); - } - } catch (MalformedURLException e) { - throw new RuntimeException(String.format("Illegal %s resource path", p)); - } - }).toArray(URL[]::new); + URL[] urls = + codeSearchPath.stream() + .filter(p -> StringUtils.isNotBlank(p) && Files.exists(Paths.get(p))) + .flatMap( + p -> { + try { + if (!Files.isDirectory(Paths.get(p))) { + if (!p.endsWith(".jar")) { + return Stream.of( + Paths.get(p).getParent().toAbsolutePath().toUri().toURL()); + } else { + return Stream.of(Paths.get(p).toAbsolutePath().toUri().toURL()); + } + } else { + List subUrls = new ArrayList<>(); + subUrls.add(Paths.get(p).toAbsolutePath().toUri().toURL()); + Collection jars = + FileUtils.listFiles( + new File(p), + new RegexFileFilter(".*\\.jar"), + DirectoryFileFilter.DIRECTORY); + for (File jar : jars) { + subUrls.add(jar.toPath().toUri().toURL()); + } + return subUrls.stream(); + } + } catch (MalformedURLException e) { + throw new RuntimeException(String.format("Illegal %s resource path", p)); + } + }) + .toArray(URL[]::new); classLoader = new URLClassLoader(urls); LOGGER.debug("Resource loaded for job {} from path {}.", jobId, urls); } @@ -152,18 +149,12 @@ public class FunctionManager { return new JobFunctionTable(classLoader); } - /** - * Manages all functions that belong to one job. - */ + /** Manages all functions that belong to one job. */ static class JobFunctionTable { - /** - * The job's corresponding class loader. - */ + /** The job's corresponding class loader. */ final ClassLoader classLoader; - /** - * Functions per class, per function name + type descriptor. - */ + /** Functions per class, per function name + type descriptor. */ ConcurrentMap, RayFunction>> functions; JobFunctionTable(ClassLoader classLoader) { @@ -187,19 +178,18 @@ public class FunctionManager { if (func == null) { if (classFunctions.containsKey(key)) { throw new RuntimeException( - String.format("RayFunction %s is overloaded, the signature can't be empty.", - descriptor.toString())); + String.format( + "RayFunction %s is overloaded, the signature can't be empty.", + descriptor.toString())); } else { throw new RuntimeException( - String.format("RayFunction %s not found", descriptor.toString())); + String.format("RayFunction %s not found", descriptor.toString())); } } return func; } - /** - * Load all functions from a class. - */ + /** Load all functions from a class. */ Map, RayFunction> loadFunctionsForClass(String className) { // If RayFunction is null, the function is overloaded. Map, RayFunction> map = new HashMap<>(); @@ -232,8 +222,9 @@ public class FunctionManager { final Type type = e instanceof Method ? Type.getType((Method) e) : Type.getType((Constructor) e); final String signature = type.getDescriptor(); - RayFunction rayFunction = new RayFunction(e, classLoader, - new JavaFunctionDescriptor(className, methodName, signature)); + RayFunction rayFunction = + new RayFunction( + e, classLoader, new JavaFunctionDescriptor(className, methodName, signature)); map.put(ImmutablePair.of(methodName, signature), rayFunction); // For cross language call java function without signature final Pair emptyDescriptor = ImmutablePair.of(methodName, ""); diff --git a/java/runtime/src/main/java/io/ray/runtime/functionmanager/JavaFunctionDescriptor.java b/java/runtime/src/main/java/io/ray/runtime/functionmanager/JavaFunctionDescriptor.java index 30d5cd881..a031e1096 100644 --- a/java/runtime/src/main/java/io/ray/runtime/functionmanager/JavaFunctionDescriptor.java +++ b/java/runtime/src/main/java/io/ray/runtime/functionmanager/JavaFunctionDescriptor.java @@ -5,22 +5,14 @@ import com.google.common.collect.ImmutableList; import io.ray.runtime.generated.Common.Language; import java.util.List; -/** - * Represents metadata of Java function. - */ +/** Represents metadata of Java function. */ public final class JavaFunctionDescriptor implements FunctionDescriptor { - /** - * Function's class name. - */ + /** Function's class name. */ public final String className; - /** - * Function's name. - */ + /** Function's name. */ public final String name; - /** - * Function's signature. - */ + /** Function's signature. */ public final String signature; public JavaFunctionDescriptor(String className, String name, String signature) { @@ -43,9 +35,9 @@ public final class JavaFunctionDescriptor implements FunctionDescriptor { return false; } JavaFunctionDescriptor that = (JavaFunctionDescriptor) o; - return Objects.equal(className, that.className) && - Objects.equal(name, that.name) && - Objects.equal(signature, that.signature); + return Objects.equal(className, that.className) + && Objects.equal(name, that.name) + && Objects.equal(signature, that.signature); } @Override diff --git a/java/runtime/src/main/java/io/ray/runtime/functionmanager/PyFunctionDescriptor.java b/java/runtime/src/main/java/io/ray/runtime/functionmanager/PyFunctionDescriptor.java index bbf26c57f..e96139905 100644 --- a/java/runtime/src/main/java/io/ray/runtime/functionmanager/PyFunctionDescriptor.java +++ b/java/runtime/src/main/java/io/ray/runtime/functionmanager/PyFunctionDescriptor.java @@ -5,9 +5,7 @@ import io.ray.runtime.generated.Common.Language; import java.util.Arrays; import java.util.List; -/** - * Represents metadata of a Python function. - */ +/** Represents metadata of a Python function. */ public class PyFunctionDescriptor implements FunctionDescriptor { public String moduleName; @@ -36,9 +34,9 @@ public class PyFunctionDescriptor implements FunctionDescriptor { return false; } PyFunctionDescriptor that = (PyFunctionDescriptor) o; - return Objects.equal(moduleName, that.moduleName) && - Objects.equal(className, that.className) && - Objects.equal(functionName, that.functionName); + return Objects.equal(moduleName, that.moduleName) + && Objects.equal(className, that.className) + && Objects.equal(functionName, that.functionName); } @Override @@ -56,4 +54,3 @@ public class PyFunctionDescriptor implements FunctionDescriptor { return Language.PYTHON; } } - diff --git a/java/runtime/src/main/java/io/ray/runtime/functionmanager/RayFunction.java b/java/runtime/src/main/java/io/ray/runtime/functionmanager/RayFunction.java index d7d472ddb..dfa251386 100644 --- a/java/runtime/src/main/java/io/ray/runtime/functionmanager/RayFunction.java +++ b/java/runtime/src/main/java/io/ray/runtime/functionmanager/RayFunction.java @@ -5,50 +5,36 @@ import java.lang.reflect.Executable; import java.lang.reflect.Method; import java.util.Optional; -/** - * Represents a Ray function (either a Method or a Constructor in Java) and its metadata. - */ +/** Represents a Ray function (either a Method or a Constructor in Java) and its metadata. */ public class RayFunction { - /** - * The executor object, can be either a Method or a Constructor. - */ + /** The executor object, can be either a Method or a Constructor. */ public final Executable executable; - /** - * This function's class loader. - */ + /** This function's class loader. */ public final ClassLoader classLoader; - /** - * Function's metadata. - */ + /** Function's metadata. */ public final JavaFunctionDescriptor functionDescriptor; - public RayFunction(Executable executable, ClassLoader classLoader, - JavaFunctionDescriptor functionDescriptor) { + public RayFunction( + Executable executable, ClassLoader classLoader, JavaFunctionDescriptor functionDescriptor) { this.executable = executable; this.classLoader = classLoader; this.functionDescriptor = functionDescriptor; } - /** - * @return True if it's a constructor, otherwise it's a method. - */ + /** Returns True if it's a constructor, otherwise it's a method. */ public boolean isConstructor() { return executable instanceof Constructor; } - /** - * @return The underlying constructor object. - */ + /** Returns The underlying constructor object. */ public Constructor getConstructor() { return (Constructor) executable; } - /** - * @return The underlying method object. - */ + /** Returns The underlying method object. */ public Method getMethod() { return (Method) executable; } @@ -57,9 +43,7 @@ public class RayFunction { return functionDescriptor; } - /** - * @return Whether this function has a return value. - */ + /** Returns Whether this function has a return value. */ public boolean hasReturn() { if (isConstructor()) { return true; @@ -68,9 +52,7 @@ public class RayFunction { } } - /** - * @return Return type. - */ + /** Returns Return type. */ public Optional> getReturnType() { if (hasReturn()) { return Optional.of(((Method) executable).getReturnType()); diff --git a/java/runtime/src/main/java/io/ray/runtime/gcs/GcsClient.java b/java/runtime/src/main/java/io/ray/runtime/gcs/GcsClient.java index 41a82c2d5..df34212e7 100644 --- a/java/runtime/src/main/java/io/ray/runtime/gcs/GcsClient.java +++ b/java/runtime/src/main/java/io/ray/runtime/gcs/GcsClient.java @@ -20,9 +20,7 @@ import org.apache.commons.lang3.ArrayUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * An implementation of GcsClient. - */ +/** An implementation of GcsClient. */ public class GcsClient { private static Logger LOGGER = LoggerFactory.getLogger(GcsClient.class); private RedisClient primary; @@ -35,9 +33,9 @@ public class GcsClient { } /** - * Get placement group by {@link PlacementGroupId} - * @param placementGroupId Id of placement group. - * @return The placement group. + * Get placement group by {@link PlacementGroupId}. + * + * @param placementGroupId Id of placement group. Returns The placement group. */ public PlacementGroup getPlacementGroupInfo(PlacementGroupId placementGroupId) { byte[] result = globalStateAccessor.getPlacementGroupInfo(placementGroupId); @@ -46,7 +44,8 @@ public class GcsClient { /** * Get all placement groups in this cluster. - * @return All placement groups. + * + *

Returns All placement groups. */ public List getAllPlacementGroupInfo() { List results = globalStateAccessor.getAllPlacementGroupInfo(); @@ -71,15 +70,20 @@ public class GcsClient { } catch (InvalidProtocolBufferException e) { throw new RuntimeException("Received invalid protobuf data from GCS."); } - final UniqueId nodeId = UniqueId - .fromByteBuffer(data.getNodeId().asReadOnlyByteBuffer()); + final UniqueId nodeId = UniqueId.fromByteBuffer(data.getNodeId().asReadOnlyByteBuffer()); // NOTE(lingxuan.zlx): we assume no duplicated node id in fetched node list // and it's only one final state for each node in recorded table. - NodeInfo nodeInfo = new NodeInfo( - nodeId, data.getNodeManagerAddress(), data.getNodeManagerHostname(), - data.getNodeManagerPort(), data.getObjectStoreSocketName(), data.getRayletSocketName(), - data.getState() == GcsNodeInfo.GcsNodeState.ALIVE, new HashMap<>()); + NodeInfo nodeInfo = + new NodeInfo( + nodeId, + data.getNodeManagerAddress(), + data.getNodeManagerHostname(), + data.getNodeManagerPort(), + data.getObjectStoreSocketName(), + data.getRayletSocketName(), + data.getState() == GcsNodeInfo.GcsNodeState.ALIVE, + new HashMap<>()); nodes.put(nodeId, nodeInfo); } @@ -119,9 +123,7 @@ public class GcsClient { return resources; } - /** - * If the actor exists in GCS. - */ + /** If the actor exists in GCS. */ public boolean actorExists(ActorId actorId) { byte[] result = globalStateAccessor.getActorInfo(actorId); return result != null; @@ -149,9 +151,7 @@ public class GcsClient { return JobId.fromInt(jobCounter); } - /** - * Destroy global state accessor when ray native runtime will be shutdown. - */ + /** Destroy global state accessor when ray native runtime will be shutdown. */ public void destroy() { // Only ray shutdown should call gcs client destroy. LOGGER.debug("Destroying global state accessor."); diff --git a/java/runtime/src/main/java/io/ray/runtime/gcs/GcsClientOptions.java b/java/runtime/src/main/java/io/ray/runtime/gcs/GcsClientOptions.java index a21306c92..8210bcba6 100644 --- a/java/runtime/src/main/java/io/ray/runtime/gcs/GcsClientOptions.java +++ b/java/runtime/src/main/java/io/ray/runtime/gcs/GcsClientOptions.java @@ -3,9 +3,7 @@ package io.ray.runtime.gcs; import com.google.common.base.Preconditions; import io.ray.runtime.config.RayConfig; -/** - * Options to create GCS Client. - */ +/** Options to create GCS Client. */ public class GcsClientOptions { public String ip; public int port; diff --git a/java/runtime/src/main/java/io/ray/runtime/gcs/GlobalStateAccessor.java b/java/runtime/src/main/java/io/ray/runtime/gcs/GlobalStateAccessor.java index 663e62a77..c9efcbcb8 100644 --- a/java/runtime/src/main/java/io/ray/runtime/gcs/GlobalStateAccessor.java +++ b/java/runtime/src/main/java/io/ray/runtime/gcs/GlobalStateAccessor.java @@ -6,18 +6,15 @@ import io.ray.api.id.PlacementGroupId; import io.ray.api.id.UniqueId; import java.util.List; -/** - * `GlobalStateAccessor` is used for accessing information from GCS. - * - **/ +/** `GlobalStateAccessor` is used for accessing information from GCS. */ public class GlobalStateAccessor { // NOTE(lingxuan.zlx): this is a singleton, it can not be changed during a Ray session. // Native pointer to the C++ GcsStateAccessor. private Long globalStateAccessorNativePointer = 0L; private static GlobalStateAccessor globalStateAccessor; - public static synchronized GlobalStateAccessor getInstance(String redisAddress, - String redisPassword) { + public static synchronized GlobalStateAccessor getInstance( + String redisAddress, String redisPassword) { if (null == globalStateAccessor) { globalStateAccessor = new GlobalStateAccessor(redisAddress, redisPassword); } @@ -32,8 +29,7 @@ public class GlobalStateAccessor { } private GlobalStateAccessor(String redisAddress, String redisPassword) { - globalStateAccessorNativePointer = - nativeCreateGlobalStateAccessor(redisAddress, redisPassword); + globalStateAccessorNativePointer = nativeCreateGlobalStateAccessor(redisAddress, redisPassword); validateGlobalStateAccessorPointer(); connect(); } @@ -43,13 +39,12 @@ public class GlobalStateAccessor { } private void validateGlobalStateAccessorPointer() { - Preconditions.checkState(globalStateAccessorNativePointer != 0, + Preconditions.checkState( + globalStateAccessorNativePointer != 0, "Global state accessor native pointer must not be 0."); } - /** - * @return A list of job info with JobInfo protobuf schema. - */ + /** Returns A list of job info with JobInfo protobuf schema. */ public List getAllJobInfo() { // Fetch a job list with protobuf bytes format from GCS. synchronized (GlobalStateAccessor.class) { @@ -58,9 +53,7 @@ public class GlobalStateAccessor { } } - /** - * @return A list of node info with GcsNodeInfo protobuf schema. - */ + /** Returns A list of node info with GcsNodeInfo protobuf schema. */ public List getAllNodeInfo() { // Fetch a node list with protobuf bytes format from GCS. synchronized (GlobalStateAccessor.class) { @@ -70,6 +63,8 @@ public class GlobalStateAccessor { } /** + * Get node resource info. + * * @param nodeId node unique id. * @return A map of node resource info in protobuf schema. */ @@ -83,8 +78,8 @@ public class GlobalStateAccessor { public byte[] getPlacementGroupInfo(PlacementGroupId placementGroupId) { synchronized (GlobalStateAccessor.class) { validateGlobalStateAccessorPointer(); - return nativeGetPlacementGroupInfo(globalStateAccessorNativePointer, - placementGroupId.getBytes()); + return nativeGetPlacementGroupInfo( + globalStateAccessorNativePointer, placementGroupId.getBytes()); } } @@ -102,9 +97,7 @@ public class GlobalStateAccessor { } } - /** - * @return A list of actor info with ActorInfo protobuf schema. - */ + /** Returns A list of actor info with ActorInfo protobuf schema. */ public List getAllActorInfo() { // Fetch a actor list with protobuf bytes format from GCS. synchronized (GlobalStateAccessor.class) { @@ -113,9 +106,7 @@ public class GlobalStateAccessor { } } - /** - * @return An actor info with ActorInfo protobuf schema. - */ + /** Returns An actor info with ActorInfo protobuf schema. */ public byte[] getActorInfo(ActorId actorId) { // Fetch an actor with protobuf bytes format from GCS. synchronized (GlobalStateAccessor.class) { @@ -152,8 +143,7 @@ public class GlobalStateAccessor { private native byte[] nativeGetActorInfo(long nativePtr, byte[] actorId); - private native byte[] nativeGetPlacementGroupInfo(long nativePtr, - byte[] placementGroupId); + private native byte[] nativeGetPlacementGroupInfo(long nativePtr, byte[] placementGroupId); private native List nativeGetAllPlacementGroupInfo(long nativePtr); } diff --git a/java/runtime/src/main/java/io/ray/runtime/gcs/RedisClient.java b/java/runtime/src/main/java/io/ray/runtime/gcs/RedisClient.java index e9802eace..77004a849 100644 --- a/java/runtime/src/main/java/io/ray/runtime/gcs/RedisClient.java +++ b/java/runtime/src/main/java/io/ray/runtime/gcs/RedisClient.java @@ -7,9 +7,7 @@ import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; -/** - * Redis client class. - */ +/** Redis client class. */ public class RedisClient { private static final int JEDIS_POOL_SIZE = 1; @@ -23,19 +21,20 @@ public class RedisClient { public RedisClient(String redisAddress, String password) { String[] ipAndPort = redisAddress.split(":"); if (ipAndPort.length != 2) { - throw new IllegalArgumentException("The argument redisAddress " + - "should be formatted as ip:port."); + throw new IllegalArgumentException( + "The argument redisAddress " + "should be formatted as ip:port."); } JedisPoolConfig jedisPoolConfig = new JedisPoolConfig(); jedisPoolConfig.setMaxTotal(JEDIS_POOL_SIZE); if (Strings.isNullOrEmpty(password)) { - jedisPool = new JedisPool(jedisPoolConfig, - ipAndPort[0], Integer.parseInt(ipAndPort[1]), 30000); + jedisPool = + new JedisPool(jedisPoolConfig, ipAndPort[0], Integer.parseInt(ipAndPort[1]), 30000); } else { - jedisPool = new JedisPool(jedisPoolConfig, ipAndPort[0], - Integer.parseInt(ipAndPort[1]), 30000, password); + jedisPool = + new JedisPool( + jedisPoolConfig, ipAndPort[0], Integer.parseInt(ipAndPort[1]), 30000, password); } } @@ -89,7 +88,7 @@ public class RedisClient { /** * Return the specified elements of the list stored at the specified key. * - * @return Multi bulk reply, specifically a list of elements in the specified range. + *

Returns Multi bulk reply, specifically a list of elements in the specified range. */ public List lrange(byte[] key, long start, long end) { try (Jedis jedis = jedisPool.getResource()) { @@ -97,9 +96,7 @@ public class RedisClient { } } - /** - * Whether the key exists in Redis. - */ + /** Whether the key exists in Redis. */ public boolean exists(byte[] key) { try (Jedis jedis = jedisPool.getResource()) { return jedis.exists(key); diff --git a/java/runtime/src/main/java/io/ray/runtime/metric/Count.java b/java/runtime/src/main/java/io/ray/runtime/metric/Count.java index 19b75f851..c43a625f9 100644 --- a/java/runtime/src/main/java/io/ray/runtime/metric/Count.java +++ b/java/runtime/src/main/java/io/ray/runtime/metric/Count.java @@ -1,14 +1,11 @@ package io.ray.runtime.metric; import com.google.common.base.Preconditions; - import java.util.Map; import java.util.concurrent.atomic.DoubleAdder; import java.util.stream.Collectors; -/** - * Count measurement is mapped to count object in stats and counts the number. - */ +/** Count measurement is mapped to count object in stats and counts the number. */ public class Count extends Metric { private DoubleAdder count; @@ -16,8 +13,12 @@ public class Count extends Metric { public Count(String name, String description, String unit, Map tags) { super(name, tags); count = new DoubleAdder(); - metricNativePointer = NativeMetric.registerCountNative(name, description, unit, - tags.keySet().stream().map(TagKey::getTagKey).collect(Collectors.toList())); + metricNativePointer = + NativeMetric.registerCountNative( + name, + description, + unit, + tags.keySet().stream().map(TagKey::getTagKey).collect(Collectors.toList())); Preconditions.checkState(metricNativePointer != 0, "Count native pointer must not be 0."); } diff --git a/java/runtime/src/main/java/io/ray/runtime/metric/Gauge.java b/java/runtime/src/main/java/io/ray/runtime/metric/Gauge.java index 63b4c7c28..bf9066e4f 100644 --- a/java/runtime/src/main/java/io/ray/runtime/metric/Gauge.java +++ b/java/runtime/src/main/java/io/ray/runtime/metric/Gauge.java @@ -4,16 +4,17 @@ import com.google.common.base.Preconditions; import java.util.Map; import java.util.stream.Collectors; - -/** - * Gauge measurement is mapped to gauge object in stats and is recording the last value. - */ +/** Gauge measurement is mapped to gauge object in stats and is recording the last value. */ public class Gauge extends Metric { public Gauge(String name, String description, String unit, Map tags) { super(name, tags); - metricNativePointer = NativeMetric.registerGaugeNative(name, description, unit, - tags.keySet().stream().map(TagKey::getTagKey).collect(Collectors.toList())); + metricNativePointer = + NativeMetric.registerGaugeNative( + name, + description, + unit, + tags.keySet().stream().map(TagKey::getTagKey).collect(Collectors.toList())); Preconditions.checkState(metricNativePointer != 0, "Gauge native pointer must not be 0."); } @@ -31,4 +32,3 @@ public class Gauge extends Metric { this.value.set(value); } } - diff --git a/java/runtime/src/main/java/io/ray/runtime/metric/Histogram.java b/java/runtime/src/main/java/io/ray/runtime/metric/Histogram.java index 5260f4101..4d6fe11b7 100644 --- a/java/runtime/src/main/java/io/ray/runtime/metric/Histogram.java +++ b/java/runtime/src/main/java/io/ray/runtime/metric/Histogram.java @@ -1,7 +1,6 @@ package io.ray.runtime.metric; import com.google.common.base.Preconditions; - import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -9,23 +8,30 @@ import java.util.Map; import java.util.stream.Collectors; /** - * Histogram measurement is mapped to histogram object in stats. - * In order to reduce JNI calls overhead, a memory historical window is used - * for storing transient value and we assume its max size is 100. + * Histogram measurement is mapped to histogram object in stats. In order to reduce JNI calls + * overhead, a memory historical window is used for storing transient value and we assume its max + * size is 100. */ public class Histogram extends Metric { private List histogramWindow; public static final int HISTOGRAM_WINDOW_SIZE = 100; - public Histogram(String name, String description, String unit, List boundaries, - Map tags) { + public Histogram( + String name, + String description, + String unit, + List boundaries, + Map tags) { super(name, tags); - metricNativePointer = NativeMetric.registerHistogramNative(name, description, unit, - boundaries.stream().mapToDouble(Double::doubleValue).toArray(), - tags.keySet().stream().map(TagKey::getTagKey).collect(Collectors.toList())); - Preconditions.checkState(metricNativePointer != 0, - "Histogram native pointer must not be 0."); + metricNativePointer = + NativeMetric.registerHistogramNative( + name, + description, + unit, + boundaries.stream().mapToDouble(Double::doubleValue).toArray(), + tags.keySet().stream().map(TagKey::getTagKey).collect(Collectors.toList())); + Preconditions.checkState(metricNativePointer != 0, "Histogram native pointer must not be 0."); histogramWindow = Collections.synchronizedList(new ArrayList<>()); } diff --git a/java/runtime/src/main/java/io/ray/runtime/metric/Metric.java b/java/runtime/src/main/java/io/ray/runtime/metric/Metric.java index 271a93f84..961cbfe9a 100644 --- a/java/runtime/src/main/java/io/ray/runtime/metric/Metric.java +++ b/java/runtime/src/main/java/io/ray/runtime/metric/Metric.java @@ -8,8 +8,8 @@ import java.util.Map; import java.util.stream.Collectors; /** - * Class metric is mapped to stats metric object in core worker. - * it must be in categories set [Gague, Count, Sum, Histogram]. + * Class metric is mapped to stats metric object in core worker. it must be in categories set + * [Gague, Count, Sum, Histogram]. */ public abstract class Metric { protected String name; @@ -33,9 +33,7 @@ public abstract class Metric { // Metric data will be flushed into stats view data inside core worker immediately after // record is called. - /** - * Flush records to stats in last aggregator. - */ + /** Flush records to stats in last aggregator. */ public void record() { Preconditions.checkState(metricNativePointer != 0, "Metric native pointer must not be 0."); // Get tag key list from map; @@ -46,20 +44,22 @@ public abstract class Metric { tagValues.add(entry.getValue()); } // Get tag value list from map; - NativeMetric.recordNative(metricNativePointer, getAndReset(), nativeTagKeyList.stream() - .map(TagKey::getTagKey).collect(Collectors.toList()), tagValues); + NativeMetric.recordNative( + metricNativePointer, + getAndReset(), + nativeTagKeyList.stream().map(TagKey::getTagKey).collect(Collectors.toList()), + tagValues); } /** * Get the value to record and then reset. * - * @return latest updating value. + *

Returns latest updating value. */ protected abstract double getAndReset(); /** - * Update gauge value without tags. - * Update metric info for user. + * Update gauge value without tags. Update metric info for user. * * @param value latest value for updating */ @@ -69,21 +69,18 @@ public abstract class Metric { * Update gauge value with dynamic tag values. * * @param value latest value for updating - * @param tags tag map + * @param tags tag map */ public void update(double value, Map tags) { update(value); this.tags = tags; } - /** - * Deallocate object from stats and reset native pointer in null. - */ + /** Deallocate object from stats and reset native pointer in null. */ public void unregister() { if (0 != metricNativePointer) { NativeMetric.unregisterMetricNative(metricNativePointer); } metricNativePointer = 0; } - } diff --git a/java/runtime/src/main/java/io/ray/runtime/metric/MetricConfig.java b/java/runtime/src/main/java/io/ray/runtime/metric/MetricConfig.java index f7890b096..695087915 100644 --- a/java/runtime/src/main/java/io/ray/runtime/metric/MetricConfig.java +++ b/java/runtime/src/main/java/io/ray/runtime/metric/MetricConfig.java @@ -2,17 +2,16 @@ package io.ray.runtime.metric; import com.google.common.base.MoreObjects; -/** - * Configurations of the metric. - */ +/** Configurations of the metric. */ public class MetricConfig { private static final long DEFAULT_TIME_INTERVAL_MS = 5000L; private static final int DEFAULT_THREAD_POLL_SIZE = 1; private static final long DEFAULT_SHUTDOWN_WAIT_TIME_MS = 3000L; - public static final MetricConfig DEFAULT_CONFIG = new MetricConfig(DEFAULT_TIME_INTERVAL_MS, - DEFAULT_THREAD_POLL_SIZE, DEFAULT_SHUTDOWN_WAIT_TIME_MS); + public static final MetricConfig DEFAULT_CONFIG = + new MetricConfig( + DEFAULT_TIME_INTERVAL_MS, DEFAULT_THREAD_POLL_SIZE, DEFAULT_SHUTDOWN_WAIT_TIME_MS); private final long timeIntervalMs; private final int threadPoolSize; @@ -73,6 +72,4 @@ public class MetricConfig { return this; } } - - -} \ No newline at end of file +} diff --git a/java/runtime/src/main/java/io/ray/runtime/metric/MetricId.java b/java/runtime/src/main/java/io/ray/runtime/metric/MetricId.java index 9c6fa9165..047c2f6ab 100644 --- a/java/runtime/src/main/java/io/ray/runtime/metric/MetricId.java +++ b/java/runtime/src/main/java/io/ray/runtime/metric/MetricId.java @@ -5,10 +5,9 @@ import java.util.Map; import java.util.Objects; /** - * MetricId represents a metric with a given type, name and tags. - * If two metrics have the same type and name but different tags(including key and value), they have - * a different MetricId. And in this way, {@link MetricRegistry} can register two metrics with same - * name but different tags. + * MetricId represents a metric with a given type, name and tags. If two metrics have the same type + * and name but different tags(including key and value), they have a different MetricId. And in this + * way, {@link MetricRegistry} can register two metrics with same name but different tags. */ public class MetricId { @@ -31,9 +30,9 @@ public class MetricId { return false; } MetricId metricId = (MetricId) o; - return type == metricId.type && - Objects.equals(name, metricId.name) && - Objects.equals(tags, metricId.tags); + return type == metricId.type + && Objects.equals(name, metricId.name) + && Objects.equals(tags, metricId.tags); } @Override @@ -61,4 +60,4 @@ public class MetricId { public Map getTags() { return tags; } -} \ No newline at end of file +} diff --git a/java/runtime/src/main/java/io/ray/runtime/metric/MetricRegistry.java b/java/runtime/src/main/java/io/ray/runtime/metric/MetricRegistry.java index 20288512a..3c3a25bd1 100644 --- a/java/runtime/src/main/java/io/ray/runtime/metric/MetricRegistry.java +++ b/java/runtime/src/main/java/io/ray/runtime/metric/MetricRegistry.java @@ -9,9 +9,7 @@ import java.util.concurrent.TimeUnit; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * MetricRegistry is a registry for metrics to be registered and updates metrics. - */ +/** MetricRegistry is a registry for metrics to be registered and updates metrics. */ public class MetricRegistry { public static final MetricRegistry DEFAULT_REGISTRY = new MetricRegistry(); @@ -32,10 +30,15 @@ public class MetricRegistry { synchronized (this) { if (!isRunning) { this.metricConfig = metricConfig; - scheduledExecutorService = new ScheduledThreadPoolExecutor(metricConfig.threadPoolSize(), - new ThreadFactoryBuilder().setNameFormat("metric-registry-%d").build()); - scheduledExecutorService.scheduleAtFixedRate(this::update, metricConfig.timeIntervalMs(), - metricConfig.timeIntervalMs(), TimeUnit.MILLISECONDS); + scheduledExecutorService = + new ScheduledThreadPoolExecutor( + metricConfig.threadPoolSize(), + new ThreadFactoryBuilder().setNameFormat("metric-registry-%d").build()); + scheduledExecutorService.scheduleAtFixedRate( + this::update, + metricConfig.timeIntervalMs(), + metricConfig.timeIntervalMs(), + TimeUnit.MILLISECONDS); isRunning = true; LOG.info("Finished startup metric registry, metricConfig is {}.", metricConfig); } @@ -47,15 +50,18 @@ public class MetricRegistry { if (isRunning && scheduledExecutorService != null) { try { scheduledExecutorService.shutdownNow(); - if (!scheduledExecutorService.awaitTermination(metricConfig.shutdownWaitTimeMs(), - TimeUnit.MILLISECONDS)) { - LOG.warn("Metric registry did not shut down in {}ms time, so try to shut down again.", + if (!scheduledExecutorService.awaitTermination( + metricConfig.shutdownWaitTimeMs(), TimeUnit.MILLISECONDS)) { + LOG.warn( + "Metric registry did not shut down in {}ms time, so try to shut down again.", metricConfig.shutdownWaitTimeMs()); scheduledExecutorService.shutdownNow(); } } catch (InterruptedException e) { - LOG.warn("Interrupted when shutting down metric registry, so try to shut down again.", - e.getMessage(), e); + LOG.warn( + "Interrupted when shutting down metric registry, so try to shut down again.", + e.getMessage(), + e); scheduledExecutorService.shutdownNow(); } if (scheduledExecutorService.isShutdown()) { @@ -106,9 +112,10 @@ public class MetricRegistry { } private void update() { - registeredMetrics.forEach((id, metric) -> { - metric.record(); - }); + registeredMetrics.forEach( + (id, metric) -> { + metric.record(); + }); } private MetricType getMetricType(Metric metric) { @@ -131,5 +138,4 @@ public class MetricRegistry { private MetricId genMetricIdByMetric(Metric metric) { return new MetricId(getMetricType(metric), metric.name, metric.tags); } - -} \ No newline at end of file +} diff --git a/java/runtime/src/main/java/io/ray/runtime/metric/MetricType.java b/java/runtime/src/main/java/io/ray/runtime/metric/MetricType.java index 6455fa624..bf2a3e9b1 100644 --- a/java/runtime/src/main/java/io/ray/runtime/metric/MetricType.java +++ b/java/runtime/src/main/java/io/ray/runtime/metric/MetricType.java @@ -1,10 +1,7 @@ package io.ray.runtime.metric; -/** - * Types of the metric. - */ +/** Types of the metric. */ public enum MetricType { - COUNT, GAUGE, @@ -12,5 +9,4 @@ public enum MetricType { SUM, HISTOGRAM - -} \ No newline at end of file +} diff --git a/java/runtime/src/main/java/io/ray/runtime/metric/Metrics.java b/java/runtime/src/main/java/io/ray/runtime/metric/Metrics.java index eda3b3760..85939ed79 100644 --- a/java/runtime/src/main/java/io/ray/runtime/metric/Metrics.java +++ b/java/runtime/src/main/java/io/ray/runtime/metric/Metrics.java @@ -5,9 +5,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -/** - * The entry of metrics for easy use. - */ +/** The entry of metrics for easy use. */ public final class Metrics { private static MetricRegistry metricRegistry; @@ -113,7 +111,7 @@ public final class Metrics { /** * Creates a metric by sub-class. * - * @return a metric + *

Returns a metric */ protected abstract M create(); @@ -126,10 +124,11 @@ public final class Metrics { private static Map generateTagKeysMap(Map tags) { Map tagKeys = new HashMap<>(tags.size() * 2); - tags.forEach((key, value) -> { - TagKey tagKey = new TagKey(key); - tagKeys.put(tagKey, value); - }); + tags.forEach( + (key, value) -> { + TagKey tagKey = new TagKey(key); + tagKeys.put(tagKey, value); + }); return tagKeys; } @@ -144,5 +143,4 @@ public final class Metrics { } } } - -} \ No newline at end of file +} diff --git a/java/runtime/src/main/java/io/ray/runtime/metric/NativeMetric.java b/java/runtime/src/main/java/io/ray/runtime/metric/NativeMetric.java index 8f6e077cc..e1f56e13e 100644 --- a/java/runtime/src/main/java/io/ray/runtime/metric/NativeMetric.java +++ b/java/runtime/src/main/java/io/ray/runtime/metric/NativeMetric.java @@ -8,22 +8,20 @@ import java.util.List; class NativeMetric { public static native void registerTagkeyNative(String tagKey); - public static native long registerCountNative(String name, String description, - String unit, List tagKeys); + public static native long registerCountNative( + String name, String description, String unit, List tagKeys); - public static native long registerGaugeNative(String name, String description, - String unit, List tagKeys); + public static native long registerGaugeNative( + String name, String description, String unit, List tagKeys); - public static native long registerHistogramNative(String name, String description, - String unit, double[] boundaries, - List tagKeys); + public static native long registerHistogramNative( + String name, String description, String unit, double[] boundaries, List tagKeys); - public static native long registerSumNative(String name, String description, - String unit, List tagKeys); + public static native long registerSumNative( + String name, String description, String unit, List tagKeys); - public static native void recordNative(long metricNativePointer, double value, - List tagKeys, List tagValues); + public static native void recordNative( + long metricNativePointer, double value, List tagKeys, List tagValues); public static native void unregisterMetricNative(long gaugePtr); - } diff --git a/java/runtime/src/main/java/io/ray/runtime/metric/Sum.java b/java/runtime/src/main/java/io/ray/runtime/metric/Sum.java index 6ad8b833f..95773518d 100644 --- a/java/runtime/src/main/java/io/ray/runtime/metric/Sum.java +++ b/java/runtime/src/main/java/io/ray/runtime/metric/Sum.java @@ -1,14 +1,13 @@ package io.ray.runtime.metric; import com.google.common.base.Preconditions; - import java.util.Map; import java.util.concurrent.atomic.DoubleAdder; import java.util.stream.Collectors; /** - * Sum measurement is mapped to sum object in stats. - * Property sum is used for storing transient sum for registry aggregation. + * Sum measurement is mapped to sum object in stats. Property sum is used for storing transient sum + * for registry aggregation. */ public class Sum extends Metric { @@ -16,8 +15,12 @@ public class Sum extends Metric { public Sum(String name, String description, String unit, Map tags) { super(name, tags); - metricNativePointer = NativeMetric.registerSumNative(name, description, unit, - tags.keySet().stream().map(TagKey::getTagKey).collect(Collectors.toList())); + metricNativePointer = + NativeMetric.registerSumNative( + name, + description, + unit, + tags.keySet().stream().map(TagKey::getTagKey).collect(Collectors.toList())); Preconditions.checkState(metricNativePointer != 0, "Count native pointer must not be 0."); this.sum = new DoubleAdder(); } diff --git a/java/runtime/src/main/java/io/ray/runtime/metric/TagKey.java b/java/runtime/src/main/java/io/ray/runtime/metric/TagKey.java index 494c505c0..21fba3e64 100644 --- a/java/runtime/src/main/java/io/ray/runtime/metric/TagKey.java +++ b/java/runtime/src/main/java/io/ray/runtime/metric/TagKey.java @@ -2,9 +2,7 @@ package io.ray.runtime.metric; import java.util.Objects; -/** - * Tagkey is mapping java object to stats tagkey object. - */ +/** Tagkey is mapping java object to stats tagkey object. */ public class TagKey { private String tagKey; @@ -37,8 +35,6 @@ public class TagKey { @Override public String toString() { - return "TagKey{" + - ", tagKey='" + tagKey + '\'' + - '}'; + return "TagKey{" + ", tagKey='" + tagKey + '\'' + '}'; } -} \ No newline at end of file +} diff --git a/java/runtime/src/main/java/io/ray/runtime/object/LocalModeObjectStore.java b/java/runtime/src/main/java/io/ray/runtime/object/LocalModeObjectStore.java index fed5459b4..e1bfc64fa 100644 --- a/java/runtime/src/main/java/io/ray/runtime/object/LocalModeObjectStore.java +++ b/java/runtime/src/main/java/io/ray/runtime/object/LocalModeObjectStore.java @@ -14,9 +14,7 @@ import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Object store methods for local mode. - */ +/** Object store methods for local mode. */ public class LocalModeObjectStore extends ObjectStore { private static final Logger LOGGER = LoggerFactory.getLogger(LocalModeObjectStore.class); @@ -100,12 +98,10 @@ public class LocalModeObjectStore extends ObjectStore { } @Override - public void addLocalReference(UniqueId workerId, ObjectId objectId) { - } + public void addLocalReference(UniqueId workerId, ObjectId objectId) {} @Override - public void removeLocalReference(UniqueId workerId, ObjectId objectId) { - } + public void removeLocalReference(UniqueId workerId, ObjectId objectId) {} @Override public Address getOwnerAddress(ObjectId id) { @@ -119,6 +115,5 @@ public class LocalModeObjectStore extends ObjectStore { @Override public void registerOwnershipInfoAndResolveFuture( - ObjectId objectId, ObjectId outerObjectId, byte[] ownerAddress) { - } + ObjectId objectId, ObjectId outerObjectId, byte[] ownerAddress) {} } diff --git a/java/runtime/src/main/java/io/ray/runtime/object/NativeObjectStore.java b/java/runtime/src/main/java/io/ray/runtime/object/NativeObjectStore.java index 38a6a16c9..24dd5b8a2 100644 --- a/java/runtime/src/main/java/io/ray/runtime/object/NativeObjectStore.java +++ b/java/runtime/src/main/java/io/ray/runtime/object/NativeObjectStore.java @@ -76,8 +76,8 @@ public class NativeObjectStore extends ObjectStore { } @Override - public void registerOwnershipInfoAndResolveFuture(ObjectId objectId, ObjectId outerObjectId, - byte[] ownerAddress) { + public void registerOwnershipInfoAndResolveFuture( + ObjectId objectId, ObjectId outerObjectId, byte[] ownerAddress) { byte[] outer = null; if (outerObjectId != null) { outer = outerObjectId.getBytes(); @@ -87,8 +87,7 @@ public class NativeObjectStore extends ObjectStore { public Map getAllReferenceCounts() { Map referenceCounts = new HashMap<>(); - for (Map.Entry entry : - nativeGetAllReferenceCounts().entrySet()) { + for (Map.Entry entry : nativeGetAllReferenceCounts().entrySet()) { referenceCounts.put(new ObjectId(entry.getKey()), entry.getValue()); } return referenceCounts; @@ -113,8 +112,8 @@ public class NativeObjectStore extends ObjectStore { private static native List nativeGet(List ids, long timeoutMs); - private static native List nativeWait(List objectIds, int numObjects, - long timeoutMs); + private static native List nativeWait( + List objectIds, int numObjects, long timeoutMs); private static native void nativeDelete(List objectIds, boolean localOnly); @@ -128,6 +127,6 @@ public class NativeObjectStore extends ObjectStore { private static native byte[] nativePromoteAndGetOwnershipInfo(byte[] objectId); - private static native void nativeRegisterOwnershipInfoAndResolveFuture(byte[] objectId, - byte[] outerObjectId, byte[] ownerAddress); + private static native void nativeRegisterOwnershipInfoAndResolveFuture( + byte[] objectId, byte[] outerObjectId, byte[] ownerAddress); } diff --git a/java/runtime/src/main/java/io/ray/runtime/object/NativeRayObject.java b/java/runtime/src/main/java/io/ray/runtime/object/NativeRayObject.java index d3b0ae823..d3ec7fc3e 100644 --- a/java/runtime/src/main/java/io/ray/runtime/object/NativeRayObject.java +++ b/java/runtime/src/main/java/io/ray/runtime/object/NativeRayObject.java @@ -7,9 +7,7 @@ import java.util.Collections; import java.util.List; import java.util.stream.Collectors; -/** - * Binary representation of a ray object. See `RayObject` class in C++ for details. - */ +/** Binary representation of a ray object. See `RayObject` class in C++ for details. */ public class NativeRayObject { public byte[] data; @@ -43,4 +41,3 @@ public class NativeRayObject { return ": " + bufferLength(data) + ", : " + bufferLength(metadata); } } - diff --git a/java/runtime/src/main/java/io/ray/runtime/object/ObjectRefImpl.java b/java/runtime/src/main/java/io/ray/runtime/object/ObjectRefImpl.java index e15804314..cb9b35bec 100644 --- a/java/runtime/src/main/java/io/ray/runtime/object/ObjectRefImpl.java +++ b/java/runtime/src/main/java/io/ray/runtime/object/ObjectRefImpl.java @@ -17,9 +17,7 @@ import java.lang.ref.Reference; import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; -/** - * Implementation of {@link ObjectRef}. - */ +/** Implementation of {@link ObjectRef}. */ public final class ObjectRefImpl implements ObjectRef, Externalizable { private static final FinalizableReferenceQueue REFERENCE_QUEUE = new FinalizableReferenceQueue(); @@ -40,8 +38,7 @@ public final class ObjectRefImpl implements ObjectRef, Externalizable { addLocalReference(); } - public ObjectRefImpl() { - } + public ObjectRefImpl() {} @Override public synchronized T get() { @@ -81,8 +78,10 @@ public final class ObjectRefImpl implements ObjectRef, Externalizable { in.readFully(ownerAddress); addLocalReference(); RayRuntimeInternal runtime = (RayRuntimeInternal) Ray.internal(); - runtime.getObjectStore().registerOwnershipInfoAndResolveFuture( - this.id, ObjectSerializer.getOuterObjectId(), ownerAddress); + runtime + .getObjectStore() + .registerOwnershipInfoAndResolveFuture( + this.id, ObjectSerializer.getOuterObjectId(), ownerAddress); } private void addLocalReference() { @@ -93,8 +92,8 @@ public final class ObjectRefImpl implements ObjectRef, Externalizable { new ObjectRefImplReference(this); } - private static final class ObjectRefImplReference extends - FinalizableWeakReference> { + private static final class ObjectRefImplReference + extends FinalizableWeakReference> { private final UniqueId workerId; private final ObjectId objectId; @@ -116,7 +115,8 @@ public final class ObjectRefImpl implements ObjectRef, Externalizable { REFERENCES.remove(this); // It's possible that GC is executed after the runtime is shutdown. if (Ray.isInitialized()) { - ((RayRuntimeInternal) (Ray.internal())).getObjectStore() + ((RayRuntimeInternal) (Ray.internal())) + .getObjectStore() .removeLocalReference(workerId, objectId); } } diff --git a/java/runtime/src/main/java/io/ray/runtime/object/ObjectSerializer.java b/java/runtime/src/main/java/io/ray/runtime/object/ObjectSerializer.java index e72f393d6..76576b969 100644 --- a/java/runtime/src/main/java/io/ray/runtime/object/ObjectSerializer.java +++ b/java/runtime/src/main/java/io/ray/runtime/object/ObjectSerializer.java @@ -25,14 +25,14 @@ import org.apache.commons.lang3.tuple.Pair; */ public class ObjectSerializer { - private static final byte[] WORKER_EXCEPTION_META = String - .valueOf(ErrorType.WORKER_DIED.getNumber()).getBytes(); - private static final byte[] ACTOR_EXCEPTION_META = String - .valueOf(ErrorType.ACTOR_DIED.getNumber()).getBytes(); - private static final byte[] UNRECONSTRUCTABLE_EXCEPTION_META = String - .valueOf(ErrorType.OBJECT_UNRECONSTRUCTABLE.getNumber()).getBytes(); - private static final byte[] TASK_EXECUTION_EXCEPTION_META = String - .valueOf(ErrorType.TASK_EXECUTION_EXCEPTION.getNumber()).getBytes(); + private static final byte[] WORKER_EXCEPTION_META = + String.valueOf(ErrorType.WORKER_DIED.getNumber()).getBytes(); + private static final byte[] ACTOR_EXCEPTION_META = + String.valueOf(ErrorType.ACTOR_DIED.getNumber()).getBytes(); + private static final byte[] UNRECONSTRUCTABLE_EXCEPTION_META = + String.valueOf(ErrorType.OBJECT_UNRECONSTRUCTABLE.getNumber()).getBytes(); + private static final byte[] TASK_EXECUTION_EXCEPTION_META = + String.valueOf(ErrorType.TASK_EXECUTION_EXCEPTION.getNumber()).getBytes(); public static final byte[] OBJECT_METADATA_TYPE_CROSS_LANGUAGE = "XLANG".getBytes(); public static final byte[] OBJECT_METADATA_TYPE_JAVA = "JAVA".getBytes(); @@ -55,11 +55,10 @@ public class ObjectSerializer { * Deserialize an object from an {@link NativeRayObject} instance. * * @param nativeRayObject The object to deserialize. - * @param objectId The associated object ID of the object. - * @return The deserialized object. + * @param objectId The associated object ID of the object. Returns The deserialized object. */ - public static Object deserialize(NativeRayObject nativeRayObject, ObjectId objectId, - Class objectType) { + public static Object deserialize( + NativeRayObject nativeRayObject, ObjectId objectId, Class objectType) { byte[] meta = nativeRayObject.metadata; byte[] data = nativeRayObject.data; @@ -70,8 +69,8 @@ public class ObjectSerializer { return ByteBuffer.wrap(data); } return data; - } else if (Bytes.indexOf(meta, OBJECT_METADATA_TYPE_CROSS_LANGUAGE) == 0 || - Bytes.indexOf(meta, OBJECT_METADATA_TYPE_JAVA) == 0) { + } else if (Bytes.indexOf(meta, OBJECT_METADATA_TYPE_CROSS_LANGUAGE) == 0 + || Bytes.indexOf(meta, OBJECT_METADATA_TYPE_JAVA) == 0) { return Serializer.decode(data, objectType); } else if (Bytes.indexOf(meta, WORKER_EXCEPTION_META) == 0) { return new RayWorkerException(); @@ -92,15 +91,14 @@ public class ObjectSerializer { return RayTaskException.fromBytes(serialized); } catch (InvalidProtocolBufferException e) { throw new IllegalArgumentException( - "Can't deserialize RayTaskException object: " + objectId - .toString()); + "Can't deserialize RayTaskException object: " + objectId.toString()); } } else if (Bytes.indexOf(meta, OBJECT_METADATA_TYPE_ACTOR_HANDLE) == 0) { byte[] serialized = Serializer.decode(data, byte[].class); return NativeActorHandle.fromBytes(serialized); } else if (Bytes.indexOf(meta, OBJECT_METADATA_TYPE_PYTHON) == 0) { - throw new IllegalArgumentException("Can't deserialize Python object: " + objectId - .toString()); + throw new IllegalArgumentException( + "Can't deserialize Python object: " + objectId.toString()); } throw new IllegalArgumentException("Unrecognized metadata " + Arrays.toString(meta)); } else { @@ -112,8 +110,7 @@ public class ObjectSerializer { /** * Serialize an Java object to an {@link NativeRayObject} instance. * - * @param object The object to serialize. - * @return The serialized object. + * @param object The object to serialize. Returns The serialized object. */ public static NativeRayObject serialize(Object object) { if (object instanceof NativeRayObject) { @@ -142,7 +139,7 @@ public class ObjectSerializer { // any other type should be the MessagePack serialized bytes. return new NativeRayObject(serializedBytes, TASK_EXECUTION_EXCEPTION_META); } else if (object instanceof NativeActorHandle) { - NativeActorHandle actorHandle = (NativeActorHandle)object; + NativeActorHandle actorHandle = (NativeActorHandle) object; byte[] serializedBytes = Serializer.encode(actorHandle.toBytes()).getLeft(); // serializedBytes is MessagePack serialized bytes // Only OBJECT_METADATA_TYPE_RAW is raw bytes, @@ -151,10 +148,12 @@ public class ObjectSerializer { } else { try { Pair serialized = Serializer.encode(object); - NativeRayObject nativeRayObject = new NativeRayObject(serialized.getLeft(), - serialized.getRight() - ? OBJECT_METADATA_TYPE_CROSS_LANGUAGE - : OBJECT_METADATA_TYPE_JAVA); + NativeRayObject nativeRayObject = + new NativeRayObject( + serialized.getLeft(), + serialized.getRight() + ? OBJECT_METADATA_TYPE_CROSS_LANGUAGE + : OBJECT_METADATA_TYPE_JAVA); nativeRayObject.setContainedObjectIds(getAndClearContainedObjectIds()); return nativeRayObject; } catch (Exception e) { diff --git a/java/runtime/src/main/java/io/ray/runtime/object/ObjectStore.java b/java/runtime/src/main/java/io/ray/runtime/object/ObjectStore.java index bfec229f1..df524af11 100644 --- a/java/runtime/src/main/java/io/ray/runtime/object/ObjectStore.java +++ b/java/runtime/src/main/java/io/ray/runtime/object/ObjectStore.java @@ -14,9 +14,7 @@ import java.util.List; import java.util.Objects; import java.util.stream.Collectors; -/** - * A class that is used to put/get objects to/from the object store. - */ +/** A class that is used to put/get objects to/from the object store. */ public abstract class ObjectStore { private final WorkerContext workerContext; @@ -28,8 +26,7 @@ public abstract class ObjectStore { /** * Put a raw object into object store. * - * @param obj The ray object. - * @return Generated ID of the object. + * @param obj The ray object. Returns Generated ID of the object. */ public abstract ObjectId putRaw(NativeRayObject obj); @@ -44,8 +41,7 @@ public abstract class ObjectStore { /** * Serialize and put an object to the object store. * - * @param object The object to put. - * @return Id of the object. + * @param object The object to put. Returns Id of the object. */ public ObjectId put(Object object) { if (object instanceof NativeRayObject) { @@ -58,7 +54,7 @@ public abstract class ObjectStore { /** * Serialize and put an object to the object store, with the given object id. * - * This method is only used for testing. + *

This method is only used for testing. * * @param object The object to put. * @param objectId Object id. @@ -75,8 +71,8 @@ public abstract class ObjectStore { * Get a list of raw objects from the object store. * * @param objectIds IDs of the objects to get. - * @param timeoutMs Timeout in milliseconds, wait infinitely if it's negative. - * @return Result list of objects data. + * @param timeoutMs Timeout in milliseconds, wait infinitely if it's negative. Returns Result list + * of objects data. */ public abstract List getRaw(List objectIds, long timeoutMs); @@ -84,8 +80,7 @@ public abstract class ObjectStore { * Get a list of objects from the object store. * * @param ids List of the object ids. - * @param Type of these objects. - * @return A list of GetResult objects. + * @param Type of these objects. Returns A list of GetResult objects. */ @SuppressWarnings("unchecked") public List get(List ids, Class elementType) { @@ -99,8 +94,7 @@ public abstract class ObjectStore { if (dataAndMeta != null) { try { ObjectSerializer.setOuterObjectId(ids.get(i)); - object = ObjectSerializer - .deserialize(dataAndMeta, ids.get(i), elementType); + object = ObjectSerializer.deserialize(dataAndMeta, ids.get(i), elementType); } finally { ObjectSerializer.resetOuterObjectId(); } @@ -124,8 +118,8 @@ public abstract class ObjectStore { * * @param objectIds IDs of the objects to wait for. * @param numObjects Number of objects that should appear. - * @param timeoutMs Timeout in milliseconds, wait infinitely if it's negative. - * @return A bitset that indicates each object has appeared or not. + * @param timeoutMs Timeout in milliseconds, wait infinitely if it's negative. Returns A bitset + * that indicates each object has appeared or not. */ public abstract List wait(List objectIds, int numObjects, long timeoutMs); @@ -135,8 +129,8 @@ public abstract class ObjectStore { * * @param waitList A list of object references to wait for. * @param numReturns The number of objects that should be returned. - * @param timeoutMs The maximum time in milliseconds to wait before returning. - * @return Two lists, one containing locally available objects, one containing the rest. + * @param timeoutMs The maximum time in milliseconds to wait before returning. Returns Two lists, + * one containing locally available objects, one containing the rest. */ public WaitResult wait(List> waitList, int numReturns, int timeoutMs) { Preconditions.checkNotNull(waitList); @@ -144,8 +138,8 @@ public abstract class ObjectStore { return new WaitResult<>(Collections.emptyList(), Collections.emptyList()); } - List ids = waitList.stream().map(ref -> ((ObjectRefImpl) ref).getId()) - .collect(Collectors.toList()); + List ids = + waitList.stream().map(ref -> ((ObjectRefImpl) ref).getId()).collect(Collectors.toList()); List ready = wait(ids, numReturns, timeoutMs); List> readyList = new ArrayList<>(); @@ -191,8 +185,7 @@ public abstract class ObjectStore { /** * Promote the given object to the underlying object store, and get the ownership info. * - * @param objectId The ID of the object to promote - * @return the serialized ownership address + * @param objectId The ID of the object to promote Returns the serialized ownership address */ public abstract byte[] promoteAndGetOwnershipInfo(ObjectId objectId); @@ -204,10 +197,10 @@ public abstract class ObjectStore { * * @param objectId The object ID to deserialize. * @param outerObjectId The object ID that contained objectId, if any. This may be nil if the - * object ID was inlined directly in a task spec or if it was passed - * out-of-band by the application (deserialized from a byte string). + * object ID was inlined directly in a task spec or if it was passed out-of-band by the + * application (deserialized from a byte string). * @param ownerAddress The address of the object's owner. */ - public abstract void registerOwnershipInfoAndResolveFuture(ObjectId objectId, - ObjectId outerObjectId, byte[] ownerAddress); + public abstract void registerOwnershipInfoAndResolveFuture( + ObjectId objectId, ObjectId outerObjectId, byte[] ownerAddress); } diff --git a/java/runtime/src/main/java/io/ray/runtime/placementgroup/PlacementGroupImpl.java b/java/runtime/src/main/java/io/ray/runtime/placementgroup/PlacementGroupImpl.java index 96f499e60..b08f7c9f5 100644 --- a/java/runtime/src/main/java/io/ray/runtime/placementgroup/PlacementGroupImpl.java +++ b/java/runtime/src/main/java/io/ray/runtime/placementgroup/PlacementGroupImpl.java @@ -8,9 +8,7 @@ import io.ray.api.placementgroup.PlacementStrategy; import java.util.List; import java.util.Map; -/** - * The default implementation of `PlacementGroup` interface. - */ +/** The default implementation of `PlacementGroup` interface. */ public class PlacementGroupImpl implements PlacementGroup { private final PlacementGroupId id; @@ -19,10 +17,12 @@ public class PlacementGroupImpl implements PlacementGroup { private final PlacementStrategy strategy; private final PlacementGroupState state; - private PlacementGroupImpl(PlacementGroupId id, String name, - List> bundles, - PlacementStrategy strategy, - PlacementGroupState state) { + private PlacementGroupImpl( + PlacementGroupId id, + String name, + List> bundles, + PlacementStrategy strategy, + PlacementGroupState state) { this.id = id; this.name = name; this.bundles = bundles; @@ -52,16 +52,15 @@ public class PlacementGroupImpl implements PlacementGroup { /** * Wait for the placement group to be ready within the specified time. - * @param timeoutSeconds Timeout in seconds. - * @return True if the placement group is created. False otherwise. + * + * @param timeoutSeconds Timeout in seconds. Returns True if the placement group is created. False + * otherwise. */ public boolean wait(int timeoutSeconds) { return Ray.internal().waitPlacementGroupReady(id, timeoutSeconds); } - /** - * A help class for create the placement group. - */ + /** A help class for create the placement group. */ public static class Builder { private PlacementGroupId id; private String name; @@ -71,8 +70,8 @@ public class PlacementGroupImpl implements PlacementGroup { /** * Set the Id of the placement group. - * @param id Id of the placement group. - * @return self. + * + * @param id Id of the placement group. Returns self. */ public Builder setId(PlacementGroupId id) { this.id = id; @@ -81,8 +80,8 @@ public class PlacementGroupImpl implements PlacementGroup { /** * Set the name of the placement group. - * @param name Name of the placement group. - * @return self. + * + * @param name Name of the placement group. Returns self. */ public Builder setName(String name) { this.name = name; @@ -91,8 +90,8 @@ public class PlacementGroupImpl implements PlacementGroup { /** * Set the bundles of the placement group. - * @param bundles the bundles of the placement group. - * @return self. + * + * @param bundles the bundles of the placement group. Returns self. */ public Builder setBundles(List> bundles) { this.bundles = bundles; @@ -101,8 +100,8 @@ public class PlacementGroupImpl implements PlacementGroup { /** * Set the placement strategy of the placement group. - * @param strategy the placement strategy of the placement group. - * @return self. + * + * @param strategy the placement strategy of the placement group. Returns self. */ public Builder setStrategy(PlacementStrategy strategy) { this.strategy = strategy; @@ -111,8 +110,8 @@ public class PlacementGroupImpl implements PlacementGroup { /** * Set the placement state of the placement group. - * @param state the state of the placement group. - * @return self. + * + * @param state the state of the placement group. Returns self. */ public Builder setState(PlacementGroupState state) { this.state = state; @@ -123,5 +122,4 @@ public class PlacementGroupImpl implements PlacementGroup { return new PlacementGroupImpl(id, name, bundles, strategy, state); } } - } diff --git a/java/runtime/src/main/java/io/ray/runtime/placementgroup/PlacementGroupUtils.java b/java/runtime/src/main/java/io/ray/runtime/placementgroup/PlacementGroupUtils.java index 0211d6a35..75305ef1f 100644 --- a/java/runtime/src/main/java/io/ray/runtime/placementgroup/PlacementGroupUtils.java +++ b/java/runtime/src/main/java/io/ray/runtime/placementgroup/PlacementGroupUtils.java @@ -12,9 +12,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -/** - * Utils for placement group. - */ +/** Utils for placement group. */ public class PlacementGroupUtils { private static List> covertToUserSpecifiedBundles(List bundles) { @@ -62,45 +60,50 @@ public class PlacementGroupUtils { /** * Generate a PlacementGroupImpl from placementGroupTableData protobuf data. - * @param placementGroupTableData protobuf data. - * @return placement group info {@link PlacementGroupImpl} + * + * @param placementGroupTableData protobuf data. Returns placement group info {@link + * PlacementGroupImpl} */ private static PlacementGroupImpl generatePlacementGroupFromPbData( PlacementGroupTableData placementGroupTableData) { - PlacementGroupState state = covertToUserSpecifiedState( - placementGroupTableData.getState()); - PlacementStrategy strategy = covertToUserSpecifiedStrategy( - placementGroupTableData.getStrategy()); + PlacementGroupState state = covertToUserSpecifiedState(placementGroupTableData.getState()); + PlacementStrategy strategy = + covertToUserSpecifiedStrategy(placementGroupTableData.getStrategy()); - List> bundles = covertToUserSpecifiedBundles( - placementGroupTableData.getBundlesList()); + List> bundles = + covertToUserSpecifiedBundles(placementGroupTableData.getBundlesList()); - PlacementGroupId placementGroupId = PlacementGroupId.fromByteBuffer( - placementGroupTableData.getPlacementGroupId().asReadOnlyByteBuffer()); + PlacementGroupId placementGroupId = + PlacementGroupId.fromByteBuffer( + placementGroupTableData.getPlacementGroupId().asReadOnlyByteBuffer()); return new PlacementGroupImpl.Builder() - .setId(placementGroupId).setName(placementGroupTableData.getName()) - .setState(state).setStrategy(strategy).setBundles(bundles) - .build(); + .setId(placementGroupId) + .setName(placementGroupTableData.getName()) + .setState(state) + .setStrategy(strategy) + .setBundles(bundles) + .build(); } /** * Generate a PlacementGroupImpl from byte array. - * @param placementGroupByteArray bytes array from native method. - * @return placement group info {@link PlacementGroupImpl} + * + * @param placementGroupByteArray bytes array from native method. Returns placement group info + * {@link PlacementGroupImpl} */ public static PlacementGroupImpl generatePlacementGroupFromByteArray( byte[] placementGroupByteArray) { - Preconditions.checkNotNull(placementGroupByteArray, - "Can't generate a placement group from empty byte array."); + Preconditions.checkNotNull( + placementGroupByteArray, "Can't generate a placement group from empty byte array."); PlacementGroupTableData placementGroupTableData; try { placementGroupTableData = PlacementGroupTableData.parseFrom(placementGroupByteArray); } catch (InvalidProtocolBufferException e) { throw new RuntimeException( - "Received invalid placement group table protobuf data from GCS.", e); + "Received invalid placement group table protobuf data from GCS.", e); } return generatePlacementGroupFromPbData(placementGroupTableData); diff --git a/java/runtime/src/main/java/io/ray/runtime/runner/RunManager.java b/java/runtime/src/main/java/io/ray/runtime/runner/RunManager.java index 4bd49deb4..2307b0489 100644 --- a/java/runtime/src/main/java/io/ray/runtime/runner/RunManager.java +++ b/java/runtime/src/main/java/io/ray/runtime/runner/RunManager.java @@ -16,18 +16,14 @@ import org.apache.commons.io.IOUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Ray service management on one box. - */ +/** Ray service management on one box. */ public class RunManager { private static final Logger LOGGER = LoggerFactory.getLogger(RunManager.class); private static final Pattern pattern = Pattern.compile("--address='([^']+)'"); - /** - * Start the head node. - */ + /** Start the head node. */ public static void startRayHead(RayConfig rayConfig) { LOGGER.debug("Starting ray runtime @ {}.", rayConfig.nodeIp); List command = new ArrayList<>(); @@ -54,9 +50,7 @@ public class RunManager { LOGGER.info("Ray runtime started @ {}.", rayConfig.nodeIp); } - /** - * Stop ray. - */ + /** Stop ray. */ public static void stopRay() { List command = new ArrayList<>(); command.add("ray"); @@ -73,10 +67,12 @@ public class RunManager { public static void getAddressInfoAndFillConfig(RayConfig rayConfig) { // NOTE(kfstorm): This method depends on an internal Python API of ray to get the // address info of the local node. - String script = String.format("import ray;" - + " print(ray._private.services.get_address_info_from_redis(" - + "'%s', '%s', redis_password='%s'))", - rayConfig.getRedisAddress(), rayConfig.nodeIp, rayConfig.redisPassword); + String script = + String.format( + "import ray;" + + " print(ray._private.services.get_address_info_from_redis(" + + "'%s', '%s', redis_password='%s'))", + rayConfig.getRedisAddress(), rayConfig.nodeIp, rayConfig.redisPassword); List command = Arrays.asList("python", "-c", script); String output = null; @@ -110,9 +106,14 @@ public class RunManager { String output = IOUtils.toString(p.getInputStream(), Charset.defaultCharset()); p.waitFor(); if (p.exitValue() != 0) { - String sb = "The exit value of the process is " + p.exitValue() - + ". Command: " + Joiner.on(" ").join(command) + "\n" - + "output:\n" + output; + String sb = + "The exit value of the process is " + + p.exitValue() + + ". Command: " + + Joiner.on(" ").join(command) + + "\n" + + "output:\n" + + output; throw new RuntimeException(sb); } return output; diff --git a/java/runtime/src/main/java/io/ray/runtime/runner/worker/DefaultWorker.java b/java/runtime/src/main/java/io/ray/runtime/runner/worker/DefaultWorker.java index 5f79172b4..635201147 100644 --- a/java/runtime/src/main/java/io/ray/runtime/runner/worker/DefaultWorker.java +++ b/java/runtime/src/main/java/io/ray/runtime/runner/worker/DefaultWorker.java @@ -3,9 +3,7 @@ package io.ray.runtime.runner.worker; import io.ray.api.Ray; import io.ray.runtime.RayRuntimeInternal; -/** - * Default implementation of the worker process. - */ +/** Default implementation of the worker process. */ public class DefaultWorker { public static void main(String[] args) { @@ -16,4 +14,4 @@ public class DefaultWorker { Ray.init(); ((RayRuntimeInternal) Ray.internal()).run(); } -} \ No newline at end of file +} diff --git a/java/runtime/src/main/java/io/ray/runtime/serializer/FstSerializer.java b/java/runtime/src/main/java/io/ray/runtime/serializer/FstSerializer.java index b19b25367..e3f255d04 100644 --- a/java/runtime/src/main/java/io/ray/runtime/serializer/FstSerializer.java +++ b/java/runtime/src/main/java/io/ray/runtime/serializer/FstSerializer.java @@ -4,17 +4,17 @@ import io.ray.runtime.actor.NativeActorHandle; import io.ray.runtime.actor.NativeActorHandleSerializer; import org.nustaq.serialization.FSTConfiguration; -/** - * Java object serialization TODO: use others (e.g. Arrow) for higher performance - */ +/** Java object serialization TODO: use others (e.g. Arrow) for higher performance */ public class FstSerializer { - private static final ThreadLocal conf = ThreadLocal.withInitial(() -> { - FSTConfiguration conf = FSTConfiguration.createDefaultConfiguration(); - conf.registerSerializer(NativeActorHandle.class, new NativeActorHandleSerializer(), true); - return conf; - }); - + private static final ThreadLocal conf = + ThreadLocal.withInitial( + () -> { + FSTConfiguration conf = FSTConfiguration.createDefaultConfiguration(); + conf.registerSerializer( + NativeActorHandle.class, new NativeActorHandleSerializer(), true); + return conf; + }); public static byte[] encode(Object obj) { FSTConfiguration current = conf.get(); @@ -22,7 +22,6 @@ public class FstSerializer { return current.asByteArray(obj); } - @SuppressWarnings("unchecked") public static T decode(byte[] bs) { FSTConfiguration current = conf.get(); diff --git a/java/runtime/src/main/java/io/ray/runtime/serializer/MessagePackSerializer.java b/java/runtime/src/main/java/io/ray/runtime/serializer/MessagePackSerializer.java index 41204616a..9bc9f6a2f 100644 --- a/java/runtime/src/main/java/io/ray/runtime/serializer/MessagePackSerializer.java +++ b/java/runtime/src/main/java/io/ray/runtime/serializer/MessagePackSerializer.java @@ -44,38 +44,40 @@ public class MessagePackSerializer { NULL_PACKER = (object, packer, javaSerializer) -> packer.packNil(); // Array packer. - ARRAY_PACKER = ((object, packer, javaSerializer) -> { - int length = Array.getLength(object); - packer.packArrayHeader(length); - for (int i = 0; i < length; ++i) { - pack(Array.get(object, i), packer, javaSerializer); - } - }); + ARRAY_PACKER = + ((object, packer, javaSerializer) -> { + int length = Array.getLength(object); + packer.packArrayHeader(length); + for (int i = 0; i < length; ++i) { + pack(Array.get(object, i), packer, javaSerializer); + } + }); // Extension packer. - EXTENSION_PACKER = ((object, packer, javaSerializer) -> { - javaSerializer.serialize(object, packer); - }); + EXTENSION_PACKER = + ((object, packer, javaSerializer) -> { + javaSerializer.serialize(object, packer); + }); - packers.put(Boolean.class, - ((object, packer, javaSerializer) -> packer.packBoolean((Boolean) object))); - packers.put(Byte.class, - ((object, packer, javaSerializer) -> packer.packByte((Byte) object))); - packers.put(Short.class, - ((object, packer, javaSerializer) -> packer.packShort((Short) object))); - packers.put(Integer.class, - ((object, packer, javaSerializer) -> packer.packInt((Integer) object))); - packers.put(Long.class, - ((object, packer, javaSerializer) -> packer.packLong((Long) object))); - packers.put(BigInteger.class, + packers.put( + Boolean.class, ((object, packer, javaSerializer) -> packer.packBoolean((Boolean) object))); + packers.put(Byte.class, ((object, packer, javaSerializer) -> packer.packByte((Byte) object))); + packers.put( + Short.class, ((object, packer, javaSerializer) -> packer.packShort((Short) object))); + packers.put( + Integer.class, ((object, packer, javaSerializer) -> packer.packInt((Integer) object))); + packers.put(Long.class, ((object, packer, javaSerializer) -> packer.packLong((Long) object))); + packers.put( + BigInteger.class, ((object, packer, javaSerializer) -> packer.packBigInteger((BigInteger) object))); - packers.put(Float.class, - ((object, packer, javaSerializer) -> packer.packFloat((Float) object))); - packers.put(Double.class, - ((object, packer, javaSerializer) -> packer.packDouble((Double) object))); - packers.put(String.class, - ((object, packer, javaSerializer) -> packer.packString((String) object))); - packers.put(byte[].class, + packers.put( + Float.class, ((object, packer, javaSerializer) -> packer.packFloat((Float) object))); + packers.put( + Double.class, ((object, packer, javaSerializer) -> packer.packDouble((Double) object))); + packers.put( + String.class, ((object, packer, javaSerializer) -> packer.packString((String) object))); + packers.put( + byte[].class, ((object, packer, javaSerializer) -> { byte[] bytes = (byte[]) object; packer.packBinaryHeader(bytes.length); @@ -97,68 +99,89 @@ public class MessagePackSerializer { // Null unpacker. unpackers.put(ValueType.NIL, (value, targetClass, javaDeserializer) -> null); // Boolean unpacker. - unpackers.put(ValueType.BOOLEAN, (value, targetClass, javaDeserializer) -> { - Preconditions.checkArgument(checkTypeCompatible(booleanClasses, targetClass), - "Boolean can't be deserialized as {}.", targetClass); - return value.asBooleanValue().getBoolean(); - }); + unpackers.put( + ValueType.BOOLEAN, + (value, targetClass, javaDeserializer) -> { + Preconditions.checkArgument( + checkTypeCompatible(booleanClasses, targetClass), + "Boolean can't be deserialized as {}.", + targetClass); + return value.asBooleanValue().getBoolean(); + }); // Integer unpacker. - unpackers.put(ValueType.INTEGER, ((value, targetClass, javaDeserializer) -> { - IntegerValue iv = value.asIntegerValue(); - if (iv.isInByteRange() && checkTypeCompatible(byteClasses, targetClass)) { - return iv.asByte(); - } else if (iv.isInShortRange() && checkTypeCompatible(shortClasses, targetClass)) { - return iv.asShort(); - } else if (iv.isInIntRange() && checkTypeCompatible(intClasses, targetClass)) { - return iv.asInt(); - } else if (iv.isInLongRange() && checkTypeCompatible(longClasses, targetClass)) { - return iv.asLong(); - } else if (checkTypeCompatible(bigIntClasses, targetClass)) { - return iv.asBigInteger(); - } - throw new IllegalArgumentException("Integer can't be deserialized as " + targetClass + "."); - })); + unpackers.put( + ValueType.INTEGER, + ((value, targetClass, javaDeserializer) -> { + IntegerValue iv = value.asIntegerValue(); + if (iv.isInByteRange() && checkTypeCompatible(byteClasses, targetClass)) { + return iv.asByte(); + } else if (iv.isInShortRange() && checkTypeCompatible(shortClasses, targetClass)) { + return iv.asShort(); + } else if (iv.isInIntRange() && checkTypeCompatible(intClasses, targetClass)) { + return iv.asInt(); + } else if (iv.isInLongRange() && checkTypeCompatible(longClasses, targetClass)) { + return iv.asLong(); + } else if (checkTypeCompatible(bigIntClasses, targetClass)) { + return iv.asBigInteger(); + } + throw new IllegalArgumentException( + "Integer can't be deserialized as " + targetClass + "."); + })); // Float unpacker. - unpackers.put(ValueType.FLOAT, ((value, targetClass, javaDeserializer) -> { - if (checkTypeCompatible(doubleClasses, targetClass)) { - return value.asFloatValue().toDouble(); - } else if (checkTypeCompatible(floatClasses, targetClass)) { - return value.asFloatValue().toFloat(); - } - throw new IllegalArgumentException("Float can't be deserialized as " + targetClass + "."); - })); + unpackers.put( + ValueType.FLOAT, + ((value, targetClass, javaDeserializer) -> { + if (checkTypeCompatible(doubleClasses, targetClass)) { + return value.asFloatValue().toDouble(); + } else if (checkTypeCompatible(floatClasses, targetClass)) { + return value.asFloatValue().toFloat(); + } + throw new IllegalArgumentException("Float can't be deserialized as " + targetClass + "."); + })); // String unpacker. - unpackers.put(ValueType.STRING, ((value, targetClass, javaDeserializer) -> { - Preconditions.checkArgument(checkTypeCompatible(stringClasses, targetClass), - "String can't be deserialized as {}.", targetClass); - return value.asStringValue().asString(); - })); + unpackers.put( + ValueType.STRING, + ((value, targetClass, javaDeserializer) -> { + Preconditions.checkArgument( + checkTypeCompatible(stringClasses, targetClass), + "String can't be deserialized as {}.", + targetClass); + return value.asStringValue().asString(); + })); // Binary unpacker. - unpackers.put(ValueType.BINARY, ((value, targetClass, javaDeserializer) -> { - Preconditions.checkArgument(checkTypeCompatible(binaryClasses, targetClass), - "Binary can't be deserialized as {}.", targetClass); - return value.asBinaryValue().asByteArray(); - })); + unpackers.put( + ValueType.BINARY, + ((value, targetClass, javaDeserializer) -> { + Preconditions.checkArgument( + checkTypeCompatible(binaryClasses, targetClass), + "Binary can't be deserialized as {}.", + targetClass); + return value.asBinaryValue().asByteArray(); + })); // Array unpacker. - unpackers.put(ValueType.ARRAY, ((value, targetClass, javaDeserializer) -> { - ArrayValue av = value.asArrayValue(); - Class componentType = - targetClass.isArray() ? targetClass.getComponentType() : Object.class; - Object array = Array.newInstance(componentType, av.size()); - for (int i = 0; i < av.size(); ++i) { - Array.set(array, i, unpack(av.get(i), componentType, javaDeserializer)); - } - return array; - })); + unpackers.put( + ValueType.ARRAY, + ((value, targetClass, javaDeserializer) -> { + ArrayValue av = value.asArrayValue(); + Class componentType = + targetClass.isArray() ? targetClass.getComponentType() : Object.class; + Object array = Array.newInstance(componentType, av.size()); + for (int i = 0; i < av.size(); ++i) { + Array.set(array, i, unpack(av.get(i), componentType, javaDeserializer)); + } + return array; + })); // Extension unpacker. - unpackers.put(ValueType.EXTENSION, ((value, targetClass, javaDeserializer) -> { - ExtensionValue ev = value.asExtensionValue(); - byte extType = ev.getType(); - if (extType == LANGUAGE_SPECIFIC_TYPE_EXTENSION_ID) { - return javaDeserializer.deserialize(ev); - } - throw new IllegalArgumentException("Unknown extension type id " + ev.getType() + "."); - })); + unpackers.put( + ValueType.EXTENSION, + ((value, targetClass, javaDeserializer) -> { + ExtensionValue ev = value.asExtensionValue(); + byte extType = ev.getType(); + if (extType == LANGUAGE_SPECIFIC_TYPE_EXTENSION_ID) { + return javaDeserializer.deserialize(ev); + } + throw new IllegalArgumentException("Unknown extension type id " + ev.getType() + "."); + })); } interface JavaSerializer { @@ -173,14 +196,13 @@ public class MessagePackSerializer { interface TypePacker { - void pack(Object object, MessagePacker packer, - JavaSerializer javaSerializer) throws IOException; + void pack(Object object, MessagePacker packer, JavaSerializer javaSerializer) + throws IOException; } interface TypeUnpacker { - Object unpack(Value value, Class targetClass, - JavaDeserializer javaDeserializer); + Object unpack(Value value, Class targetClass, JavaDeserializer javaDeserializer); } private static boolean checkTypeCompatible(List> expected, Class actual) { @@ -230,12 +252,15 @@ public class MessagePackSerializer { packer.writePayload(new byte[MESSAGE_PACK_OFFSET]); // Serialize input object by MessagePack. MutableBoolean isCrossLanguage = new MutableBoolean(true); - pack(obj, packer, ((object, packer1) -> { - byte[] payload = FstSerializer.encode(object); - packer1.packExtensionTypeHeader(LANGUAGE_SPECIFIC_TYPE_EXTENSION_ID, payload.length); - packer1.addPayload(payload); - isCrossLanguage.setFalse(); - })); + pack( + obj, + packer, + ((object, packer1) -> { + byte[] payload = FstSerializer.encode(object); + packer1.packExtensionTypeHeader(LANGUAGE_SPECIFIC_TYPE_EXTENSION_ID, payload.length); + packer1.addPayload(payload); + isCrossLanguage.setFalse(); + })); byte[] msgpackBytes = packer.toByteArray(); // Serialize MessagePack bytes length. MessageBufferPacker headerPacker = MessagePack.newDefaultBufferPacker(); @@ -252,7 +277,6 @@ public class MessagePackSerializer { } } - @SuppressWarnings("unchecked") public static T decode(byte[] bs, Class type) { try { @@ -263,14 +287,13 @@ public class MessagePackSerializer { // Check MessagePack bytes length is valid. Preconditions.checkState(MESSAGE_PACK_OFFSET + msgpackBytesLength <= bs.length); // Deserialize MessagePack bytes from MESSAGE_PACK_OFFSET. - MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(bs, MESSAGE_PACK_OFFSET, - (int) msgpackBytesLength); + MessageUnpacker unpacker = + MessagePack.newDefaultUnpacker(bs, MESSAGE_PACK_OFFSET, (int) msgpackBytesLength); Value v = unpacker.unpackValue(); if (type == null) { type = Object.class; } - return (T) unpack(v, type, - ((ExtensionValue ev) -> FstSerializer.decode(ev.getData()))); + return (T) unpack(v, type, ((ExtensionValue ev) -> FstSerializer.decode(ev.getData()))); } catch (Exception e) { throw new RuntimeException(e); } diff --git a/java/runtime/src/main/java/io/ray/runtime/task/ArgumentsBuilder.java b/java/runtime/src/main/java/io/ray/runtime/task/ArgumentsBuilder.java index 1e7d2adea..db210ba73 100644 --- a/java/runtime/src/main/java/io/ray/runtime/task/ArgumentsBuilder.java +++ b/java/runtime/src/main/java/io/ray/runtime/task/ArgumentsBuilder.java @@ -16,9 +16,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -/** - * Helper methods to convert arguments from/to objects. - */ +/** Helper methods to convert arguments from/to objects. */ public class ArgumentsBuilder { /** @@ -28,15 +26,11 @@ public class ArgumentsBuilder { // TODO(kfstorm): Read from internal config `max_direct_call_object_size`. public static final int LARGEST_SIZE_PASS_BY_VALUE = 100 * 1024; - /** - * This dummy type is also defined in signature.py. Please keep it synced. - */ - private static final NativeRayObject PYTHON_DUMMY_TYPE = ObjectSerializer - .serialize("__RAY_DUMMY__".getBytes()); + /** This dummy type is also defined in signature.py. Please keep it synced. */ + private static final NativeRayObject PYTHON_DUMMY_TYPE = + ObjectSerializer.serialize("__RAY_DUMMY__".getBytes()); - /** - * Convert real function arguments to task spec arguments. - */ + /** Convert real function arguments to task spec arguments. */ public static List wrap(Object[] args, Language language) { List ret = new ArrayList<>(); for (Object arg : args) { @@ -51,20 +45,21 @@ public class ArgumentsBuilder { value = ObjectSerializer.serialize(arg); if (language != Language.JAVA) { boolean isCrossData = - Bytes.indexOf(value.metadata, - ObjectSerializer.OBJECT_METADATA_TYPE_CROSS_LANGUAGE) == 0 || - Bytes.indexOf(value.metadata, - ObjectSerializer.OBJECT_METADATA_TYPE_RAW) == 0 || - Bytes.indexOf(value.metadata, - ObjectSerializer.OBJECT_METADATA_TYPE_ACTOR_HANDLE) == 0; + Bytes.indexOf(value.metadata, ObjectSerializer.OBJECT_METADATA_TYPE_CROSS_LANGUAGE) + == 0 + || Bytes.indexOf(value.metadata, ObjectSerializer.OBJECT_METADATA_TYPE_RAW) == 0 + || Bytes.indexOf( + value.metadata, ObjectSerializer.OBJECT_METADATA_TYPE_ACTOR_HANDLE) + == 0; if (!isCrossData) { - throw new IllegalArgumentException(String.format("Can't transfer %s data to %s", - Arrays.toString(value.metadata), language.getValueDescriptor().getName())); + throw new IllegalArgumentException( + String.format( + "Can't transfer %s data to %s", + Arrays.toString(value.metadata), language.getValueDescriptor().getName())); } } if (value.data.length > LARGEST_SIZE_PASS_BY_VALUE) { - id = ((RayRuntimeInternal) Ray.internal()).getObjectStore() - .putRaw(value); + id = ((RayRuntimeInternal) Ray.internal()).getObjectStore().putRaw(value); address = ((RayRuntimeInternal) Ray.internal()).getWorkerContext().getRpcAddress(); value = null; } @@ -81,9 +76,7 @@ public class ArgumentsBuilder { return ret; } - /** - * Convert list of NativeRayObject/ByteBuffer to real function arguments. - */ + /** Convert list of NativeRayObject/ByteBuffer to real function arguments. */ public static Object[] unwrap(List args, Class[] types) { Object[] realArgs = new Object[args.size()]; for (int i = 0; i < args.size(); i++) { diff --git a/java/runtime/src/main/java/io/ray/runtime/task/FunctionArg.java b/java/runtime/src/main/java/io/ray/runtime/task/FunctionArg.java index d61c0c525..aca00b50e 100644 --- a/java/runtime/src/main/java/io/ray/runtime/task/FunctionArg.java +++ b/java/runtime/src/main/java/io/ray/runtime/task/FunctionArg.java @@ -6,25 +6,18 @@ import io.ray.runtime.generated.Common.Address; import io.ray.runtime.object.NativeRayObject; /** - * Represents a function argument in task spec. - * Either `id` or `data` should be null, when id is not null, this argument will be - * passed by reference, otherwise it will be passed by value. + * Represents a function argument in task spec. Either `id` or `data` should be null, when id is not + * null, this argument will be passed by reference, otherwise it will be passed by value. */ public class FunctionArg { - /** - * The id of this argument (passed by reference). - */ + /** The id of this argument (passed by reference). */ public final ObjectId id; - /** - * The owner address of this argument (passed by reference). - */ + /** The owner address of this argument (passed by reference). */ public final Address ownerAddress; - /** - * Serialized data of this argument (passed by value). - */ + /** Serialized data of this argument (passed by value). */ public final NativeRayObject value; private FunctionArg(ObjectId id, Address ownerAddress) { @@ -42,16 +35,12 @@ public class FunctionArg { this.value = nativeRayObject; } - /** - * Create a FunctionArg that will be passed by reference. - */ + /** Create a FunctionArg that will be passed by reference. */ public static FunctionArg passByReference(ObjectId id, Address ownerAddress) { return new FunctionArg(id, ownerAddress); } - /** - * Create a FunctionArg that will be passed by value. - */ + /** Create a FunctionArg that will be passed by value. */ public static FunctionArg passByValue(NativeRayObject value) { return new FunctionArg(value); } diff --git a/java/runtime/src/main/java/io/ray/runtime/task/LocalModeTaskExecutor.java b/java/runtime/src/main/java/io/ray/runtime/task/LocalModeTaskExecutor.java index 7d0eeaf57..90d93b0a4 100644 --- a/java/runtime/src/main/java/io/ray/runtime/task/LocalModeTaskExecutor.java +++ b/java/runtime/src/main/java/io/ray/runtime/task/LocalModeTaskExecutor.java @@ -3,16 +3,12 @@ package io.ray.runtime.task; import io.ray.api.id.UniqueId; import io.ray.runtime.RayRuntimeInternal; -/** - * Task executor for local mode. - */ +/** Task executor for local mode. */ public class LocalModeTaskExecutor extends TaskExecutor { static class LocalActorContext extends TaskExecutor.ActorContext { - /** - * The worker ID of the actor. - */ + /** The worker ID of the actor. */ private final UniqueId workerId; public LocalActorContext(UniqueId workerId) { @@ -32,5 +28,4 @@ public class LocalModeTaskExecutor extends TaskExecutor placementGroups = new ConcurrentHashMap<>(); - public LocalModeTaskSubmitter(RayRuntimeInternal runtime, TaskExecutor taskExecutor, - LocalModeObjectStore objectStore) { + public LocalModeTaskSubmitter( + RayRuntimeInternal runtime, TaskExecutor taskExecutor, LocalModeObjectStore objectStore) { this.runtime = runtime; this.taskExecutor = taskExecutor; this.objectStore = objectStore; @@ -117,8 +115,9 @@ public class LocalModeTaskSubmitter implements TaskSubmitter { } } if (taskSpec.getType() == TaskType.ACTOR_TASK) { - ObjectId dummyObjectId = new ObjectId( - taskSpec.getActorTaskSpec().getPreviousActorTaskDummyObjectId().toByteArray()); + ObjectId dummyObjectId = + new ObjectId( + taskSpec.getActorTaskSpec().getPreviousActorTaskDummyObjectId().toByteArray()); if (!objectStore.isObjectReady(dummyObjectId)) { unreadyObjects.add(dummyObjectId); } @@ -126,9 +125,8 @@ public class LocalModeTaskSubmitter implements TaskSubmitter { return unreadyObjects; } - private TaskSpec.Builder getTaskSpecBuilder(TaskType taskType, - FunctionDescriptor functionDescriptor, - List args) { + private TaskSpec.Builder getTaskSpecBuilder( + TaskType taskType, FunctionDescriptor functionDescriptor, List args) { byte[] taskIdBytes = new byte[TaskId.LENGTH]; new Random().nextBytes(taskIdBytes); List functionDescriptorList = functionDescriptor.toList(); @@ -136,64 +134,83 @@ public class LocalModeTaskSubmitter implements TaskSubmitter { return TaskSpec.newBuilder() .setType(taskType) .setLanguage(Language.JAVA) - .setJobId( - ByteString.copyFrom(runtime.getRayConfig().getJobId().getBytes())) + .setJobId(ByteString.copyFrom(runtime.getRayConfig().getJobId().getBytes())) .setTaskId(ByteString.copyFrom(taskIdBytes)) - .setFunctionDescriptor(Common.FunctionDescriptor.newBuilder() - .setJavaFunctionDescriptor( - Common.JavaFunctionDescriptor.newBuilder() - .setClassName(functionDescriptorList.get(0)) - .setFunctionName(functionDescriptorList.get(1)) - .setSignature(functionDescriptorList.get(2)))) - .addAllArgs(args.stream().map(arg -> arg.id != null ? TaskArg.newBuilder() - .setObjectRef(ObjectReference.newBuilder().setObjectId( - ByteString.copyFrom(arg.id.getBytes()))).build() - : TaskArg.newBuilder().setData(ByteString.copyFrom(arg.value.data)) - .setMetadata(arg.value.metadata != null ? ByteString - .copyFrom(arg.value.metadata) : ByteString.EMPTY).build()) - .collect(Collectors.toList())); + .setFunctionDescriptor( + Common.FunctionDescriptor.newBuilder() + .setJavaFunctionDescriptor( + Common.JavaFunctionDescriptor.newBuilder() + .setClassName(functionDescriptorList.get(0)) + .setFunctionName(functionDescriptorList.get(1)) + .setSignature(functionDescriptorList.get(2)))) + .addAllArgs( + args.stream() + .map( + arg -> + arg.id != null + ? TaskArg.newBuilder() + .setObjectRef( + ObjectReference.newBuilder() + .setObjectId(ByteString.copyFrom(arg.id.getBytes()))) + .build() + : TaskArg.newBuilder() + .setData(ByteString.copyFrom(arg.value.data)) + .setMetadata( + arg.value.metadata != null + ? ByteString.copyFrom(arg.value.metadata) + : ByteString.EMPTY) + .build()) + .collect(Collectors.toList())); } @Override - public List submitTask(FunctionDescriptor functionDescriptor, List args, - int numReturns, CallOptions options) { + public List submitTask( + FunctionDescriptor functionDescriptor, + List args, + int numReturns, + CallOptions options) { Preconditions.checkState(numReturns <= 1); - TaskSpec taskSpec = getTaskSpecBuilder(TaskType.NORMAL_TASK, functionDescriptor, args) - .setNumReturns(numReturns) - .build(); + TaskSpec taskSpec = + getTaskSpecBuilder(TaskType.NORMAL_TASK, functionDescriptor, args) + .setNumReturns(numReturns) + .build(); submitTaskSpec(taskSpec); return getReturnIds(taskSpec); } @Override public BaseActorHandle createActor( - FunctionDescriptor functionDescriptor, List args, - ActorCreationOptions options) throws IllegalArgumentException { + FunctionDescriptor functionDescriptor, List args, ActorCreationOptions options) + throws IllegalArgumentException { if (options != null) { if (options.group != null) { - PlacementGroupImpl group = (PlacementGroupImpl)options.group; - Preconditions.checkArgument(options.bundleIndex >= 0 - && options.bundleIndex < group.getBundles().size(), + PlacementGroupImpl group = (PlacementGroupImpl) options.group; + Preconditions.checkArgument( + options.bundleIndex >= 0 && options.bundleIndex < group.getBundles().size(), String.format("Bundle index %s is invalid", options.bundleIndex)); } } ActorId actorId = ActorId.fromRandom(); - TaskSpec taskSpec = getTaskSpecBuilder(TaskType.ACTOR_CREATION_TASK, functionDescriptor, args) - .setNumReturns(1) - .setActorCreationTaskSpec(ActorCreationTaskSpec.newBuilder() - .setActorId(ByteString.copyFrom(actorId.toByteBuffer())) - .build()) - .build(); + TaskSpec taskSpec = + getTaskSpecBuilder(TaskType.ACTOR_CREATION_TASK, functionDescriptor, args) + .setNumReturns(1) + .setActorCreationTaskSpec( + ActorCreationTaskSpec.newBuilder() + .setActorId(ByteString.copyFrom(actorId.toByteBuffer())) + .build()) + .build(); submitTaskSpec(taskSpec); - final LocalModeActorHandle actorHandle - = new LocalModeActorHandle(actorId, getReturnIds(taskSpec).get(0)); + final LocalModeActorHandle actorHandle = + new LocalModeActorHandle(actorId, getReturnIds(taskSpec).get(0)); actorHandles.put(actorId, actorHandle.copy()); if (StringUtils.isNotBlank(options.name)) { - String fullName = options.global ? options.name : - String.format("%s-%s", Ray.getRuntimeContext().getCurrentJobId(), options.name); - Preconditions.checkArgument(!namedActors.containsKey(fullName), - String.format("Actor of name %s exists", fullName)); + String fullName = + options.global + ? options.name + : String.format("%s-%s", Ray.getRuntimeContext().getCurrentJobId(), options.name); + Preconditions.checkArgument( + !namedActors.containsKey(fullName), String.format("Actor of name %s exists", fullName)); namedActors.put(fullName, actorHandle); } return actorHandle; @@ -201,22 +218,29 @@ public class LocalModeTaskSubmitter implements TaskSubmitter { @Override public List submitActorTask( - BaseActorHandle actor, FunctionDescriptor functionDescriptor, - List args, int numReturns, CallOptions options) { + BaseActorHandle actor, + FunctionDescriptor functionDescriptor, + List args, + int numReturns, + CallOptions options) { Preconditions.checkState(numReturns <= 1); TaskSpec.Builder builder = getTaskSpecBuilder(TaskType.ACTOR_TASK, functionDescriptor, args); - List returnIds = getReturnIds( - TaskId.fromBytes(builder.getTaskId().toByteArray()), numReturns + 1); - TaskSpec taskSpec = builder - .setNumReturns(numReturns + 1) - .setActorTaskSpec( - ActorTaskSpec.newBuilder().setActorId(ByteString.copyFrom(actor.getId().getBytes())) - .setPreviousActorTaskDummyObjectId(ByteString.copyFrom( - ((LocalModeActorHandle) actor) - .exchangePreviousActorTaskDummyObjectId(returnIds.get(returnIds.size() - 1)) - .getBytes())) - .build()) - .build(); + List returnIds = + getReturnIds(TaskId.fromBytes(builder.getTaskId().toByteArray()), numReturns + 1); + TaskSpec taskSpec = + builder + .setNumReturns(numReturns + 1) + .setActorTaskSpec( + ActorTaskSpec.newBuilder() + .setActorId(ByteString.copyFrom(actor.getId().getBytes())) + .setPreviousActorTaskDummyObjectId( + ByteString.copyFrom( + ((LocalModeActorHandle) actor) + .exchangePreviousActorTaskDummyObjectId( + returnIds.get(returnIds.size() - 1)) + .getBytes())) + .build()) + .build(); submitTaskSpec(taskSpec); if (numReturns == 0) { return ImmutableList.of(); @@ -226,11 +250,15 @@ public class LocalModeTaskSubmitter implements TaskSubmitter { } @Override - public PlacementGroup createPlacementGroup(String name, List> bundles, - PlacementStrategy strategy) { - PlacementGroupImpl placementGroup = new PlacementGroupImpl.Builder() - .setId(PlacementGroupId.fromRandom()).setName(name) - .setBundles(bundles).setStrategy(strategy).build(); + public PlacementGroup createPlacementGroup( + String name, List> bundles, PlacementStrategy strategy) { + PlacementGroupImpl placementGroup = + new PlacementGroupImpl.Builder() + .setId(PlacementGroupId.fromRandom()) + .setName(name) + .setBundles(bundles) + .setStrategy(strategy) + .build(); placementGroups.put(placementGroup.getId(), placementGroup); return placementGroup; } @@ -251,8 +279,8 @@ public class LocalModeTaskSubmitter implements TaskSubmitter { } public Optional getActor(String name, boolean global) { - String fullName = global ? name : - String.format("%s-%s", Ray.getRuntimeContext().getCurrentJobId(), name); + String fullName = + global ? name : String.format("%s-%s", Ray.getRuntimeContext().getCurrentJobId(), name); ActorHandle actorHandle = namedActors.get(fullName); if (null == actorHandle) { return Optional.empty(); @@ -289,14 +317,15 @@ public class LocalModeTaskSubmitter implements TaskSubmitter { synchronized (taskAndObjectLock) { Set unreadyObjects = getUnreadyObjects(taskSpec); - final Runnable runnable = () -> { - try { - executeTask(taskSpec); - } catch (Exception ex) { - LOGGER.error("Unexpected exception when executing a task.", ex); - System.exit(-1); - } - }; + final Runnable runnable = + () -> { + try { + executeTask(taskSpec); + } catch (Exception ex) { + LOGGER.error("Unexpected exception when executing a task.", ex); + System.exit(-1); + } + }; if (unreadyObjects.isEmpty()) { // If all dependencies are ready, execute this task. @@ -318,7 +347,8 @@ public class LocalModeTaskSubmitter implements TaskSubmitter { executorService.submit(runnable); } catch (RejectedExecutionException e) { if (executorService.isShutdown()) { - LOGGER.warn("Ignore task submission due to the ExecutorService is shutdown. Task: {}", + LOGGER.warn( + "Ignore task submission due to the ExecutorService is shutdown. Task: {}", taskSpec); } } @@ -338,16 +368,20 @@ public class LocalModeTaskSubmitter implements TaskSubmitter { Preconditions.checkNotNull(actorContext); } taskExecutor.setActorContext(actorContext); - List args = getFunctionArgs(taskSpec).stream() - .map(arg -> arg.id != null ? - objectStore.getRaw(Collections.singletonList(arg.id), -1).get(0) - : arg.value) - .collect(Collectors.toList()); + List args = + getFunctionArgs(taskSpec).stream() + .map( + arg -> + arg.id != null + ? objectStore.getRaw(Collections.singletonList(arg.id), -1).get(0) + : arg.value) + .collect(Collectors.toList()); runtime.setIsContextSet(true); ((LocalModeWorkerContext) runtime.getWorkerContext()).setCurrentTask(taskSpec); - UniqueId workerId = actorContext != null - ? ((LocalModeTaskExecutor.LocalActorContext) actorContext).getWorkerId() - : UniqueId.randomId(); + UniqueId workerId = + actorContext != null + ? ((LocalModeTaskExecutor.LocalActorContext) actorContext).getWorkerId() + : UniqueId.randomId(); ((LocalModeWorkerContext) runtime.getWorkerContext()).setCurrentWorkerId(workerId); List rayFunctionInfo = getJavaFunctionDescriptor(taskSpec).toList(); taskExecutor.checkByteBufferArguments(rayFunctionInfo); @@ -379,10 +413,9 @@ public class LocalModeTaskSubmitter implements TaskSubmitter { } private static JavaFunctionDescriptor getJavaFunctionDescriptor(TaskSpec taskSpec) { - Common.FunctionDescriptor functionDescriptor = - taskSpec.getFunctionDescriptor(); - if (functionDescriptor.getFunctionDescriptorCase() == - Common.FunctionDescriptor.FunctionDescriptorCase.JAVA_FUNCTION_DESCRIPTOR) { + Common.FunctionDescriptor functionDescriptor = taskSpec.getFunctionDescriptor(); + if (functionDescriptor.getFunctionDescriptorCase() + == Common.FunctionDescriptor.FunctionDescriptorCase.JAVA_FUNCTION_DESCRIPTOR) { return new JavaFunctionDescriptor( functionDescriptor.getJavaFunctionDescriptor().getClassName(), functionDescriptor.getJavaFunctionDescriptor().getFunctionName(), @@ -397,30 +430,35 @@ public class LocalModeTaskSubmitter implements TaskSubmitter { for (int i = 0; i < taskSpec.getArgsCount(); i++) { TaskArg arg = taskSpec.getArgs(i); if (arg.getObjectRef().getObjectId() != ByteString.EMPTY) { - functionArgs.add(FunctionArg - .passByReference(new ObjectId(arg.getObjectRef().getObjectId().toByteArray()), - Address.getDefaultInstance())); + functionArgs.add( + FunctionArg.passByReference( + new ObjectId(arg.getObjectRef().getObjectId().toByteArray()), + Address.getDefaultInstance())); } else { - functionArgs.add(FunctionArg.passByValue( - new NativeRayObject(arg.getData().toByteArray(), arg.getMetadata().toByteArray()))); + functionArgs.add( + FunctionArg.passByValue( + new NativeRayObject(arg.getData().toByteArray(), arg.getMetadata().toByteArray()))); } } return functionArgs; } private static List getReturnIds(TaskSpec taskSpec) { - return getReturnIds(TaskId.fromBytes(taskSpec.getTaskId().toByteArray()), - taskSpec.getNumReturns()); + return getReturnIds( + TaskId.fromBytes(taskSpec.getTaskId().toByteArray()), taskSpec.getNumReturns()); } private static List getReturnIds(TaskId taskId, long numReturns) { List returnIds = new ArrayList<>(); for (int i = 0; i < numReturns; i++) { - returnIds.add(ObjectId.fromByteBuffer( - (ByteBuffer) ByteBuffer.allocate(ObjectId.LENGTH).put(taskId.getBytes()) - .putInt(TaskId.LENGTH, i + 1).position(0))); + returnIds.add( + ObjectId.fromByteBuffer( + (ByteBuffer) + ByteBuffer.allocate(ObjectId.LENGTH) + .put(taskId.getBytes()) + .putInt(TaskId.LENGTH, i + 1) + .position(0))); } return returnIds; } - } diff --git a/java/runtime/src/main/java/io/ray/runtime/task/NativeTaskExecutor.java b/java/runtime/src/main/java/io/ray/runtime/task/NativeTaskExecutor.java index 63aa5c0d7..e13e98fd8 100644 --- a/java/runtime/src/main/java/io/ray/runtime/task/NativeTaskExecutor.java +++ b/java/runtime/src/main/java/io/ray/runtime/task/NativeTaskExecutor.java @@ -3,13 +3,10 @@ package io.ray.runtime.task; import io.ray.api.id.UniqueId; import io.ray.runtime.RayRuntimeInternal; -/** - * Task executor for cluster mode. - */ +/** Task executor for cluster mode. */ public class NativeTaskExecutor extends TaskExecutor { - static class NativeActorContext extends TaskExecutor.ActorContext { - } + static class NativeActorContext extends TaskExecutor.ActorContext {} public NativeTaskExecutor(RayRuntimeInternal runtime) { super(runtime); diff --git a/java/runtime/src/main/java/io/ray/runtime/task/NativeTaskSubmitter.java b/java/runtime/src/main/java/io/ray/runtime/task/NativeTaskSubmitter.java index 15e193360..221e42e43 100644 --- a/java/runtime/src/main/java/io/ray/runtime/task/NativeTaskSubmitter.java +++ b/java/runtime/src/main/java/io/ray/runtime/task/NativeTaskSubmitter.java @@ -20,16 +20,18 @@ import java.util.Optional; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; -/** - * Task submitter for cluster mode. This is a wrapper class for core worker task interface. - */ +/** Task submitter for cluster mode. This is a wrapper class for core worker task interface. */ public class NativeTaskSubmitter implements TaskSubmitter { @Override - public List submitTask(FunctionDescriptor functionDescriptor, List args, - int numReturns, CallOptions options) { - List returnIds = nativeSubmitTask(functionDescriptor, functionDescriptor.hashCode(), - args, numReturns, options); + public List submitTask( + FunctionDescriptor functionDescriptor, + List args, + int numReturns, + CallOptions options) { + List returnIds = + nativeSubmitTask( + functionDescriptor, functionDescriptor.hashCode(), args, numReturns, options); if (returnIds == null) { return ImmutableList.of(); } @@ -37,25 +39,26 @@ public class NativeTaskSubmitter implements TaskSubmitter { } @Override - public BaseActorHandle createActor(FunctionDescriptor functionDescriptor, List args, - ActorCreationOptions options) throws IllegalArgumentException { + public BaseActorHandle createActor( + FunctionDescriptor functionDescriptor, List args, ActorCreationOptions options) + throws IllegalArgumentException { if (options != null) { if (options.group != null) { - PlacementGroupImpl group = (PlacementGroupImpl)options.group; - Preconditions.checkArgument(options.bundleIndex >= 0 - && options.bundleIndex < group.getBundles().size(), + PlacementGroupImpl group = (PlacementGroupImpl) options.group; + Preconditions.checkArgument( + options.bundleIndex >= 0 && options.bundleIndex < group.getBundles().size(), String.format("Bundle index %s is invalid", options.bundleIndex)); } if (StringUtils.isNotBlank(options.name)) { Optional actor = options.global ? Ray.getGlobalActor(options.name) : Ray.getActor(options.name); - Preconditions.checkArgument(!actor.isPresent(), - String.format("Actor of name %s exists", options.name)); + Preconditions.checkArgument( + !actor.isPresent(), String.format("Actor of name %s exists", options.name)); } } - byte[] actorId = nativeCreateActor(functionDescriptor, functionDescriptor.hashCode(), args, - options); + byte[] actorId = + nativeCreateActor(functionDescriptor, functionDescriptor.hashCode(), args, options); return NativeActorHandle.create(actorId, functionDescriptor.getLanguage()); } @@ -66,11 +69,20 @@ public class NativeTaskSubmitter implements TaskSubmitter { @Override public List submitActorTask( - BaseActorHandle actor, FunctionDescriptor functionDescriptor, - List args, int numReturns, CallOptions options) { + BaseActorHandle actor, + FunctionDescriptor functionDescriptor, + List args, + int numReturns, + CallOptions options) { Preconditions.checkState(actor instanceof NativeActorHandle); - List returnIds = nativeSubmitActorTask(actor.getId().getBytes(), - functionDescriptor, functionDescriptor.hashCode(), args, numReturns, options); + List returnIds = + nativeSubmitActorTask( + actor.getId().getBytes(), + functionDescriptor, + functionDescriptor.hashCode(), + args, + numReturns, + options); if (returnIds == null) { return ImmutableList.of(); } @@ -78,12 +90,15 @@ public class NativeTaskSubmitter implements TaskSubmitter { } @Override - public PlacementGroup createPlacementGroup(String name, List> bundles, - PlacementStrategy strategy) { + public PlacementGroup createPlacementGroup( + String name, List> bundles, PlacementStrategy strategy) { byte[] bytes = nativeCreatePlacementGroup(name, bundles, strategy.value()); return new PlacementGroupImpl.Builder() - .setId(PlacementGroupId.fromBytes(bytes)) - .setName(name).setBundles(bundles).setStrategy(strategy).build(); + .setId(PlacementGroupId.fromBytes(bytes)) + .setName(name) + .setBundles(bundles) + .setStrategy(strategy) + .build(); } @Override @@ -96,22 +111,32 @@ public class NativeTaskSubmitter implements TaskSubmitter { return nativeWaitPlacementGroupReady(id.getBytes(), timeoutMs); } - private static native List nativeSubmitTask(FunctionDescriptor functionDescriptor, - int functionDescriptorHash, List args, int numReturns, CallOptions callOptions); + private static native List nativeSubmitTask( + FunctionDescriptor functionDescriptor, + int functionDescriptorHash, + List args, + int numReturns, + CallOptions callOptions); - private static native byte[] nativeCreateActor(FunctionDescriptor functionDescriptor, - int functionDescriptorHash, List args, + private static native byte[] nativeCreateActor( + FunctionDescriptor functionDescriptor, + int functionDescriptorHash, + List args, ActorCreationOptions actorCreationOptions); - private static native List nativeSubmitActorTask(byte[] actorId, - FunctionDescriptor functionDescriptor, int functionDescriptorHash, List args, - int numReturns, CallOptions callOptions); + private static native List nativeSubmitActorTask( + byte[] actorId, + FunctionDescriptor functionDescriptor, + int functionDescriptorHash, + List args, + int numReturns, + CallOptions callOptions); - private static native byte[] nativeCreatePlacementGroup(String name, - List> bundles, int strategy); + private static native byte[] nativeCreatePlacementGroup( + String name, List> bundles, int strategy); private static native void nativeRemovePlacementGroup(byte[] placementGroupId); - private static native boolean nativeWaitPlacementGroupReady(byte[] placementGroupId, - int timeoutMs); + private static native boolean nativeWaitPlacementGroupReady( + byte[] placementGroupId, int timeoutMs); } diff --git a/java/runtime/src/main/java/io/ray/runtime/task/TaskExecutor.java b/java/runtime/src/main/java/io/ray/runtime/task/TaskExecutor.java index 97d71b0bd..99a5cf7c8 100644 --- a/java/runtime/src/main/java/io/ray/runtime/task/TaskExecutor.java +++ b/java/runtime/src/main/java/io/ray/runtime/task/TaskExecutor.java @@ -20,9 +20,7 @@ import java.util.concurrent.ConcurrentHashMap; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * The task executor, which executes tasks assigned by raylet continuously. - */ +/** The task executor, which executes tasks assigned by raylet continuously. */ public abstract class TaskExecutor { private static final Logger LOGGER = LoggerFactory.getLogger(TaskExecutor.class); @@ -35,14 +33,10 @@ public abstract class TaskExecutor { static class ActorContext { - /** - * The current actor object, if this worker is an actor, otherwise null. - */ + /** The current actor object, if this worker is an actor, otherwise null. */ Object currentActor = null; - /** - * The exception that failed the actor creation task, if any. - */ + /** The exception that failed the actor creation task, if any. */ Throwable actorCreationException = null; } @@ -74,9 +68,7 @@ public abstract class TaskExecutor { return runtime.getFunctionManager().getFunction(jobId, functionDescriptor); } - /** - * The return value indicates which parameters are ByteBuffer. - */ + /** The return value indicates which parameters are ByteBuffer. */ protected boolean[] checkByteBufferArguments(List rayFunctionInfo) { localRayFunction.set(null); try { @@ -93,8 +85,7 @@ public abstract class TaskExecutor { return results; } - protected List execute(List rayFunctionInfo, - List argsBytes) { + protected List execute(List rayFunctionInfo, List argsBytes) { runtime.setIsContextSet(true); TaskType taskType = runtime.getWorkerContext().getCurrentTaskType(); TaskId taskId = runtime.getWorkerContext().getCurrentTaskId(); @@ -130,8 +121,8 @@ public abstract class TaskExecutor { } actor = actorContext.currentActor; } - Object[] args = ArgumentsBuilder - .unwrap(argsBytes, rayFunction.executable.getParameterTypes()); + Object[] args = + ArgumentsBuilder.unwrap(argsBytes, rayFunction.executable.getParameterTypes()); // Execute the task. Object result; try { @@ -168,8 +159,9 @@ public abstract class TaskExecutor { boolean hasReturn = rayFunction != null && rayFunction.hasReturn(); boolean isCrossLanguage = parseFunctionDescriptor(rayFunctionInfo).signature.equals(""); if (hasReturn || isCrossLanguage) { - returnObjects.add(ObjectSerializer - .serialize(new RayTaskException("Error executing task " + taskId, e))); + returnObjects.add( + ObjectSerializer.serialize( + new RayTaskException("Error executing task " + taskId, e))); } } else { actorContext.actorCreationException = e; @@ -184,8 +176,7 @@ public abstract class TaskExecutor { private JavaFunctionDescriptor parseFunctionDescriptor(List rayFunctionInfo) { Preconditions.checkState(rayFunctionInfo != null && rayFunctionInfo.size() == 3); - return new JavaFunctionDescriptor(rayFunctionInfo.get(0), rayFunctionInfo.get(1), - rayFunctionInfo.get(2)); + return new JavaFunctionDescriptor( + rayFunctionInfo.get(0), rayFunctionInfo.get(1), rayFunctionInfo.get(2)); } - } diff --git a/java/runtime/src/main/java/io/ray/runtime/task/TaskSubmitter.java b/java/runtime/src/main/java/io/ray/runtime/task/TaskSubmitter.java index 17a8d34f9..ca195d6ce 100644 --- a/java/runtime/src/main/java/io/ray/runtime/task/TaskSubmitter.java +++ b/java/runtime/src/main/java/io/ray/runtime/task/TaskSubmitter.java @@ -12,70 +12,76 @@ import io.ray.runtime.functionmanager.FunctionDescriptor; import java.util.List; import java.util.Map; -/** - * A set of methods to submit tasks and create actors. - */ +/** A set of methods to submit tasks and create actors. */ public interface TaskSubmitter { /** * Submit a normal task. + * * @param functionDescriptor The remote function to execute. * @param args Arguments of this task. * @param numReturns Return object count. - * @param options Options for this task. - * @return Ids of the return objects. + * @param options Options for this task. Returns Ids of the return objects. */ - List submitTask(FunctionDescriptor functionDescriptor, List args, - int numReturns, CallOptions options); + List submitTask( + FunctionDescriptor functionDescriptor, + List args, + int numReturns, + CallOptions options); /** * Create an actor. + * * @param functionDescriptor The remote function that generates the actor object. * @param args Arguments of this task. - * @param options Options for this actor creation task. - * @return Handle to the actor. + * @param options Options for this actor creation task. Returns Handle to the actor. * @throws IllegalArgumentException if actor of specified name exists */ - BaseActorHandle createActor(FunctionDescriptor functionDescriptor, List args, - ActorCreationOptions options) throws IllegalArgumentException; + BaseActorHandle createActor( + FunctionDescriptor functionDescriptor, List args, ActorCreationOptions options) + throws IllegalArgumentException; /** * Submit an actor task. + * * @param actor Handle to the actor. * @param functionDescriptor The remote function to execute. * @param args Arguments of this task. * @param numReturns Return object count. - * @param options Options for this task. - * @return Ids of the return objects. + * @param options Options for this task. Returns Ids of the return objects. */ - List submitActorTask(BaseActorHandle actor, FunctionDescriptor functionDescriptor, - List args, int numReturns, CallOptions options); + List submitActorTask( + BaseActorHandle actor, + FunctionDescriptor functionDescriptor, + List args, + int numReturns, + CallOptions options); /** * Create a placement group. * * @param name Name of the placement group. * @param bundles Pre-allocated resource list. - * @param strategy Actor placement strategy. - * @return A handle to the created placement group. + * @param strategy Actor placement strategy. Returns A handle to the created placement group. */ - PlacementGroup createPlacementGroup(String name, List> bundles, - PlacementStrategy strategy); + PlacementGroup createPlacementGroup( + String name, List> bundles, PlacementStrategy strategy); /** * Remove a placement group by id. + * * @param id Id of the placement group. */ void removePlacementGroup(PlacementGroupId id); /** * Wait for the placement group to be ready within the specified time. + * * @param id Id of placement group. - * @param timeoutMs Timeout in milliseconds. - * @return True if the placement group is created. False otherwise. + * @param timeoutMs Timeout in milliseconds. Returns True if the placement group is created. False + * otherwise. */ boolean waitPlacementGroupReady(PlacementGroupId id, int timeoutMs); BaseActorHandle getActor(ActorId actorId); - } diff --git a/java/runtime/src/main/java/io/ray/runtime/util/BinaryFileUtil.java b/java/runtime/src/main/java/io/ray/runtime/util/BinaryFileUtil.java index b73e7826d..85c327a44 100644 --- a/java/runtime/src/main/java/io/ray/runtime/util/BinaryFileUtil.java +++ b/java/runtime/src/main/java/io/ray/runtime/util/BinaryFileUtil.java @@ -16,13 +16,12 @@ public class BinaryFileUtil { public static final String CORE_WORKER_JAVA_LIBRARY = "core_worker_library_java"; /** - * Extract a platform-native resource file to destDir. - * Note that this a process-safe operation. If multi processes extract the file to same - * directory concurrently, this operation will be protected by a file lock. + * Extract a platform-native resource file to destDir. Note that this a process-safe + * operation. If multi processes extract the file to same directory concurrently, this operation + * will be protected by a file lock. * - * @param destDir a directory to extract resource file to - * @param fileName resource file name - * @return extracted resource file + * @param destDir a directory to extract resource file to + * @param fileName resource file name Returns extracted resource file */ public static File getNativeFile(String destDir, String fileName) { final File dir = new File(destDir); @@ -34,8 +33,7 @@ public class BinaryFileUtil { } } String lockFilePath = destDir + File.separator + "file_lock"; - try (FileLock ignored = new RandomAccessFile(lockFilePath, "rw") - .getChannel().lock()) { + try (FileLock ignored = new RandomAccessFile(lockFilePath, "rw").getChannel().lock()) { String resourceDir; if (SystemUtils.IS_OS_MAC) { resourceDir = "native/darwin/"; diff --git a/java/runtime/src/main/java/io/ray/runtime/util/IdUtil.java b/java/runtime/src/main/java/io/ray/runtime/util/IdUtil.java index 23b24728c..4f7bf2580 100644 --- a/java/runtime/src/main/java/io/ray/runtime/util/IdUtil.java +++ b/java/runtime/src/main/java/io/ray/runtime/util/IdUtil.java @@ -12,15 +12,16 @@ public class IdUtil { /** * Compute the actor ID of the task which created this object. - * @return The actor ID of the task which created this object. + * + *

Returns The actor ID of the task which created this object. */ public static ActorId getActorIdFromObjectId(ObjectId objectId) { byte[] taskIdBytes = new byte[TaskId.LENGTH]; System.arraycopy(objectId.getBytes(), 0, taskIdBytes, 0, TaskId.LENGTH); TaskId taskId = TaskId.fromBytes(taskIdBytes); byte[] actorIdBytes = new byte[ActorId.LENGTH]; - System.arraycopy(taskId.getBytes(), TaskId.UNIQUE_BYTES_LENGTH, - actorIdBytes, 0, ActorId.LENGTH); + System.arraycopy( + taskId.getBytes(), TaskId.UNIQUE_BYTES_LENGTH, actorIdBytes, 0, ActorId.LENGTH); return ActorId.fromBytes(actorIdBytes); } } diff --git a/java/runtime/src/main/java/io/ray/runtime/util/JniExceptionUtil.java b/java/runtime/src/main/java/io/ray/runtime/util/JniExceptionUtil.java index 7f3b727c9..7ad1647af 100644 --- a/java/runtime/src/main/java/io/ray/runtime/util/JniExceptionUtil.java +++ b/java/runtime/src/main/java/io/ray/runtime/util/JniExceptionUtil.java @@ -9,10 +9,14 @@ public final class JniExceptionUtil { private static final Logger LOGGER = LoggerFactory.getLogger(JniExceptionUtil.class); - public static String getStackTrace(String fileName, int lineNumber, String function, - Throwable throwable) { - LOGGER.error("An unexpected exception occurred while executing Java code from JNI ({}:{} {}).", - fileName, lineNumber, function, throwable); + public static String getStackTrace( + String fileName, int lineNumber, String function, Throwable throwable) { + LOGGER.error( + "An unexpected exception occurred while executing Java code from JNI ({}:{} {}).", + fileName, + lineNumber, + function, + throwable); // Return the exception in string form to JNI. return ExceptionUtils.getStackTrace(throwable); } diff --git a/java/runtime/src/main/java/io/ray/runtime/util/JniUtils.java b/java/runtime/src/main/java/io/ray/runtime/util/JniUtils.java index f63cf72d3..b3e763e21 100644 --- a/java/runtime/src/main/java/io/ray/runtime/util/JniUtils.java +++ b/java/runtime/src/main/java/io/ray/runtime/util/JniUtils.java @@ -15,20 +15,20 @@ public class JniUtils { private static String defaultDestDir; /** - * Loads the native library specified by the libraryName argument. - * The libraryName argument must not contain any platform specific - * prefix, file extension or path. + * Loads the native library specified by the libraryName argument. The + * libraryName argument must not contain any platform specific prefix, file extension or + * path. * - * @param libraryName the name of the library. + * @param libraryName the name of the library. */ public static synchronized void loadLibrary(String libraryName) { loadLibrary(getDefaultDestDir(), libraryName); } /** - * Loads the native library specified by the libraryName argument. - * The libraryName argument must not contain any platform specific - * prefix, file extension or path. + * Loads the native library specified by the libraryName argument. The + * libraryName argument must not contain any platform specific prefix, file extension or + * path. * * @param libraryName the name of the library. * @param exportSymbols export symbols of library so that it can be used by other libs. @@ -38,9 +38,9 @@ public class JniUtils { } /** - * Loads the native library specified by the libraryName argument. - * The libraryName argument must not contain any platform specific - * prefix, file extension or path. + * Loads the native library specified by the libraryName argument. The + * libraryName argument must not contain any platform specific prefix, file extension or + * path. * * @param destDir The destination dir the library to be extracted. * @param libraryName the name of the library. @@ -50,16 +50,16 @@ public class JniUtils { } /** - * Loads the native library specified by the libraryName argument. - * The libraryName argument must not contain any platform specific - * prefix, file extension or path. + * Loads the native library specified by the libraryName argument. The + * libraryName argument must not contain any platform specific prefix, file extension or + * path. * * @param destDir The destination dir the library to be extracted. - * @param libraryName the name of the library. + * @param libraryName the name of the library. * @param exportSymbols export symbols of library so that it can be used by other libs. */ - public static synchronized void loadLibrary(String destDir, String libraryName, - boolean exportSymbols) { + public static synchronized void loadLibrary( + String destDir, String libraryName, boolean exportSymbols) { if (!loadedLibs.contains(libraryName)) { LOGGER.debug("Loading native library {}.", libraryName); // Load native library. @@ -77,9 +77,7 @@ public class JniUtils { } } - /** - * Cache the result so that multiple calls return the same dest dir. - */ + /** Cache the result so that multiple calls return the same dest dir. */ private static synchronized String getDefaultDestDir() { if (defaultDestDir == null) { try { diff --git a/java/runtime/src/main/java/io/ray/runtime/util/LambdaUtils.java b/java/runtime/src/main/java/io/ray/runtime/util/LambdaUtils.java index 9a8a3a901..a36d7d9fe 100644 --- a/java/runtime/src/main/java/io/ray/runtime/util/LambdaUtils.java +++ b/java/runtime/src/main/java/io/ray/runtime/util/LambdaUtils.java @@ -4,14 +4,10 @@ import java.io.Serializable; import java.lang.invoke.SerializedLambda; import java.lang.reflect.Method; -/** - * see http://cr.openjdk.java.net/~briangoetz/lambda/lambda-translation.html. - */ +/** see http://cr.openjdk.java.net/~briangoetz/lambda/lambda-translation.html. */ public final class LambdaUtils { - private LambdaUtils() { - } - + private LambdaUtils() {} public static SerializedLambda getSerializedLambda(Serializable lambda) { // Note. @@ -26,6 +22,4 @@ public final class LambdaUtils { throw new RuntimeException("failed to getSerializedLambda:" + lambda.getClass().getName(), e); } } - - -} \ No newline at end of file +} diff --git a/java/runtime/src/main/java/io/ray/runtime/util/LoggingUtil.java b/java/runtime/src/main/java/io/ray/runtime/util/LoggingUtil.java index 29f3bd75a..d31120fde 100644 --- a/java/runtime/src/main/java/io/ray/runtime/util/LoggingUtil.java +++ b/java/runtime/src/main/java/io/ray/runtime/util/LoggingUtil.java @@ -44,5 +44,4 @@ public class LoggingUtil { appender.activateOptions(); Logger.getLogger("io.ray").addAppender(appender); } - } diff --git a/java/runtime/src/main/java/io/ray/runtime/util/NetworkUtil.java b/java/runtime/src/main/java/io/ray/runtime/util/NetworkUtil.java index 56b047982..26aca4e77 100644 --- a/java/runtime/src/main/java/io/ray/runtime/util/NetworkUtil.java +++ b/java/runtime/src/main/java/io/ray/runtime/util/NetworkUtil.java @@ -27,8 +27,8 @@ public class NetworkUtil { if (!current.isUp() || current.isLoopback() || current.isVirtual()) { continue; } - if (!Strings.isNullOrEmpty(interfaceName) && !interfaceName - .equals(current.getDisplayName())) { + if (!Strings.isNullOrEmpty(interfaceName) + && !interfaceName.equals(current.getDisplayName())) { continue; } Enumeration addresses = current.getInetAddresses(); @@ -65,7 +65,8 @@ public class NetworkUtil { throw new IllegalArgumentException("Invalid start port: " + port); } - try (ServerSocket ss = new ServerSocket(port); DatagramSocket ds = new DatagramSocket(port)) { + try (ServerSocket ss = new ServerSocket(port); + DatagramSocket ds = new DatagramSocket(port)) { ss.setReuseAddress(true); ds.setReuseAddress(true); return true; diff --git a/java/runtime/src/main/java/io/ray/runtime/util/ResourceUtil.java b/java/runtime/src/main/java/io/ray/runtime/util/ResourceUtil.java index 3b32bef66..0c7a93d27 100644 --- a/java/runtime/src/main/java/io/ray/runtime/util/ResourceUtil.java +++ b/java/runtime/src/main/java/io/ray/runtime/util/ResourceUtil.java @@ -8,11 +8,11 @@ public class ResourceUtil { public static final String GPU_LITERAL = "GPU"; /** - * Convert resources map to a string that is used - * for the command line argument of starting raylet. + * Convert resources map to a string that is used for the command line argument of starting + * raylet. * - * @param resources The resources map to be converted. - * @return The starting-raylet command line argument, like "CPU,4,GPU,0". + * @param resources The resources map to be converted. Returns The starting-raylet command line + * argument, like "CPU,4,GPU,0". */ public static String getResourcesStringFromMap(Map resources) { StringBuilder builder = new StringBuilder(); @@ -32,11 +32,10 @@ public class ResourceUtil { /** * Parse the static resources configure field and convert to the resources map. * - * @param resources The static resources string to be parsed. - * @return The map whose key represents the resource name - * and the value represents the resource quantity. - * @throws IllegalArgumentException If the resources string's format does match, - * it will throw an IllegalArgumentException. + * @param resources The static resources string to be parsed. Returns The map whose key represents + * the resource name and the value represents the resource quantity. + * @throws IllegalArgumentException If the resources string's format does match, it will throw an + * IllegalArgumentException. */ public static Map getResourcesMapFromString(String resources) throws IllegalArgumentException { diff --git a/java/runtime/src/main/java/io/ray/runtime/util/SystemUtil.java b/java/runtime/src/main/java/io/ray/runtime/util/SystemUtil.java index 37bb88bb1..52a915581 100644 --- a/java/runtime/src/main/java/io/ray/runtime/util/SystemUtil.java +++ b/java/runtime/src/main/java/io/ray/runtime/util/SystemUtil.java @@ -5,9 +5,7 @@ import java.lang.management.ManagementFactory; import java.lang.management.RuntimeMXBean; import java.util.concurrent.locks.ReentrantLock; -/** - * some utilities for system process. - */ +/** some utilities for system process. */ public class SystemUtil { static final ReentrantLock pidlock = new ReentrantLock(); @@ -33,13 +31,12 @@ public class SystemUtil { } } return pid; - } public static boolean isProcessAlive(int pid) { Process process; try { - process = Runtime.getRuntime().exec(new String[]{"ps", "-p", String.valueOf(pid)}); + process = Runtime.getRuntime().exec(new String[] {"ps", "-p", String.valueOf(pid)}); process.waitFor(); } catch (InterruptedException | IOException e) { throw new RuntimeException(e); diff --git a/java/runtime/src/main/java/io/ray/runtime/util/generator/BaseGenerator.java b/java/runtime/src/main/java/io/ray/runtime/util/generator/BaseGenerator.java index 24a055175..c6008501a 100644 --- a/java/runtime/src/main/java/io/ray/runtime/util/generator/BaseGenerator.java +++ b/java/runtime/src/main/java/io/ray/runtime/util/generator/BaseGenerator.java @@ -20,5 +20,4 @@ public abstract class BaseGenerator { sb.append(" "); } } - } diff --git a/java/runtime/src/main/java/io/ray/runtime/util/generator/RayCallGenerator.java b/java/runtime/src/main/java/io/ray/runtime/util/generator/RayCallGenerator.java index 16624d6be..a7991613b 100644 --- a/java/runtime/src/main/java/io/ray/runtime/util/generator/RayCallGenerator.java +++ b/java/runtime/src/main/java/io/ray/runtime/util/generator/RayCallGenerator.java @@ -13,9 +13,7 @@ import org.apache.commons.io.FileUtils; */ public class RayCallGenerator extends BaseGenerator { - /** - * @return Whole file content of `RayCall.java`. - */ + /** Returns Whole file content of `RayCall.java`. */ private String generateRayCallDotJava() { sb = new StringBuilder(); @@ -70,9 +68,7 @@ public class RayCallGenerator extends BaseGenerator { return sb.toString(); } - /** - * @return Whole file content of `ActorCall.java`. - */ + /** Returns Whole file content of `ActorCall.java`. */ private String generateActorCallDotJava() { sb = new StringBuilder(); @@ -102,9 +98,7 @@ public class RayCallGenerator extends BaseGenerator { return sb.toString(); } - /** - * @return Whole file content of `PyActorCall.java`. - */ + /** Returns Whole file content of `PyActorCall.java`. */ private String generatePyActorCallDotJava() { sb = new StringBuilder(); @@ -128,16 +122,16 @@ public class RayCallGenerator extends BaseGenerator { } /** - * Build `Ray.call`, `Ray.createActor` and `actor.call` methods with - * the given number of parameters. + * Build `Ray.call`, `Ray.createActor` and `actor.call` methods with the given number of + * parameters. * * @param numParameters the number of parameters * @param forActor Build `actor.call` when true, otherwise build `Ray.call`. * @param hasReturn if true, Build api for functions with return. * @param forActorCreation Build `Ray.createActor` when true, otherwise build `Ray.call`. */ - private void buildCalls(int numParameters, boolean forActor, - boolean forActorCreation, boolean hasReturn) { + private void buildCalls( + int numParameters, boolean forActor, boolean forActorCreation, boolean hasReturn) { // Template of the generated function: // [modifiers] [genericTypes] [returnType] [callFunc]([argsDeclaration]) { // Objects[] args = new Object[]{[args]}; @@ -186,10 +180,12 @@ public class RayCallGenerator extends BaseGenerator { rayFuncGenericTypes = rayFuncGenericTypes.replace("<", ""; - String returnType = forActorCreation ? "PyActorCreator" : - forActor ? "PyActorTaskCaller" : "PyTaskCaller"; + String returnType = + forActorCreation ? "PyActorCreator" : forActor ? "PyActorTaskCaller" : "PyTaskCaller"; String funcName = forActorCreation ? "actor" : "task"; - String caller = forActorCreation ? "PyActorCreator" : - forActor ? "PyActorTaskCaller<>" : "PyTaskCaller<>"; + String caller = + forActorCreation ? "PyActorCreator" : forActor ? "PyActorTaskCaller<>" : "PyTaskCaller<>"; funcArgs += ", args"; // Method signature. - newLine(1, String.format( - "%s%s %s %s(%s) {", modifiers, genericType, - returnType, funcName, paramPrefix + paramList - )); + newLine( + 1, + String.format( + "%s%s %s %s(%s) {", + modifiers, genericType, returnType, funcName, paramPrefix + paramList)); // Method body. newLine(2, String.format("Object[] args = new Object[]{%s};", argList)); if (forActor) { - newLine(2, String.format("return new %s((PyActorHandle)this, %s);", - caller, funcArgs)); + newLine(2, String.format("return new %s((PyActorHandle)this, %s);", caller, funcArgs)); } else { newLine(2, String.format("return new %s(%s);", caller, funcArgs)); } @@ -323,17 +323,18 @@ public class RayCallGenerator extends BaseGenerator { } public static void main(String[] args) throws IOException { - String path = System.getProperty("user.dir") - + "/api/src/main/java/io/ray/api/RayCall.java"; - FileUtils.write(new File(path), new RayCallGenerator().generateRayCallDotJava(), + String path = System.getProperty("user.dir") + "/api/src/main/java/io/ray/api/RayCall.java"; + FileUtils.write( + new File(path), new RayCallGenerator().generateRayCallDotJava(), Charset.defaultCharset()); + path = System.getProperty("user.dir") + "/api/src/main/java/io/ray/api/ActorCall.java"; + FileUtils.write( + new File(path), + new RayCallGenerator().generateActorCallDotJava(), Charset.defaultCharset()); - path = System.getProperty("user.dir") - + "/api/src/main/java/io/ray/api/ActorCall.java"; - FileUtils.write(new File(path), new RayCallGenerator().generateActorCallDotJava(), - Charset.defaultCharset()); - path = System.getProperty("user.dir") - + "/api/src/main/java/io/ray/api/PyActorCall.java"; - FileUtils.write(new File(path), new RayCallGenerator().generatePyActorCallDotJava(), + path = System.getProperty("user.dir") + "/api/src/main/java/io/ray/api/PyActorCall.java"; + FileUtils.write( + new File(path), + new RayCallGenerator().generatePyActorCallDotJava(), Charset.defaultCharset()); } } diff --git a/java/runtime/src/main/java/io/ray/runtime/util/generator/RayFuncGenerator.java b/java/runtime/src/main/java/io/ray/runtime/util/generator/RayFuncGenerator.java index 3567a8fe6..61cb2a99f 100644 --- a/java/runtime/src/main/java/io/ray/runtime/util/generator/RayFuncGenerator.java +++ b/java/runtime/src/main/java/io/ray/runtime/util/generator/RayFuncGenerator.java @@ -5,9 +5,7 @@ import java.io.IOException; import java.nio.charset.Charset; import org.apache.commons.io.FileUtils; -/** - * A util class that generates all the RayFuncX classes under io.ray.api.function package. - */ +/** A util class that generates all the RayFuncX classes under io.ray.api.function package. */ public class RayFuncGenerator extends BaseGenerator { private String generate(int numParameters, boolean hasReturn) { @@ -36,15 +34,18 @@ public class RayFuncGenerator extends BaseGenerator { newLine("package io.ray.api.function;"); newLine(""); newLine("/**"); - String comment = String.format( - " * Functional interface for a remote function that has %d parameter%s.", - numParameters, numParameters > 1 ? "s" : ""); + String comment = + String.format( + " * Functional interface for a remote function that has %d parameter%s.", + numParameters, numParameters > 1 ? "s" : ""); newLine(comment); newLine(" */"); newLine("@FunctionalInterface"); String className = "RayFunc" + (hasReturn ? "" : "Void") + numParameters; - newLine(String.format("public interface %s%s extends %s {", - className, genericTypes, hasReturn ? "RayFuncR" : "RayFuncVoid")); + newLine( + String.format( + "public interface %s%s extends %s {", + className, genericTypes, hasReturn ? "RayFuncR" : "RayFuncVoid")); newLine(""); indents(1); newLine(String.format("%s apply(%s) throws Exception;", hasReturn ? "R" : "void", paramList)); @@ -54,19 +55,16 @@ public class RayFuncGenerator extends BaseGenerator { } public static void main(String[] args) throws IOException { - String root = System.getProperty("user.dir") - + "/api/src/main/java/io/ray/api/function/"; + String root = System.getProperty("user.dir") + "/api/src/main/java/io/ray/api/function/"; RayFuncGenerator generator = new RayFuncGenerator(); for (int i = 0; i <= MAX_PARAMETERS; i++) { // Functions that have return. String content = generator.generate(i, true); - FileUtils.write(new File(root + "RayFunc" + i + ".java"), content, - Charset.defaultCharset()); + FileUtils.write(new File(root + "RayFunc" + i + ".java"), content, Charset.defaultCharset()); // Functions that don't have return. content = generator.generate(i, false); - FileUtils.write(new File(root + "RayFuncVoid" + i + ".java"), content, - Charset.defaultCharset()); + FileUtils.write( + new File(root + "RayFuncVoid" + i + ".java"), content, Charset.defaultCharset()); } } - } diff --git a/java/runtime/src/test/java/io/ray/runtime/UniqueIdTest.java b/java/runtime/src/test/java/io/ray/runtime/UniqueIdTest.java index 7496f1baf..acde127f3 100644 --- a/java/runtime/src/test/java/io/ray/runtime/UniqueIdTest.java +++ b/java/runtime/src/test/java/io/ray/runtime/UniqueIdTest.java @@ -12,12 +12,14 @@ public class UniqueIdTest { @Test public void testConstructUniqueId() { // Test `fromHexString()` - UniqueId id1 = UniqueId.fromHexString("00000000123456789ABCDEF123456789ABCDEF0123456789ABCDEF00"); + UniqueId id1 = + UniqueId.fromHexString("00000000123456789ABCDEF123456789ABCDEF0123456789ABCDEF00"); Assert.assertEquals("00000000123456789abcdef123456789abcdef0123456789abcdef00", id1.toString()); Assert.assertFalse(id1.isNil()); try { - UniqueId id2 = UniqueId.fromHexString("000000123456789ABCDEF123456789ABCDEF0123456789ABCDEF00"); + UniqueId id2 = + UniqueId.fromHexString("000000123456789ABCDEF123456789ABCDEF0123456789ABCDEF00"); // This shouldn't be happened. Assert.assertTrue(false); } catch (IllegalArgumentException e) { @@ -33,16 +35,18 @@ public class UniqueIdTest { } // Test `fromByteBuffer()` - byte[] bytes = DatatypeConverter.parseHexBinary("0123456789ABCDEF0123456789ABCDEF012345670123456789ABCDEF"); + byte[] bytes = + DatatypeConverter.parseHexBinary( + "0123456789ABCDEF0123456789ABCDEF012345670123456789ABCDEF"); ByteBuffer byteBuffer = ByteBuffer.wrap(bytes, 0, 28); UniqueId id4 = UniqueId.fromByteBuffer(byteBuffer); Assert.assertTrue(Arrays.equals(bytes, id4.getBytes())); Assert.assertEquals("0123456789abcdef0123456789abcdef012345670123456789abcdef", id4.toString()); - // Test `genNil()` UniqueId id6 = UniqueId.NIL; - Assert.assertEquals("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF".toLowerCase(), id6.toString()); + Assert.assertEquals( + "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF".toLowerCase(), id6.toString()); Assert.assertTrue(id6.isNil()); } } diff --git a/java/runtime/src/test/java/io/ray/runtime/config/RayConfigTest.java b/java/runtime/src/test/java/io/ray/runtime/config/RayConfigTest.java index 6409bb037..a25e51c3e 100644 --- a/java/runtime/src/test/java/io/ray/runtime/config/RayConfigTest.java +++ b/java/runtime/src/test/java/io/ray/runtime/config/RayConfigTest.java @@ -30,8 +30,8 @@ public class RayConfigTest { } RayConfig rayConfig = RayConfig.create(); Assert.assertEquals(WorkerType.DRIVER, rayConfig.workerMode); - Assert.assertEquals(Collections.singletonList("path/to/ray/job/resource/path"), - rayConfig.codeSearchPath); + Assert.assertEquals( + Collections.singletonList("path/to/ray/job/resource/path"), rayConfig.codeSearchPath); Assert.assertEquals(rayConfig.rayletConfigParameters.get("one"), 1); Assert.assertEquals(rayConfig.rayletConfigParameters.get("zero"), 0); Assert.assertEquals(rayConfig.rayletConfigParameters.get("positive-integer"), 123); diff --git a/java/runtime/src/test/java/io/ray/runtime/functionmanager/FunctionManagerTest.java b/java/runtime/src/test/java/io/ray/runtime/functionmanager/FunctionManagerTest.java index c6270c997..6d2a228d9 100644 --- a/java/runtime/src/test/java/io/ray/runtime/functionmanager/FunctionManagerTest.java +++ b/java/runtime/src/test/java/io/ray/runtime/functionmanager/FunctionManagerTest.java @@ -18,9 +18,7 @@ import org.testng.Assert; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -/** - * Tests for {@link FunctionManager} - */ +/** Tests for {@link FunctionManager} */ public class FunctionManagerTest { public static Object foo() { @@ -36,7 +34,6 @@ public class FunctionManagerTest { public Object bar() { return null; } - } public interface ChildClassInterface { @@ -44,13 +41,11 @@ public class FunctionManagerTest { default String interfaceName() { return getClass().getName(); } - } public static class ChildClass extends ParentClass implements ChildClassInterface { - public ChildClass() { - } + public ChildClass() {} @Override public Object bar() { @@ -80,17 +75,20 @@ public class FunctionManagerTest { fooFunc = FunctionManagerTest::foo; childClassConstructor = ChildClass::new; childClassBarFunc = ChildClass::bar; - fooDescriptor = new JavaFunctionDescriptor(FunctionManagerTest.class.getName(), "foo", - "()Ljava/lang/Object;"); - childClassBarDescriptor = new JavaFunctionDescriptor(ChildClass.class.getName(), "bar", - "()Ljava/lang/Object;"); - childClassConstructorDescriptor = new JavaFunctionDescriptor(ChildClass.class.getName(), - FunctionManager.CONSTRUCTOR_NAME, - "()V"); - overloadFunctionDescriptorInt = new JavaFunctionDescriptor(FunctionManagerTest.class.getName(), - "overloadFunction", "(I)Ljava/lang/Object;"); - overloadFunctionDescriptorDouble = new JavaFunctionDescriptor(FunctionManagerTest.class.getName(), - "overloadFunction", "(D)Ljava/lang/Object;"); + fooDescriptor = + new JavaFunctionDescriptor( + FunctionManagerTest.class.getName(), "foo", "()Ljava/lang/Object;"); + childClassBarDescriptor = + new JavaFunctionDescriptor(ChildClass.class.getName(), "bar", "()Ljava/lang/Object;"); + childClassConstructorDescriptor = + new JavaFunctionDescriptor( + ChildClass.class.getName(), FunctionManager.CONSTRUCTOR_NAME, "()V"); + overloadFunctionDescriptorInt = + new JavaFunctionDescriptor( + FunctionManagerTest.class.getName(), "overloadFunction", "(I)Ljava/lang/Object;"); + overloadFunctionDescriptorDouble = + new JavaFunctionDescriptor( + FunctionManagerTest.class.getName(), "overloadFunction", "(D)Ljava/lang/Object;"); } @Test @@ -131,70 +129,87 @@ public class FunctionManagerTest { Assert.assertEquals(func.getFunctionDescriptor(), childClassConstructorDescriptor); // Test raise overload exception - Assert.expectThrows(RuntimeException.class, () -> { - functionManager.getFunction(JobId.NIL, - new JavaFunctionDescriptor(FunctionManagerTest.class.getName(), - "overloadFunction", "")); - }); + Assert.expectThrows( + RuntimeException.class, + () -> { + functionManager.getFunction( + JobId.NIL, + new JavaFunctionDescriptor( + FunctionManagerTest.class.getName(), "overloadFunction", "")); + }); } @Test public void testInheritance() { final FunctionManager functionManager = new FunctionManager(null); // Check inheritance can work and FunctionManager can find method in parent class. - fooDescriptor = new JavaFunctionDescriptor(ParentClass.class.getName(), "foo", - "()Ljava/lang/Object;"); - Assert.assertEquals(functionManager.getFunction(JobId.NIL, fooDescriptor) - .executable.getDeclaringClass(), ParentClass.class); - RayFunction fooFunc = functionManager.getFunction(JobId.NIL, - new JavaFunctionDescriptor(ChildClass.class.getName(), "foo", - "()Ljava/lang/Object;")); + fooDescriptor = + new JavaFunctionDescriptor(ParentClass.class.getName(), "foo", "()Ljava/lang/Object;"); + Assert.assertEquals( + functionManager.getFunction(JobId.NIL, fooDescriptor).executable.getDeclaringClass(), + ParentClass.class); + RayFunction fooFunc = + functionManager.getFunction( + JobId.NIL, + new JavaFunctionDescriptor(ChildClass.class.getName(), "foo", "()Ljava/lang/Object;")); Assert.assertEquals(fooFunc.executable.getDeclaringClass(), ParentClass.class); // Check FunctionManager can use method in child class if child class methods overrides methods // in parent class. - childClassBarDescriptor = new JavaFunctionDescriptor(ParentClass.class.getName(), "bar", - "()Ljava/lang/Object;"); - Assert.assertEquals(functionManager.getFunction(JobId.NIL, childClassBarDescriptor) - .executable.getDeclaringClass(), ParentClass.class); - RayFunction barFunc = functionManager.getFunction(JobId.NIL, - new JavaFunctionDescriptor(ChildClass.class.getName(), "bar", - "()Ljava/lang/Object;")); + childClassBarDescriptor = + new JavaFunctionDescriptor(ParentClass.class.getName(), "bar", "()Ljava/lang/Object;"); + Assert.assertEquals( + functionManager + .getFunction(JobId.NIL, childClassBarDescriptor) + .executable + .getDeclaringClass(), + ParentClass.class); + RayFunction barFunc = + functionManager.getFunction( + JobId.NIL, + new JavaFunctionDescriptor(ChildClass.class.getName(), "bar", "()Ljava/lang/Object;")); Assert.assertEquals(barFunc.executable.getDeclaringClass(), ChildClass.class); // Check interface default methods. - RayFunction interfaceNameFunc = functionManager.getFunction(JobId.NIL, - new JavaFunctionDescriptor(ChildClass.class.getName(), "interfaceName", - "()Ljava/lang/String;")); - Assert.assertEquals(interfaceNameFunc.executable.getDeclaringClass(), - ChildClassInterface.class); + RayFunction interfaceNameFunc = + functionManager.getFunction( + JobId.NIL, + new JavaFunctionDescriptor( + ChildClass.class.getName(), "interfaceName", "()Ljava/lang/String;")); + Assert.assertEquals( + interfaceNameFunc.executable.getDeclaringClass(), ChildClassInterface.class); } @Test public void testLoadFunctionTableForClass() { JobFunctionTable functionTable = new JobFunctionTable(getClass().getClassLoader()); - Map, RayFunction> res = functionTable - .loadFunctionsForClass(ChildClass.class.getName()); + Map, RayFunction> res = + functionTable.loadFunctionsForClass(ChildClass.class.getName()); // The result should be 4 entries: // 1, the constructor with signature // 2, the constructor without signature // 3, bar with signature // 4, bar without signature Assert.assertEquals(res.size(), 11); - Assert.assertTrue(res.containsKey( - ImmutablePair.of(childClassBarDescriptor.name, childClassBarDescriptor.signature))); - Assert.assertTrue(res.containsKey( - ImmutablePair.of(childClassConstructorDescriptor.name, childClassConstructorDescriptor.signature))); - Assert.assertTrue(res.containsKey( - ImmutablePair.of(childClassBarDescriptor.name, ""))); - Assert.assertTrue(res.containsKey( - ImmutablePair.of(childClassConstructorDescriptor.name, ""))); - Assert.assertTrue(res.containsKey( - ImmutablePair.of(overloadFunctionDescriptorInt.name, overloadFunctionDescriptorInt.signature))); - Assert.assertTrue(res.containsKey( - ImmutablePair.of(overloadFunctionDescriptorDouble.name, overloadFunctionDescriptorDouble.signature))); - Assert.assertTrue(res.containsKey( - ImmutablePair.of(overloadFunctionDescriptorInt.name, ""))); + Assert.assertTrue( + res.containsKey( + ImmutablePair.of(childClassBarDescriptor.name, childClassBarDescriptor.signature))); + Assert.assertTrue( + res.containsKey( + ImmutablePair.of( + childClassConstructorDescriptor.name, childClassConstructorDescriptor.signature))); + Assert.assertTrue(res.containsKey(ImmutablePair.of(childClassBarDescriptor.name, ""))); + Assert.assertTrue(res.containsKey(ImmutablePair.of(childClassConstructorDescriptor.name, ""))); + Assert.assertTrue( + res.containsKey( + ImmutablePair.of( + overloadFunctionDescriptorInt.name, overloadFunctionDescriptorInt.signature))); + Assert.assertTrue( + res.containsKey( + ImmutablePair.of( + overloadFunctionDescriptorDouble.name, + overloadFunctionDescriptorDouble.signature))); + Assert.assertTrue(res.containsKey(ImmutablePair.of(overloadFunctionDescriptorInt.name, ""))); Pair overloadKey = ImmutablePair.of(overloadFunctionDescriptorInt.name, ""); RayFunction func = res.get(overloadKey); // The function is overloaded. @@ -230,12 +245,11 @@ public class FunctionManagerTest { } // Test loading the function. - JavaFunctionDescriptor descriptor = new JavaFunctionDescriptor( - "DemoApp", "hello", "()Ljava/lang/String;"); - final FunctionManager functionManager = new FunctionManager( - Collections.singletonList(codeSearchPath)); + JavaFunctionDescriptor descriptor = + new JavaFunctionDescriptor("DemoApp", "hello", "()Ljava/lang/String;"); + final FunctionManager functionManager = + new FunctionManager(Collections.singletonList(codeSearchPath)); RayFunction func = functionManager.getFunction(jobId, descriptor); Assert.assertEquals(func.getFunctionDescriptor(), descriptor); } - } diff --git a/java/runtime/src/test/java/io/ray/runtime/serializer/SerializerTest.java b/java/runtime/src/test/java/io/ray/runtime/serializer/SerializerTest.java index 7cdb67516..74baf103a 100644 --- a/java/runtime/src/test/java/io/ray/runtime/serializer/SerializerTest.java +++ b/java/runtime/src/test/java/io/ray/runtime/serializer/SerializerTest.java @@ -12,8 +12,8 @@ public class SerializerTest { public void testBasicSerialization() { // Test serialize / deserialize primitive types with type conversion. { - Object[] foo = new Object[]{"hello", (byte) 1, 2.0, (short) 3, 4, 5L, - new String[]{"hello", "world"}}; + Object[] foo = + new Object[] {"hello", (byte) 1, 2.0, (short) 3, 4, 5L, new String[] {"hello", "world"}}; Pair serialized = Serializer.encode(foo); Object[] bar = Serializer.decode(serialized.getLeft(), Object[].class); Assert.assertTrue(serialized.getRight()); @@ -26,10 +26,12 @@ public class SerializerTest { } // Test multidimensional array. { - Object[][] foo = new Object[][]{{1, 2}, {"3", 4}}; - Assert.expectThrows(RuntimeException.class, () -> { - Object[][] bar = Serializer.decode(Serializer.encode(foo).getLeft(), Integer[][].class); - }); + Object[][] foo = new Object[][] {{1, 2}, {"3", 4}}; + Assert.expectThrows( + RuntimeException.class, + () -> { + Object[][] bar = Serializer.decode(Serializer.encode(foo).getLeft(), Integer[][].class); + }); Pair serialized = Serializer.encode(foo); Object[][] bar = Serializer.decode(serialized.getLeft(), Object[][].class); Assert.assertTrue(serialized.getRight()); diff --git a/java/test.sh b/java/test.sh index 3352f5b33..8336c1da1 100755 --- a/java/test.sh +++ b/java/test.sh @@ -8,6 +8,14 @@ set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) java -version +pushd "$ROOT_DIR" + echo "Check java code format." + # check google java style + mvn -T16 spotless:check + # check naming and others + mvn -T16 checkstyle:check +popd + run_testng() { local exit_code if "$@"; then @@ -27,11 +35,6 @@ run_testng() { } pushd "$ROOT_DIR"/.. -echo "Linting Java code with checkstyle." -# NOTE(hchen): The `test_tag_filters` option causes bazel to ignore caches. -# Thus, we add the `build_tests_only` option to avoid re-building everything. -bazel test //java:all --test_tag_filters="checkstyle" --build_tests_only - echo "Build java maven deps." bazel build //java:gen_maven_deps @@ -70,13 +73,11 @@ for file in "$docdemo_path"*.java; do echo "Running $class" java -cp bazel-bin/java/all_tests_deploy.jar "io.ray.docdemo.$class" done - popd - pushd "$ROOT_DIR" echo "Testing maven install." -mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN clean install -DskipTests +mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN clean install -DskipTests -Dcheckstyle.skip # Ensure mvn test works mvn test -pl test -Dtest="io.ray.test.HelloWorldTest" popd diff --git a/java/test/pom.xml b/java/test/pom.xml index f401f3cff..c9e34821b 100644 --- a/java/test/pom.xml +++ b/java/test/pom.xml @@ -117,6 +117,41 @@ + + + com.diffplug.spotless + spotless-maven-plugin + 2.6.1 + + + + + + + + + .java + + + + + + + + true + 4 + + + + + + + 1.7 + + + + + diff --git a/java/test/src/main/java/io/ray/benchmark/ActorPressTest.java b/java/test/src/main/java/io/ray/benchmark/ActorPressTest.java index 3ffd1b365..ae7c21de3 100644 --- a/java/test/src/main/java/io/ray/benchmark/ActorPressTest.java +++ b/java/test/src/main/java/io/ray/benchmark/ActorPressTest.java @@ -63,5 +63,4 @@ public class ActorPressTest extends RayBenchmarkTest { return remoteResult; } } - } diff --git a/java/test/src/main/java/io/ray/benchmark/MaxPressureTest.java b/java/test/src/main/java/io/ray/benchmark/MaxPressureTest.java index 35f4ada42..ca7b4f9ca 100644 --- a/java/test/src/main/java/io/ray/benchmark/MaxPressureTest.java +++ b/java/test/src/main/java/io/ray/benchmark/MaxPressureTest.java @@ -37,5 +37,4 @@ public class MaxPressureTest extends RayBenchmarkTest { public boolean checkResult(Object o) { return (int) o == 0; } - } diff --git a/java/test/src/main/java/io/ray/benchmark/MicroBenchmarks.java b/java/test/src/main/java/io/ray/benchmark/MicroBenchmarks.java index 62c0ea534..aeb8d4fd2 100644 --- a/java/test/src/main/java/io/ray/benchmark/MicroBenchmarks.java +++ b/java/test/src/main/java/io/ray/benchmark/MicroBenchmarks.java @@ -20,24 +20,30 @@ public class MicroBenchmarks { } final long duration = System.nanoTime() - start; LOGGER.info( - "Benchmark \"{}\" finished, repeated {} times, total duration {} ms," + - " average duration {} ns.", - name, numRepeats, duration / 1_000_000, duration / numRepeats); + "Benchmark \"{}\" finished, repeated {} times, total duration {} ms," + + " average duration {} ns.", + name, + numRepeats, + duration / 1_000_000, + duration / numRepeats); } /** * Benchmark task submission. * - * Note, this benchmark is supposed to measure the elapased time in Java worker, we should disable - * submitting tasks to raylet in `raylet_client.cc` before running this benchmark. + *

Note, this benchmark is supposed to measure the elapased time in Java worker, we should + * disable submitting tasks to raylet in `raylet_client.cc` before running this benchmark. */ public static void benchmarkTaskSubmission() { final int numRepeats = 1_000_000; Ray.init(); try { - time(() -> { - Ray.task(MicroBenchmarks::simpleFunction).remote(); - }, numRepeats, "task submission"); + time( + () -> { + Ray.task(MicroBenchmarks::simpleFunction).remote(); + }, + numRepeats, + "task submission"); } finally { Ray.shutdown(); } diff --git a/java/test/src/main/java/io/ray/benchmark/PressureTestParameter.java b/java/test/src/main/java/io/ray/benchmark/PressureTestParameter.java index 6d14765cc..6548dabdc 100644 --- a/java/test/src/main/java/io/ray/benchmark/PressureTestParameter.java +++ b/java/test/src/main/java/io/ray/benchmark/PressureTestParameter.java @@ -7,17 +7,17 @@ public class PressureTestParameter implements Serializable { private static final long serialVersionUID = -52054601722982473L; - private Integer clientNum = 1; //number of test client + private Integer clientNum = 1; // number of test client - private PressureTestType pressureTestType = PressureTestType.RATE_LIMITER; //pressure test type + private PressureTestType pressureTestType = PressureTestType.RATE_LIMITER; // pressure test type - private Integer totalNum = 1; //total number of task under the mode of MAX + private Integer totalNum = 1; // total number of task under the mode of MAX - private Integer totalQps = 1; //total qps of task under the mode of RATE_LIMITER + private Integer totalQps = 1; // total qps of task under the mode of RATE_LIMITER - private Integer duration = 1; //duration of the pressure test under the mode of RATE_LIMITER + private Integer duration = 1; // duration of the pressure test under the mode of RATE_LIMITER - private RayBenchmarkTest rayBenchmarkTest; //reference of current test case instance + private RayBenchmarkTest rayBenchmarkTest; // reference of current test case instance // reference of the Actor, if only test remote funtion it could be null private ActorHandle rayActor; diff --git a/java/test/src/main/java/io/ray/benchmark/PressureTestType.java b/java/test/src/main/java/io/ray/benchmark/PressureTestType.java index 30b282250..e4cb2e493 100644 --- a/java/test/src/main/java/io/ray/benchmark/PressureTestType.java +++ b/java/test/src/main/java/io/ray/benchmark/PressureTestType.java @@ -1,7 +1,6 @@ package io.ray.benchmark; public enum PressureTestType { - SINGLE_LATENCY, RATE_LIMITER, MAX diff --git a/java/test/src/main/java/io/ray/benchmark/RayBenchmarkTest.java b/java/test/src/main/java/io/ray/benchmark/RayBenchmarkTest.java index 3c93f77b1..be24a8ab1 100644 --- a/java/test/src/main/java/io/ray/benchmark/RayBenchmarkTest.java +++ b/java/test/src/main/java/io/ray/benchmark/RayBenchmarkTest.java @@ -18,7 +18,7 @@ import org.testng.Assert; public abstract class RayBenchmarkTest extends BaseTest implements Serializable { private static final Logger LOGGER = LoggerFactory.getLogger(RayBenchmarkTest.class); - //not thread safe ,but we only have one thread here + // not thread safe ,but we only have one thread here public static final DecimalFormat df = new DecimalFormat("00.00"); private static final long serialVersionUID = 416045641835782523L; @@ -29,7 +29,7 @@ public abstract class RayBenchmarkTest extends BaseTest implements Serializab PressureTestType pressureTestType = pressureTestParameter.getPressureTestType(); RayBenchmarkTest rayBenchmarkTest = pressureTestParameter.getRayBenchmarkTest(); int clientNum = pressureTestParameter.getClientNum(); - //SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + // SimpleDateFormat formatter = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); int len; String logPrefix; RateLimiter rateLimiter = null; @@ -70,7 +70,6 @@ public abstract class RayBenchmarkTest extends BaseTest implements Serializab } catch (Exception e) { LOGGER.error("singleClient", e); return null; - } } @@ -151,5 +150,4 @@ public abstract class RayBenchmarkTest extends BaseTest implements Serializab pressureTestParameter.setPressureTestType(PressureTestType.MAX); notSinglePressTest(pressureTestParameter); } - } diff --git a/java/test/src/main/java/io/ray/docdemo/RayDemo.java b/java/test/src/main/java/io/ray/docdemo/RayDemo.java index 90f76694e..b16b02d87 100644 --- a/java/test/src/main/java/io/ray/docdemo/RayDemo.java +++ b/java/test/src/main/java/io/ray/docdemo/RayDemo.java @@ -9,9 +9,10 @@ import java.util.stream.Collectors; /** * This class contains demo code of the Ray introduction doc - * (https://docs.ray.io/en/master/index.html and https://docs.ray.io/en/master/ray-overview/index.html). + * (https://docs.ray.io/en/master/index.html and + * https://docs.ray.io/en/master/ray-overview/index.html). * - * Please keep them in sync. + *

Please keep them in sync. */ public class RayDemo { @@ -43,7 +44,7 @@ public class RayDemo { objectRefList.add(Ray.task(RayDemo::square, i).remote()); } // Get the actual results of the tasks with `get`. - System.out.println(Ray.get(objectRefList)); // [0, 1, 4, 9] + System.out.println(Ray.get(objectRefList)); // [0, 1, 4, 9] } { @@ -60,10 +61,11 @@ public class RayDemo { counter.task(Counter::increment).remote(); } // Invoke the `read` method on each actor, and print the results. - List> objectRefList = counters.stream() - .map(counter -> counter.task(Counter::read).remote()) - .collect(Collectors.toList()); - System.out.println(Ray.get(objectRefList)); // [1, 1, 1, 1] + List> objectRefList = + counters.stream() + .map(counter -> counter.task(Counter::read).remote()) + .collect(Collectors.toList()); + System.out.println(Ray.get(objectRefList)); // [1, 1, 1, 1] } } } diff --git a/java/test/src/main/java/io/ray/docdemo/UsingActorsDemo.java b/java/test/src/main/java/io/ray/docdemo/UsingActorsDemo.java index 8d86d8c7e..82ee26cda 100644 --- a/java/test/src/main/java/io/ray/docdemo/UsingActorsDemo.java +++ b/java/test/src/main/java/io/ray/docdemo/UsingActorsDemo.java @@ -2,15 +2,15 @@ package io.ray.docdemo; import io.ray.api.ActorHandle; import io.ray.api.Ray; -import io.ray.docdemo.WalkthroughDemo.Counter; import java.util.Optional; import java.util.concurrent.TimeUnit; import org.testng.Assert; /** - * This class contains demo code of the Ray core Using Actors doc (https://docs.ray.io/en/master/actors.html). + * This class contains demo code of the Ray core Using Actors doc + * (https://docs.ray.io/en/master/actors.html). * - * Please keep them in sync. + *

Please keep them in sync. */ public class UsingActorsDemo { @@ -40,9 +40,7 @@ public class UsingActorsDemo { } } - public static class GpuActor { - - } + public static class GpuActor {} public static class MyRayApp { @@ -82,12 +80,12 @@ public class UsingActorsDemo { } { - ActorHandle a1 = Ray.actor(Counter::new).setResource("CPU", 1.0) - .setResource("Custom1", 1.0).remote(); - ActorHandle a2 = Ray.actor(Counter::new).setResource("CPU", 2.0) - .setResource("Custom2", 1.0).remote(); - ActorHandle a3 = Ray.actor(Counter::new).setResource("CPU", 3.0) - .setResource("Custom3", 1.0).remote(); + ActorHandle a1 = + Ray.actor(Counter::new).setResource("CPU", 1.0).setResource("Custom1", 1.0).remote(); + ActorHandle a2 = + Ray.actor(Counter::new).setResource("CPU", 2.0).setResource("Custom2", 1.0).remote(); + ActorHandle a3 = + Ray.actor(Counter::new).setResource("CPU", 3.0).setResource("Custom3", 1.0).remote(); } { diff --git a/java/test/src/main/java/io/ray/docdemo/WalkthroughDemo.java b/java/test/src/main/java/io/ray/docdemo/WalkthroughDemo.java index da6e21e54..002c8b38a 100644 --- a/java/test/src/main/java/io/ray/docdemo/WalkthroughDemo.java +++ b/java/test/src/main/java/io/ray/docdemo/WalkthroughDemo.java @@ -11,9 +11,10 @@ import java.util.concurrent.TimeUnit; import org.testng.Assert; /** - * This class contains demo code of the Ray core walkthrough doc (https://docs.ray.io/en/master/walkthrough.html). + * This class contains demo code of the Ray core walkthrough doc + * (https://docs.ray.io/en/master/walkthrough.html). * - * Please keep them in sync. + *

Please keep them in sync. */ public class WalkthroughDemo { @@ -79,9 +80,9 @@ public class WalkthroughDemo { List results = Ray.get(objectRefs); Assert.assertEquals(results, ImmutableList.of(0, 1, 2)); - WaitResult waitResult = Ray.wait(objectRefs, /*num_returns=*/1, /*timeoutMs=*/1000); - System.out.println(waitResult.getReady()); // List of ready objects. - System.out.println(waitResult.getUnready()); // list of unready objects. + WaitResult waitResult = Ray.wait(objectRefs, /*num_returns=*/ 1, /*timeoutMs=*/ 1000); + System.out.println(waitResult.getReady()); // List of ready objects. + System.out.println(waitResult.getUnready()); // list of unready objects. } // A regular Java class. diff --git a/java/test/src/main/java/io/ray/test/ActorConcurrentCallTest.java b/java/test/src/main/java/io/ray/test/ActorConcurrentCallTest.java index 3eac9fcdc..f3c812617 100644 --- a/java/test/src/main/java/io/ray/test/ActorConcurrentCallTest.java +++ b/java/test/src/main/java/io/ray/test/ActorConcurrentCallTest.java @@ -28,9 +28,7 @@ public class ActorConcurrentCallTest extends BaseTest { public void testConcurrentCall() { ActorHandle actor = - Ray.actor(ConcurrentActor::new) - .setMaxConcurrency(3) - .remote(); + Ray.actor(ConcurrentActor::new).setMaxConcurrency(3).remote(); ObjectRef obj1 = actor.task(ConcurrentActor::countDown).remote(); ObjectRef obj2 = actor.task(ConcurrentActor::countDown).remote(); ObjectRef obj3 = actor.task(ConcurrentActor::countDown).remote(); @@ -40,5 +38,4 @@ public class ActorConcurrentCallTest extends BaseTest { Assert.assertEquals(obj2.get(), "ok"); Assert.assertEquals(obj3.get(), "ok"); } - } diff --git a/java/test/src/main/java/io/ray/test/ActorRestartTest.java b/java/test/src/main/java/io/ray/test/ActorRestartTest.java index f24bb5e34..577487e6e 100644 --- a/java/test/src/main/java/io/ray/test/ActorRestartTest.java +++ b/java/test/src/main/java/io/ray/test/ActorRestartTest.java @@ -70,4 +70,3 @@ public class ActorRestartTest extends BaseTest { } } } - diff --git a/java/test/src/main/java/io/ray/test/ActorTest.java b/java/test/src/main/java/io/ray/test/ActorTest.java index 78b8f2468..838d68ee5 100644 --- a/java/test/src/main/java/io/ray/test/ActorTest.java +++ b/java/test/src/main/java/io/ray/test/ActorTest.java @@ -57,14 +57,13 @@ public class ActorTest extends BaseTest { // Test calling an actor Assert.assertEquals(Integer.valueOf(1), actor.task(Counter::getValue).remote().get()); actor.task(Counter::increase, 1).remote(); - Assert.assertEquals(Integer.valueOf(3), - actor.task(Counter::increaseAndGet, 1).remote().get()); + Assert.assertEquals(Integer.valueOf(3), actor.task(Counter::increaseAndGet, 1).remote().get()); } /** * Test getting an object twice from the local memory store. * - * Objects are stored in core worker's local memory. And it will be removed after the first + *

Objects are stored in core worker's local memory. And it will be removed after the first * get. To enable getting it twice, we cache the object in `RayObjectImpl`. */ public void testGetObjectTwice() { @@ -78,7 +77,8 @@ public class ActorTest extends BaseTest { public void testCallActorWithLargeObject() { ActorHandle actor = Ray.actor(Counter::new, 1).remote(); TestUtils.LargeObject largeObject = new TestUtils.LargeObject(); - Assert.assertEquals(Integer.valueOf(largeObject.data.length + 1), + Assert.assertEquals( + Integer.valueOf(largeObject.data.length + 1), actor.task(Counter::accessLargeObject, largeObject).remote().get()); } @@ -111,13 +111,17 @@ public class ActorTest extends BaseTest { public void testPassActorAsParameter() { ActorHandle actor = Ray.actor(Counter::new, 0).remote(); - Assert.assertEquals(Integer.valueOf(1), + Assert.assertEquals( + Integer.valueOf(1), Ray.task(ActorTest::testActorAsFirstParameter, actor, 1).remote().get()); - Assert.assertEquals(Integer.valueOf(11), + Assert.assertEquals( + Integer.valueOf(11), Ray.task(ActorTest::testActorAsSecondParameter, 10, actor).remote().get()); - Assert.assertEquals(Integer.valueOf(111), - Ray.task(ActorTest::testActorAsFieldOfParameter, - Collections.singletonList(actor), 100).remote().get()); + Assert.assertEquals( + Integer.valueOf(111), + Ray.task(ActorTest::testActorAsFieldOfParameter, Collections.singletonList(actor), 100) + .remote() + .get()); } // This test case follows `test_internal_free` in `python/ray/tests/test_advanced.py`. @@ -150,7 +154,6 @@ public class ActorTest extends BaseTest { default String interfaceName() { return ChildClassInterface.class.getName(); } - } public static class ChildClass extends Counter implements ChildClassInterface { @@ -163,7 +166,6 @@ public class ActorTest extends BaseTest { public void increase(int delta) { super.increase(-delta); } - } @Test(groups = {"cluster"}) @@ -176,8 +178,8 @@ public class ActorTest extends BaseTest { counter.task(Counter::increase, 10).remote(); Assert.assertEquals(counter.task(Counter::getValue).remote().get(), Integer.valueOf(80)); // test interface default methods - Assert.assertEquals(counter.task(ChildClassInterface::interfaceName).remote().get(), + Assert.assertEquals( + counter.task(ChildClassInterface::interfaceName).remote().get(), ChildClassInterface.class.getName()); } - } diff --git a/java/test/src/main/java/io/ray/test/AnnotationTransformer.java b/java/test/src/main/java/io/ray/test/AnnotationTransformer.java index 57e5b35e9..503383e09 100644 --- a/java/test/src/main/java/io/ray/test/AnnotationTransformer.java +++ b/java/test/src/main/java/io/ray/test/AnnotationTransformer.java @@ -8,8 +8,8 @@ import org.testng.annotations.ITestAnnotation; public class AnnotationTransformer implements IAnnotationTransformer { @Override - public void transform(ITestAnnotation annotation, Class testClass, - Constructor testConstructor, Method testMethod) { + public void transform( + ITestAnnotation annotation, Class testClass, Constructor testConstructor, Method testMethod) { annotation.setRetryAnalyzer(RetryAnalyzer.class); } } diff --git a/java/test/src/main/java/io/ray/test/BaseTaskOptionsTest.java b/java/test/src/main/java/io/ray/test/BaseTaskOptionsTest.java index 481ffd25f..cf777da8b 100644 --- a/java/test/src/main/java/io/ray/test/BaseTaskOptionsTest.java +++ b/java/test/src/main/java/io/ray/test/BaseTaskOptionsTest.java @@ -16,8 +16,8 @@ public class BaseTaskOptionsTest { @Test public void testLegalResources() { - Map resources = ImmutableMap.of( - "CPU", 0.5, "GPU", 3.0, "memory", 1024.0, "A", 4294967296.0); + Map resources = + ImmutableMap.of("CPU", 0.5, "GPU", 3.0, "memory", 1024.0, "A", 4294967296.0); new MockActorCreationOptions(resources); } @@ -45,5 +45,4 @@ public class BaseTaskOptionsTest { Map resources = ImmutableMap.of("CPU", 3.5); new MockActorCreationOptions(resources); } - -} \ No newline at end of file +} diff --git a/java/test/src/main/java/io/ray/test/BaseTest.java b/java/test/src/main/java/io/ray/test/BaseTest.java index 68698dbf9..e75d3ad26 100644 --- a/java/test/src/main/java/io/ray/test/BaseTest.java +++ b/java/test/src/main/java/io/ray/test/BaseTest.java @@ -18,5 +18,4 @@ public class BaseTest { public void tearDownBase() { Ray.shutdown(); } - } diff --git a/java/test/src/main/java/io/ray/test/ClassLoaderTest.java b/java/test/src/main/java/io/ray/test/ClassLoaderTest.java index 631422d79..50c726aba 100644 --- a/java/test/src/main/java/io/ray/test/ClassLoaderTest.java +++ b/java/test/src/main/java/io/ray/test/ClassLoaderTest.java @@ -22,8 +22,8 @@ import org.testng.annotations.Test; public class ClassLoaderTest extends BaseTest { - private final String codeSearchPath = FileUtils.getTempDirectoryPath() - + "/ray_test/ClassLoaderTest"; + private final String codeSearchPath = + FileUtils.getTempDirectoryPath() + "/ray_test/ClassLoaderTest"; @BeforeClass public void setUp() { @@ -47,40 +47,41 @@ public class ClassLoaderTest extends BaseTest { // In this test case the class is expected to be loaded from the job code search path, // so we need to put the compiled class file into the job code search path and load it // later. - String testJavaFile = "" - + "import java.lang.management.ManagementFactory;\n" - + "import java.lang.management.RuntimeMXBean;\n" - + "\n" - + "public class ClassLoaderTester {\n" - + "\n" - + " static volatile int value;\n" - + "\n" - + " public int getPid() {\n" - + " RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();\n" - + " String name = runtime.getName();\n" - + " int index = name.indexOf(\"@\");\n" - + " if (index != -1) {\n" - + " return Integer.parseInt(name.substring(0, index));\n" - + " } else {\n" - + " throw new RuntimeException(\"parse pid error:\" + name);\n" - + " }\n" - + " }\n" - + "\n" - + " public int increase() throws InterruptedException {\n" - + " return increaseInternal();\n" - + " }\n" - + "\n" - + " public static synchronized int increaseInternal() throws InterruptedException {\n" - + " int oldValue = value;\n" - + " Thread.sleep(10 * 1000);\n" - + " value = oldValue + 1;\n" - + " return value;\n" - + " }\n" - + "\n" - + " public int getClassLoaderHashCode() {\n" - + " return this.getClass().getClassLoader().hashCode();\n" - + " }\n" - + "}"; + String testJavaFile = + "" + + "import java.lang.management.ManagementFactory;\n" + + "import java.lang.management.RuntimeMXBean;\n" + + "\n" + + "public class ClassLoaderTester {\n" + + "\n" + + " static volatile int value;\n" + + "\n" + + " public int getPid() {\n" + + " RuntimeMXBean runtime = ManagementFactory.getRuntimeMXBean();\n" + + " String name = runtime.getName();\n" + + " int index = name.indexOf(\"@\");\n" + + " if (index != -1) {\n" + + " return Integer.parseInt(name.substring(0, index));\n" + + " } else {\n" + + " throw new RuntimeException(\"parse pid error:\" + name);\n" + + " }\n" + + " }\n" + + "\n" + + " public int increase() throws InterruptedException {\n" + + " return increaseInternal();\n" + + " }\n" + + "\n" + + " public static synchronized int increaseInternal() throws InterruptedException {\n" + + " int oldValue = value;\n" + + " Thread.sleep(10 * 1000);\n" + + " value = oldValue + 1;\n" + + " return value;\n" + + " }\n" + + "\n" + + " public int getClassLoaderHashCode() {\n" + + " return this.getClass().getClassLoader().hashCode();\n" + + " }\n" + + "}"; // Write the demo java file to the job code search path. String javaFilePath = codeSearchPath + "/ClassLoaderTester.java"; @@ -88,54 +89,60 @@ public class ClassLoaderTest extends BaseTest { // Compile the java file. JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - int result = compiler.run(null, null, null, "-d", - codeSearchPath, javaFilePath); + int result = compiler.run(null, null, null, "-d", codeSearchPath, javaFilePath); if (result != 0) { throw new RuntimeException("Couldn't compile ClassLoaderTester.java."); } - FunctionDescriptor constructor = new JavaFunctionDescriptor( - "ClassLoaderTester", "", "()V"); + FunctionDescriptor constructor = + new JavaFunctionDescriptor("ClassLoaderTester", "", "()V"); ActorHandle actor1 = createActor(constructor); - FunctionDescriptor getPid = new JavaFunctionDescriptor( - "ClassLoaderTester", "getPid", "()I"); - int pid = this.callActorFunction(actor1, getPid, new Object[0], - Optional.of(Integer.class)).get(); + FunctionDescriptor getPid = new JavaFunctionDescriptor("ClassLoaderTester", "getPid", "()I"); + int pid = + this.callActorFunction(actor1, getPid, new Object[0], Optional.of(Integer.class)) + .get(); ActorHandle actor2; while (true) { // Create another actor which share the same process of actor 1. actor2 = createActor(constructor); - int actor2Pid = this.callActorFunction(actor2, getPid, new Object[0], - Optional.of(Integer.class)).get(); + int actor2Pid = + this.callActorFunction(actor2, getPid, new Object[0], Optional.of(Integer.class)) + .get(); if (actor2Pid == pid) { break; } } - FunctionDescriptor getClassLoaderHashCode = new JavaFunctionDescriptor( - "ClassLoaderTester", "getClassLoaderHashCode", "()I"); - ObjectRef hashCode1 = callActorFunction(actor1, getClassLoaderHashCode, new Object[0], - Optional.of(Integer.class)); - ObjectRef hashCode2 = callActorFunction(actor2, getClassLoaderHashCode, new Object[0], - Optional.of(Integer.class)); + FunctionDescriptor getClassLoaderHashCode = + new JavaFunctionDescriptor("ClassLoaderTester", "getClassLoaderHashCode", "()I"); + ObjectRef hashCode1 = + callActorFunction( + actor1, getClassLoaderHashCode, new Object[0], Optional.of(Integer.class)); + ObjectRef hashCode2 = + callActorFunction( + actor2, getClassLoaderHashCode, new Object[0], Optional.of(Integer.class)); Assert.assertEquals(hashCode1.get(), hashCode2.get()); - FunctionDescriptor increase = new JavaFunctionDescriptor( - "ClassLoaderTester", "increase", "()I"); - ObjectRef value1 = callActorFunction(actor1, increase, new Object[0], - Optional.of(Integer.class)); - ObjectRef value2 = callActorFunction(actor2, increase, new Object[0], - Optional.of(Integer.class)); + FunctionDescriptor increase = + new JavaFunctionDescriptor("ClassLoaderTester", "increase", "()I"); + ObjectRef value1 = + callActorFunction(actor1, increase, new Object[0], Optional.of(Integer.class)); + ObjectRef value2 = + callActorFunction(actor2, increase, new Object[0], Optional.of(Integer.class)); Assert.assertNotEquals(value1.get(), value2.get()); } - private ActorHandle createActor(FunctionDescriptor functionDescriptor) - throws Exception { - Method createActorMethod = AbstractRayRuntime.class.getDeclaredMethod("createActorImpl", - FunctionDescriptor.class, Object[].class, ActorCreationOptions.class); + private ActorHandle createActor(FunctionDescriptor functionDescriptor) throws Exception { + Method createActorMethod = + AbstractRayRuntime.class.getDeclaredMethod( + "createActorImpl", + FunctionDescriptor.class, + Object[].class, + ActorCreationOptions.class); createActorMethod.setAccessible(true); - return (ActorHandle) createActorMethod - .invoke(TestUtils.getUnderlyingRuntime(), functionDescriptor, new Object[0], null); + return (ActorHandle) + createActorMethod.invoke( + TestUtils.getUnderlyingRuntime(), functionDescriptor, new Object[0], null); } private ObjectRef callActorFunction( @@ -144,10 +151,16 @@ public class ClassLoaderTest extends BaseTest { Object[] args, Optional> returnType) throws Exception { - Method callActorFunctionMethod = AbstractRayRuntime.class.getDeclaredMethod("callActorFunction", - BaseActorHandle.class, FunctionDescriptor.class, Object[].class, Optional.class); + Method callActorFunctionMethod = + AbstractRayRuntime.class.getDeclaredMethod( + "callActorFunction", + BaseActorHandle.class, + FunctionDescriptor.class, + Object[].class, + Optional.class); callActorFunctionMethod.setAccessible(true); - return (ObjectRef) callActorFunctionMethod - .invoke(TestUtils.getUnderlyingRuntime(), rayActor, functionDescriptor, args, returnType); + return (ObjectRef) + callActorFunctionMethod.invoke( + TestUtils.getUnderlyingRuntime(), rayActor, functionDescriptor, args, returnType); } } diff --git a/java/test/src/main/java/io/ray/test/CrossLanguageInvocationTest.java b/java/test/src/main/java/io/ray/test/CrossLanguageInvocationTest.java index 5db99230f..ab275b95e 100644 --- a/java/test/src/main/java/io/ray/test/CrossLanguageInvocationTest.java +++ b/java/test/src/main/java/io/ray/test/CrossLanguageInvocationTest.java @@ -32,26 +32,25 @@ public class CrossLanguageInvocationTest extends BaseTest { @BeforeClass public void beforeClass() { // Delete and re-create the temp dir. - File tempDir = new File( - System.getProperty("java.io.tmpdir") + File.separator + "ray_cross_language_test"); + File tempDir = + new File(System.getProperty("java.io.tmpdir") + File.separator + "ray_cross_language_test"); FileUtils.deleteQuietly(tempDir); tempDir.mkdirs(); tempDir.deleteOnExit(); // Write the test Python file to the temp dir. - InputStream in = CrossLanguageInvocationTest.class - .getResourceAsStream("/" + PYTHON_MODULE + ".py"); - File pythonFile = new File( - tempDir.getAbsolutePath() + File.separator + PYTHON_MODULE + ".py"); + InputStream in = + CrossLanguageInvocationTest.class.getResourceAsStream("/" + PYTHON_MODULE + ".py"); + File pythonFile = new File(tempDir.getAbsolutePath() + File.separator + PYTHON_MODULE + ".py"); try { FileUtils.copyInputStreamToFile(in, pythonFile); } catch (IOException e) { throw new RuntimeException(e); } - System.setProperty("ray.job.code-search-path", - System.getProperty("java.class.path") + File.pathSeparator - + tempDir.getAbsolutePath()); + System.setProperty( + "ray.job.code-search-path", + System.getProperty("java.class.path") + File.pathSeparator + tempDir.getAbsolutePath()); } @AfterClass @@ -61,48 +60,49 @@ public class CrossLanguageInvocationTest extends BaseTest { @Test public void testCallingPythonFunction() { - Object[] inputs = new Object[]{ - true, // Boolean - Byte.MAX_VALUE, // Byte - Short.MAX_VALUE, // Short - Integer.MAX_VALUE, // Integer - Long.MAX_VALUE, // Long - // BigInteger can support max value of 2^64-1, please refer to: - // https://github.com/msgpack/msgpack/blob/master/spec.md#int-format-family - // If BigInteger larger than 2^64-1, the value can only be transferred among Java workers. - BigInteger.valueOf(Long.MAX_VALUE), // BigInteger - "Hello World!", // String - 1.234f, // Float - 1.234, // Double - "example binary".getBytes()}; // byte[] + Object[] inputs = + new Object[] { + true, // Boolean + Byte.MAX_VALUE, // Byte + Short.MAX_VALUE, // Short + Integer.MAX_VALUE, // Integer + Long.MAX_VALUE, // Long + // BigInteger can support max value of 2^64-1, please refer to: + // https://github.com/msgpack/msgpack/blob/master/spec.md#int-format-family + // If BigInteger larger than 2^64-1, the value can only be transferred among Java workers. + BigInteger.valueOf(Long.MAX_VALUE), // BigInteger + "Hello World!", // String + 1.234f, // Float + 1.234, // Double + "example binary".getBytes() + }; // byte[] for (Object o : inputs) { - ObjectRef res = Ray.task( - PyFunction.of(PYTHON_MODULE, "py_return_input", o.getClass()), - o).remote(); + ObjectRef res = + Ray.task(PyFunction.of(PYTHON_MODULE, "py_return_input", o.getClass()), o).remote(); Assert.assertEquals(res.get(), o); } // null { Object input = null; - ObjectRef res = Ray.task( - PyFunction.of(PYTHON_MODULE, "py_return_input", Object.class), input).remote(); + ObjectRef res = + Ray.task(PyFunction.of(PYTHON_MODULE, "py_return_input", Object.class), input).remote(); Object r = res.get(); Assert.assertEquals(r, input); } // array { - int[] input = new int[]{1, 2}; - ObjectRef res = Ray.task( - PyFunction.of(PYTHON_MODULE, "py_return_input", int[].class), input).remote(); + int[] input = new int[] {1, 2}; + ObjectRef res = + Ray.task(PyFunction.of(PYTHON_MODULE, "py_return_input", int[].class), input).remote(); int[] r = res.get(); Assert.assertEquals(r, input); } // array of Object { - Object[] input = new Object[]{1, 2.3f, 4.56, "789", "10".getBytes(), null, true, - new int[]{1, 2}}; - ObjectRef res = Ray.task( - PyFunction.of(PYTHON_MODULE, "py_return_input", Object[].class), input).remote(); + Object[] input = + new Object[] {1, 2.3f, 4.56, "789", "10".getBytes(), null, true, new int[] {1, 2}}; + ObjectRef res = + Ray.task(PyFunction.of(PYTHON_MODULE, "py_return_input", Object[].class), input).remote(); Object[] r = res.get(); // If we tell the value type is Object, then all numbers will be Number type. Assert.assertEquals(((Number) r[0]).intValue(), input[0]); @@ -124,39 +124,47 @@ public class CrossLanguageInvocationTest extends BaseTest { } // Unsupported types, all Java specific types, e.g. List / Map... { - Assert.expectThrows(Exception.class, () -> { - List input = Arrays.asList(1, 2); - ObjectRef> res = Ray.task( - PyFunction.of(PYTHON_MODULE, "py_return_input", - (Class>) input.getClass()), input).remote(); - List r = res.get(); - Assert.assertEquals(r, input); - }); + Assert.expectThrows( + Exception.class, + () -> { + List input = Arrays.asList(1, 2); + ObjectRef> res = + Ray.task( + PyFunction.of( + PYTHON_MODULE, + "py_return_input", + (Class>) input.getClass()), + input) + .remote(); + List r = res.get(); + Assert.assertEquals(r, input); + }); } } @Test public void testPythonCallJavaFunction() { - ObjectRef res = Ray.task(PyFunction.of( - PYTHON_MODULE, "py_func_call_java_function", String.class)).remote(); + ObjectRef res = + Ray.task(PyFunction.of(PYTHON_MODULE, "py_func_call_java_function", String.class)).remote(); Assert.assertEquals(res.get(), "success"); } @Test public void testCallingPythonActor() { - PyActorHandle actor = Ray.actor( - PyActorClass.of(PYTHON_MODULE, "Counter"), "1".getBytes()).remote(); - ObjectRef res = actor.task( - PyActorMethod.of("increase", byte[].class), - "1".getBytes()).remote(); + PyActorHandle actor = + Ray.actor(PyActorClass.of(PYTHON_MODULE, "Counter"), "1".getBytes()).remote(); + ObjectRef res = + actor.task(PyActorMethod.of("increase", byte[].class), "1".getBytes()).remote(); Assert.assertEquals(res.get(), "2".getBytes()); } @Test public void testPythonCallJavaActor() { - ObjectRef res = Ray.task( - PyFunction.of(PYTHON_MODULE, "py_func_call_java_actor", byte[].class), - "1".getBytes()).remote(); + ObjectRef res = + Ray.task( + PyFunction.of(PYTHON_MODULE, "py_func_call_java_actor", byte[].class), + "1".getBytes()) + .remote(); Assert.assertEquals(res.get(), "Counter1".getBytes()); } @@ -164,8 +172,9 @@ public class CrossLanguageInvocationTest extends BaseTest { public void testPassActorHandleFromPythonToJava() { // Call a python function which creates a python actor // and pass the actor handle to callPythonActorHandle. - ObjectRef res = Ray.task(PyFunction.of( - PYTHON_MODULE, "py_func_pass_python_actor_handle", byte[].class)).remote(); + ObjectRef res = + Ray.task(PyFunction.of(PYTHON_MODULE, "py_func_pass_python_actor_handle", byte[].class)) + .remote(); Assert.assertEquals(res.get(), "3".getBytes()); } @@ -174,21 +183,21 @@ public class CrossLanguageInvocationTest extends BaseTest { // Create a java actor, and pass actor handle to python. ActorHandle javaActor = Ray.actor(TestActor::new, "1".getBytes()).remote(); Preconditions.checkState(javaActor instanceof NativeActorHandle); - ObjectRef res = Ray.task( - PyFunction.of(PYTHON_MODULE, - "py_func_call_java_actor_from_handle", - byte[].class), - javaActor).remote(); + ObjectRef res = + Ray.task( + PyFunction.of(PYTHON_MODULE, "py_func_call_java_actor_from_handle", byte[].class), + javaActor) + .remote(); Assert.assertEquals(res.get(), "12".getBytes()); // Create a python actor, and pass actor handle to python. - PyActorHandle pyActor = Ray.actor( - PyActorClass.of(PYTHON_MODULE, "Counter"), "1".getBytes()).remote(); + PyActorHandle pyActor = + Ray.actor(PyActorClass.of(PYTHON_MODULE, "Counter"), "1".getBytes()).remote(); Preconditions.checkState(pyActor instanceof NativeActorHandle); - res = Ray.task( - PyFunction.of(PYTHON_MODULE, - "py_func_call_python_actor_from_handle", - byte[].class), - pyActor).remote(); + res = + Ray.task( + PyFunction.of(PYTHON_MODULE, "py_func_call_python_actor_from_handle", byte[].class), + pyActor) + .remote(); Assert.assertEquals(res.get(), "3".getBytes()); } @@ -197,8 +206,8 @@ public class CrossLanguageInvocationTest extends BaseTest { try { throw new RayException("Test Exception"); } catch (RayException e) { - String formattedException = org.apache.commons.lang3.exception.ExceptionUtils - .getStackTrace(e); + String formattedException = + org.apache.commons.lang3.exception.ExceptionUtils.getStackTrace(e); io.ray.runtime.generated.Common.RayException exception = io.ray.runtime.generated.Common.RayException.parseFrom(e.toBytes()); Assert.assertEquals(exception.getFormattedExceptionString(), formattedException); @@ -207,8 +216,9 @@ public class CrossLanguageInvocationTest extends BaseTest { @Test public void testRaiseExceptionFromPython() { - ObjectRef res = Ray.task(PyFunction.of( - PYTHON_MODULE, "py_func_python_raise_exception", Object.class)).remote(); + ObjectRef res = + Ray.task(PyFunction.of(PYTHON_MODULE, "py_func_python_raise_exception", Object.class)) + .remote(); try { res.get(); } catch (RuntimeException ex) { @@ -218,8 +228,8 @@ public class CrossLanguageInvocationTest extends BaseTest { Assert.assertEquals(e.getLanguage(), Language.PYTHON); // ex.cause is null. Assert.assertNull(ex.getCause()); - Assert.assertTrue(ex.getMessage().contains("ZeroDivisionError: division by zero"), - ex.getMessage()); + Assert.assertTrue( + ex.getMessage().contains("ZeroDivisionError: division by zero"), ex.getMessage()); return; } Assert.fail(); @@ -227,15 +237,16 @@ public class CrossLanguageInvocationTest extends BaseTest { @Test public void testThrowExceptionFromJava() { - ObjectRef res = Ray.task(PyFunction.of( - PYTHON_MODULE, "py_func_java_throw_exception", Object.class)).remote(); + ObjectRef res = + Ray.task(PyFunction.of(PYTHON_MODULE, "py_func_java_throw_exception", Object.class)) + .remote(); try { res.get(); } catch (RuntimeException ex) { final String message = ex.getMessage(); Assert.assertTrue(message.contains("py_func_java_throw_exception"), message); - Assert.assertTrue(message.contains("io.ray.test.CrossLanguageInvocationTest.throwException"), - message); + Assert.assertTrue( + message.contains("io.ray.test.CrossLanguageInvocationTest.throwException"), message); Assert.assertTrue(message.contains("java.lang.ArithmeticException: / by zero"), message); return; } @@ -244,8 +255,9 @@ public class CrossLanguageInvocationTest extends BaseTest { @Test public void testRaiseExceptionFromNestPython() { - ObjectRef res = Ray.task( - PyFunction.of(PYTHON_MODULE, "py_func_nest_python_raise_exception", Object.class)).remote(); + ObjectRef res = + Ray.task(PyFunction.of(PYTHON_MODULE, "py_func_nest_python_raise_exception", Object.class)) + .remote(); try { res.get(); } catch (RuntimeException ex) { @@ -261,15 +273,18 @@ public class CrossLanguageInvocationTest extends BaseTest { @Test public void testThrowExceptionFromNestJava() { - ObjectRef res = Ray.task( - PyFunction.of(PYTHON_MODULE, "py_func_nest_java_throw_exception", Object.class)).remote(); + ObjectRef res = + Ray.task(PyFunction.of(PYTHON_MODULE, "py_func_nest_java_throw_exception", Object.class)) + .remote(); try { res.get(); } catch (RuntimeException ex) { final String message = ex.getMessage(); Assert.assertTrue(message.contains("py_func_nest_java_throw_exception"), message); - Assert.assertEquals(org.apache.commons.lang3.StringUtils - .countMatches(message, "io.ray.runtime.exception.RayTaskException"), 2); + Assert.assertEquals( + org.apache.commons.lang3.StringUtils.countMatches( + message, "io.ray.runtime.exception.RayTaskException"), + 2); Assert.assertTrue(message.contains("py_func_java_throw_exception"), message); Assert.assertTrue(message.contains("java.lang.ArithmeticException: / by zero"), message); return; @@ -279,7 +294,7 @@ public class CrossLanguageInvocationTest extends BaseTest { public static Object[] pack(int i, String s, double f, Object[] o) { // This function will be called from test_cross_language_invocation.py - return new Object[]{i, s, f, o}; + return new Object[] {i, s, f, o}; } public static Object returnInput(Object o) { @@ -308,9 +323,8 @@ public class CrossLanguageInvocationTest extends BaseTest { public static byte[] callPythonActorHandle(PyActorHandle actor) { // This function will be called from test_cross_language_invocation.py - ObjectRef res = actor.task( - PyActorMethod.of("increase", byte[].class), - "1".getBytes()).remote(); + ObjectRef res = + actor.task(PyActorMethod.of("increase", byte[].class), "1".getBytes()).remote(); Assert.assertEquals(res.get(), "3".getBytes()); return (byte[]) res.get(); } @@ -321,14 +335,16 @@ public class CrossLanguageInvocationTest extends BaseTest { } public static Object throwJavaException() { - ObjectRef res = Ray.task( - PyFunction.of(PYTHON_MODULE, "py_func_java_throw_exception", Object.class)).remote(); + ObjectRef res = + Ray.task(PyFunction.of(PYTHON_MODULE, "py_func_java_throw_exception", Object.class)) + .remote(); return res.get(); } public static Object raisePythonException() { - ObjectRef res = Ray.task( - PyFunction.of(PYTHON_MODULE, "py_func_python_raise_exception", Object.class)).remote(); + ObjectRef res = + Ray.task(PyFunction.of(PYTHON_MODULE, "py_func_python_raise_exception", Object.class)) + .remote(); return res.get(); } diff --git a/java/test/src/main/java/io/ray/test/DynamicResourceTest.java b/java/test/src/main/java/io/ray/test/DynamicResourceTest.java index eeaa55b2c..ef9f6481d 100644 --- a/java/test/src/main/java/io/ray/test/DynamicResourceTest.java +++ b/java/test/src/main/java/io/ray/test/DynamicResourceTest.java @@ -16,29 +16,34 @@ public class DynamicResourceTest extends BaseTest { } // Dynamic resources not supported yet. - @Test(groups = {"cluster"}, enabled = false) + @Test( + groups = {"cluster"}, + enabled = false) public void testSetResource() { // Call a task in advance to warm up the cluster to avoid being too slow to start workers. TestUtils.warmUpCluster(); String resourceName = "A"; - ObjectRef obj = Ray.task(DynamicResourceTest::sayHi) - .setResource(resourceName, 10.0) - .remote(); + ObjectRef obj = + Ray.task(DynamicResourceTest::sayHi).setResource(resourceName, 10.0).remote(); WaitResult result = Ray.wait(ImmutableList.of(obj), 1, 1000); Assert.assertEquals(result.getReady().size(), 0); Ray.setResource(resourceName, 10.0); - boolean resourceReady = TestUtils.waitForCondition(() -> { - List nodes = Ray.getRuntimeContext().getAllNodeInfo(); - // NOTE: GCS updates node resources asynchronously. - // If we directly get the value of "A" for comparison without check whether "A" exists or not, - // it maybe lead to NPE. - if (nodes.size() != 1 || !nodes.get(0).resources.containsKey(resourceName)) { - return false; - } - return (0 == Double.compare(10.0, nodes.get(0).resources.get(resourceName))); - }, 2000); + boolean resourceReady = + TestUtils.waitForCondition( + () -> { + List nodes = Ray.getRuntimeContext().getAllNodeInfo(); + // NOTE: GCS updates node resources asynchronously. + // If we directly get the value of "A" for comparison without check whether "A" exists + // or not, + // it maybe lead to NPE. + if (nodes.size() != 1 || !nodes.get(0).resources.containsKey(resourceName)) { + return false; + } + return (0 == Double.compare(10.0, nodes.get(0).resources.get(resourceName))); + }, + 2000); Assert.assertTrue(resourceReady); @@ -46,7 +51,5 @@ public class DynamicResourceTest extends BaseTest { result = Ray.wait(ImmutableList.of(obj), 1, 1000); Assert.assertEquals(result.getReady().size(), 1); Assert.assertEquals(obj.get(), "hi"); - } - } diff --git a/java/test/src/main/java/io/ray/test/ExitActorTest.java b/java/test/src/main/java/io/ray/test/ExitActorTest.java index 1ea86d293..279af55c0 100644 --- a/java/test/src/main/java/io/ray/test/ExitActorTest.java +++ b/java/test/src/main/java/io/ray/test/ExitActorTest.java @@ -35,7 +35,7 @@ public class ExitActorTest extends BaseTest { try { Field field = TaskExecutor.class.getDeclaredField("actorContextMap"); field.setAccessible(true); - return ((Map)field.get(taskExecutor)).size(); + return ((Map) field.get(taskExecutor)).size(); } catch (Exception e) { throw new RuntimeException(e); } @@ -48,8 +48,7 @@ public class ExitActorTest extends BaseTest { } public void testExitActor() throws IOException, InterruptedException { - ActorHandle actor = Ray.actor(ExitingActor::new) - .setMaxRestarts(10000).remote(); + ActorHandle actor = Ray.actor(ExitingActor::new).setMaxRestarts(10000).remote(); Assert.assertEquals(1, (int) (actor.task(ExitingActor::incr).remote().get())); int pid = actor.task(ExitingActor::getPid).remote().get(); Runtime.getRuntime().exec("kill -9 " + pid); @@ -64,8 +63,7 @@ public class ExitActorTest extends BaseTest { public void testExitActorInMultiWorker() { Assert.assertTrue(TestUtils.getNumWorkersPerProcess() > 1); - ActorHandle actor1 = Ray.actor(ExitingActor::new) - .setMaxRestarts(10000).remote(); + ActorHandle actor1 = Ray.actor(ExitingActor::new).setMaxRestarts(10000).remote(); int pid = actor1.task(ExitingActor::getPid).remote().get(); Assert.assertEquals( 1, (int) actor1.task(ExitingActor::getSizeOfActorContextMap).remote().get()); @@ -94,18 +92,19 @@ public class ExitActorTest extends BaseTest { } public void testExitActorWithDynamicOptions() { - ActorHandle actor = Ray.actor(ExitingActor::new) - .setMaxRestarts(10000) - // Set dummy JVM options to start a worker process with only one worker. - .setJvmOptions(" ") - .remote(); + ActorHandle actor = + Ray.actor(ExitingActor::new) + .setMaxRestarts(10000) + // Set dummy JVM options to start a worker process with only one worker. + .setJvmOptions(" ") + .remote(); int pid = actor.task(ExitingActor::getPid).remote().get(); Assert.assertTrue(SystemUtil.isProcessAlive(pid)); ObjectRef obj1 = actor.task(ExitingActor::exit).remote(); Assert.assertThrows(RayActorException.class, obj1::get); // Now the actor shouldn't be reconstructed anymore. - Assert.assertThrows(RayActorException.class, - () -> actor.task(ExitingActor::getPid).remote().get()); + Assert.assertThrows( + RayActorException.class, () -> actor.task(ExitingActor::getPid).remote().get()); // Now the worker process should be dead. Assert.assertTrue(TestUtils.waitForCondition(() -> !SystemUtil.isProcessAlive(pid), 5000)); } diff --git a/java/test/src/main/java/io/ray/test/FailureTest.java b/java/test/src/main/java/io/ray/test/FailureTest.java index 923875d5f..218c78271 100644 --- a/java/test/src/main/java/io/ray/test/FailureTest.java +++ b/java/test/src/main/java/io/ray/test/FailureTest.java @@ -133,8 +133,8 @@ public class FailureTest extends BaseTest { } public void testGetThrowsQuicklyWhenFoundException() { - List> badFunctions = Arrays.asList(FailureTest::badFunc, - FailureTest::badFunc2); + List> badFunctions = + Arrays.asList(FailureTest::badFunc, FailureTest::badFunc2); TestUtils.warmUpCluster(); for (RayFunc0 badFunc : badFunctions) { ObjectRef obj1 = Ray.task(badFunc).remote(); @@ -146,29 +146,37 @@ public class FailureTest extends BaseTest { } catch (RuntimeException e) { Instant end = Instant.now(); long duration = Duration.between(start, end).toMillis(); - Assert.assertTrue(duration < 5000, "Should fail quickly. " + - "Actual execution time: " + duration + " ms."); + Assert.assertTrue( + duration < 5000, + "Should fail quickly. " + "Actual execution time: " + duration + " ms."); } } } public void testExceptionSerialization() { - RayTaskException ex1 = Assert.expectThrows(RayTaskException.class, () -> { - Ray.put(new RayTaskException("xxx", new RayActorException())).get(); - }); + RayTaskException ex1 = + Assert.expectThrows( + RayTaskException.class, + () -> { + Ray.put(new RayTaskException("xxx", new RayActorException())).get(); + }); Assert.assertEquals(ex1.getCause().getClass(), RayActorException.class); - RayTaskException ex2 = Assert.expectThrows(RayTaskException.class, () -> { - Ray.put(new RayTaskException("xxx", new RayWorkerException())).get(); - }); + RayTaskException ex2 = + Assert.expectThrows( + RayTaskException.class, + () -> { + Ray.put(new RayTaskException("xxx", new RayWorkerException())).get(); + }); Assert.assertEquals(ex2.getCause().getClass(), RayWorkerException.class); ObjectId objectId = ObjectId.fromRandom(); - RayTaskException ex3 = Assert.expectThrows(RayTaskException.class, () -> { - Ray.put(new RayTaskException("xxx", new UnreconstructableException(objectId))) - .get(); - }); + RayTaskException ex3 = + Assert.expectThrows( + RayTaskException.class, + () -> { + Ray.put(new RayTaskException("xxx", new UnreconstructableException(objectId))).get(); + }); Assert.assertEquals(ex3.getCause().getClass(), UnreconstructableException.class); Assert.assertEquals(((UnreconstructableException) ex3.getCause()).objectId, objectId); } } - diff --git a/java/test/src/main/java/io/ray/test/GcsClientTest.java b/java/test/src/main/java/io/ray/test/GcsClientTest.java index 45ed69d23..3e44bed25 100644 --- a/java/test/src/main/java/io/ray/test/GcsClientTest.java +++ b/java/test/src/main/java/io/ray/test/GcsClientTest.java @@ -46,6 +46,5 @@ public class GcsClientTest extends BaseTest { for (int i = 2; i < 100; ++i) { Assert.assertEquals(gcsClient.nextJobId(), JobId.fromInt(i)); } - } } diff --git a/java/test/src/main/java/io/ray/test/GlobalGcTest.java b/java/test/src/main/java/io/ray/test/GlobalGcTest.java index f7acdf990..6c66f3c54 100644 --- a/java/test/src/main/java/io/ray/test/GlobalGcTest.java +++ b/java/test/src/main/java/io/ray/test/GlobalGcTest.java @@ -57,13 +57,14 @@ public class GlobalGcTest extends BaseTest { private void testGlobalGcWhenFull(boolean withPut) { // Local driver. - WeakReference localRef = new WeakReference<>( - new LargeObjectWithCyclicRef()); + WeakReference localRef = + new WeakReference<>(new LargeObjectWithCyclicRef()); // Remote workers. - List> actors = IntStream - .range(0, 2).mapToObj(i -> Ray.actor(GarbageHolder::new).remote()) - .collect(Collectors.toList()); + List> actors = + IntStream.range(0, 2) + .mapToObj(i -> Ray.actor(GarbageHolder::new).remote()) + .collect(Collectors.toList()); Assert.assertNotNull(localRef.get()); for (ActorHandle actor : actors) { @@ -82,8 +83,11 @@ public class GlobalGcTest extends BaseTest { actors.get(0).task(GarbageHolder::returnLargeObject).remote().get(); } - TestUtils.waitForCondition(() -> localRef.get() == null && actors.stream().noneMatch( - a -> a.task(GarbageHolder::hasGarbage).remote().get()), 10 * 1000); + TestUtils.waitForCondition( + () -> + localRef.get() == null + && actors.stream().noneMatch(a -> a.task(GarbageHolder::hasGarbage).remote().get()), + 10 * 1000); } public void testGlobalGcWhenFullWithPut() { diff --git a/java/test/src/main/java/io/ray/test/HelloWorldTest.java b/java/test/src/main/java/io/ray/test/HelloWorldTest.java index b9d2ba1d3..0374b54ce 100644 --- a/java/test/src/main/java/io/ray/test/HelloWorldTest.java +++ b/java/test/src/main/java/io/ray/test/HelloWorldTest.java @@ -5,9 +5,7 @@ import io.ray.api.Ray; import org.testng.Assert; import org.testng.annotations.Test; -/** - * Hello world. - */ +/** Hello world. */ public class HelloWorldTest extends BaseTest { private static String hello() { @@ -29,5 +27,4 @@ public class HelloWorldTest extends BaseTest { String helloWorld = Ray.task(HelloWorldTest::merge, hello, world).remote().get(); Assert.assertEquals("hello,world!", helloWorld); } - } diff --git a/java/test/src/main/java/io/ray/test/JobConfigTest.java b/java/test/src/main/java/io/ray/test/JobConfigTest.java index d4725131e..4ba9e484d 100644 --- a/java/test/src/main/java/io/ray/test/JobConfigTest.java +++ b/java/test/src/main/java/io/ray/test/JobConfigTest.java @@ -64,7 +64,6 @@ public class JobConfigTest extends BaseTest { Assert.assertEquals(TestUtils.getNumWorkersPerProcess(), 3); } - public void testInActor() { ActorHandle actor = Ray.actor(MyActor::new).remote(); diff --git a/java/test/src/main/java/io/ray/test/KillActorTest.java b/java/test/src/main/java/io/ray/test/KillActorTest.java index 88b5b483b..d862d3e12 100644 --- a/java/test/src/main/java/io/ray/test/KillActorTest.java +++ b/java/test/src/main/java/io/ray/test/KillActorTest.java @@ -61,13 +61,10 @@ public class KillActorTest extends BaseTest { } private void testKillActor(BiConsumer, Boolean> kill, boolean noRestart) { - ActorHandle actor = Ray.actor(HangActor::new) - .setMaxRestarts(1) - .remote(); + ActorHandle actor = Ray.actor(HangActor::new).setMaxRestarts(1).remote(); ObjectRef result = actor.task(HangActor::hang).remote(); // The actor will hang in this task. - Assert.assertEquals(0, - Ray.wait(ImmutableList.of(result), 1, 500).getReady().size()); + Assert.assertEquals(0, Ray.wait(ImmutableList.of(result), 1, 500).getReady().size()); // Kill the actor kill.accept(actor, noRestart); @@ -84,8 +81,8 @@ public class KillActorTest extends BaseTest { if (noRestart) { // The actor should not be restarted. - Assert.expectThrows(RayActorException.class, - () -> actor.task(HangActor::hang).remote().get()); + Assert.expectThrows( + RayActorException.class, () -> actor.task(HangActor::hang).remote().get()); } else { Assert.assertEquals(actor.task(HangActor::ping).remote().get(), "pong"); } @@ -100,10 +97,11 @@ public class KillActorTest extends BaseTest { public void testRemoteKill() { testKillActor(KillActorTest::remoteKill, false); testKillActor(KillActorTest::remoteKill, true); - testKillActor((actor, noRestart) -> { - ActorHandle killer = Ray.actor(KillerActor::new).remote(); - killer.task(KillerActor::killWithoutRestart, actor).remote(); - }, true); + testKillActor( + (actor, noRestart) -> { + ActorHandle killer = Ray.actor(KillerActor::new).remote(); + killer.task(KillerActor::killWithoutRestart, actor).remote(); + }, + true); } - } diff --git a/java/test/src/main/java/io/ray/test/MetricTest.java b/java/test/src/main/java/io/ray/test/MetricTest.java index 50aba6819..bb615a7e5 100644 --- a/java/test/src/main/java/io/ray/test/MetricTest.java +++ b/java/test/src/main/java/io/ray/test/MetricTest.java @@ -24,14 +24,14 @@ public class MetricTest extends BaseTest { return value <= other + 1e-5 && value >= other - 1e-5; } - private MetricConfig initRayMetrics(long timeIntervalMs, - int threadPoolSize, - long shutdownWaitTimeMs) { - MetricConfig config = MetricConfig.builder() - .timeIntervalMs(timeIntervalMs) - .threadPoolSize(threadPoolSize) - .shutdownWaitTimeMs(shutdownWaitTimeMs) - .create(); + private MetricConfig initRayMetrics( + long timeIntervalMs, int threadPoolSize, long shutdownWaitTimeMs) { + MetricConfig config = + MetricConfig.builder() + .timeIntervalMs(timeIntervalMs) + .threadPoolSize(threadPoolSize) + .shutdownWaitTimeMs(shutdownWaitTimeMs) + .create(); Metrics.init(config); return config; } @@ -121,8 +121,7 @@ public class MetricTest extends BaseTest { boundaries.add(10.0); boundaries.add(15.0); boundaries.add(12.0); - Histogram histogram = new Histogram("metric_histogram", "histogram", "1pc", - boundaries, tags); + Histogram histogram = new Histogram("metric_histogram", "histogram", "1pc", boundaries, tags); for (int i = 1; i <= 200; ++i) { histogram.update(i * 1.0d); } @@ -229,5 +228,4 @@ public class MetricTest extends BaseTest { } Assert.assertTrue(doubleEqual(histogram.getValue(), 200.0d)); } - } diff --git a/java/test/src/main/java/io/ray/test/MultiDriverTest.java b/java/test/src/main/java/io/ray/test/MultiDriverTest.java index 80d24d79d..9c781f562 100644 --- a/java/test/src/main/java/io/ray/test/MultiDriverTest.java +++ b/java/test/src/main/java/io/ray/test/MultiDriverTest.java @@ -73,22 +73,23 @@ public class MultiDriverTest extends BaseTest { // Wait for drivers to finish. for (Process driver : drivers) { driver.waitFor(); - Assert.assertEquals(driver.exitValue(), 0, - "The driver exited with code " + driver.exitValue()); + Assert.assertEquals( + driver.exitValue(), 0, "The driver exited with code " + driver.exitValue()); } // Read driver outputs and check for any PID overlap. Set pids = new HashSet<>(); for (Process driver : drivers) { - try (BufferedReader reader = new BufferedReader( - new InputStreamReader(driver.getInputStream()))) { + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(driver.getInputStream()))) { String line; int previousSize = pids.size(); while ((line = reader.readLine()) != null) { if (line.startsWith(PID_LIST_PREFIX)) { for (String pidString : line.substring(PID_LIST_PREFIX.length()).split(",")) { // Make sure the PIDs don't overlap. - Assert.assertTrue(pids.add(Integer.valueOf(pidString)), + Assert.assertTrue( + pids.add(Integer.valueOf(pidString)), "Worker process with PID " + line + " is shared by multiple drivers."); } break; @@ -103,15 +104,16 @@ public class MultiDriverTest extends BaseTest { private Process startDriver() throws IOException { RayConfig rayConfig = TestUtils.getRuntime().getRayConfig(); - ProcessBuilder builder = new ProcessBuilder( - "java", - "-cp", - System.getProperty("java.class.path"), - "-Dray.address=" + rayConfig.getRedisAddress(), - "-Dray.object-store.socket-name=" + rayConfig.objectStoreSocketName, - "-Dray.raylet.socket-name=" + rayConfig.rayletSocketName, - "-Dray.raylet.node-manager-port=" + String.valueOf(rayConfig.getNodeManagerPort()), - MultiDriverTest.class.getName()); + ProcessBuilder builder = + new ProcessBuilder( + "java", + "-cp", + System.getProperty("java.class.path"), + "-Dray.address=" + rayConfig.getRedisAddress(), + "-Dray.object-store.socket-name=" + rayConfig.objectStoreSocketName, + "-Dray.raylet.socket-name=" + rayConfig.rayletSocketName, + "-Dray.raylet.node-manager-port=" + String.valueOf(rayConfig.getNodeManagerPort()), + MultiDriverTest.class.getName()); builder.redirectError(Redirect.INHERIT); return builder.start(); } diff --git a/java/test/src/main/java/io/ray/test/MultiLanguageClusterTest.java b/java/test/src/main/java/io/ray/test/MultiLanguageClusterTest.java index 0b244a841..7908f611d 100644 --- a/java/test/src/main/java/io/ray/test/MultiLanguageClusterTest.java +++ b/java/test/src/main/java/io/ray/test/MultiLanguageClusterTest.java @@ -17,5 +17,4 @@ public class MultiLanguageClusterTest extends BaseTest { ObjectRef obj = Ray.task(MultiLanguageClusterTest::echo, "hello").remote(); Assert.assertEquals("hello", obj.get()); } - } diff --git a/java/test/src/main/java/io/ray/test/MultiThreadingTest.java b/java/test/src/main/java/io/ray/test/MultiThreadingTest.java index f9371748d..cffd90562 100644 --- a/java/test/src/main/java/io/ray/test/MultiThreadingTest.java +++ b/java/test/src/main/java/io/ray/test/MultiThreadingTest.java @@ -49,13 +49,16 @@ public class MultiThreadingTest extends BaseTest { public ActorId getCurrentActorId() throws Exception { final Object[] result = new Object[1]; - Thread thread = new Thread(Ray.wrapRunnable(() -> { - try { - result[0] = Ray.getRuntimeContext().getCurrentActorId(); - } catch (Exception e) { - result[0] = e; - } - })); + Thread thread = + new Thread( + Ray.wrapRunnable( + () -> { + try { + result[0] = Ray.getRuntimeContext().getCurrentActorId(); + } catch (Exception e) { + result[0] = e; + } + })); thread.start(); thread.join(); if (result[0] instanceof Exception) { @@ -69,49 +72,59 @@ public class MultiThreadingTest extends BaseTest { static String testMultiThreading() { Random random = new Random(); // Test calling normal functions. - runTestCaseInMultipleThreads(() -> { - int arg = random.nextInt(); - ObjectRef obj = Ray.task(MultiThreadingTest::echo, arg).remote(); - Assert.assertEquals(arg, (int) obj.get()); - }, LOOP_COUNTER); + runTestCaseInMultipleThreads( + () -> { + int arg = random.nextInt(); + ObjectRef obj = Ray.task(MultiThreadingTest::echo, arg).remote(); + Assert.assertEquals(arg, (int) obj.get()); + }, + LOOP_COUNTER); // Test calling actors. ActorHandle echoActor = Ray.actor(Echo::new).remote(); - runTestCaseInMultipleThreads(() -> { - int arg = random.nextInt(); - ObjectRef obj = echoActor.task(Echo::echo, arg).remote(); - Assert.assertEquals(arg, (int) obj.get()); - }, LOOP_COUNTER); + runTestCaseInMultipleThreads( + () -> { + int arg = random.nextInt(); + ObjectRef obj = echoActor.task(Echo::echo, arg).remote(); + Assert.assertEquals(arg, (int) obj.get()); + }, + LOOP_COUNTER); // Test creating multi actors - runTestCaseInMultipleThreads(() -> { - int arg = random.nextInt(); - ActorHandle echoActor1 = Ray.actor(Echo::new).remote(); - try { - // Sleep a while to test the case that another actor is created before submitting - // tasks to this actor. - TimeUnit.MILLISECONDS.sleep(10); - } catch (InterruptedException e) { - LOGGER.warn("Got exception while sleeping.", e); - } - ObjectRef obj = echoActor1.task(Echo::echo, arg).remote(); - Assert.assertEquals(arg, (int) obj.get()); - }, 1); + runTestCaseInMultipleThreads( + () -> { + int arg = random.nextInt(); + ActorHandle echoActor1 = Ray.actor(Echo::new).remote(); + try { + // Sleep a while to test the case that another actor is created before submitting + // tasks to this actor. + TimeUnit.MILLISECONDS.sleep(10); + } catch (InterruptedException e) { + LOGGER.warn("Got exception while sleeping.", e); + } + ObjectRef obj = echoActor1.task(Echo::echo, arg).remote(); + Assert.assertEquals(arg, (int) obj.get()); + }, + 1); // Test put and get. - runTestCaseInMultipleThreads(() -> { - int arg = random.nextInt(); - ObjectRef obj = Ray.put(arg); - Assert.assertEquals(arg, (int) obj.get()); - }, LOOP_COUNTER); + runTestCaseInMultipleThreads( + () -> { + int arg = random.nextInt(); + ObjectRef obj = Ray.put(arg); + Assert.assertEquals(arg, (int) obj.get()); + }, + LOOP_COUNTER); TestUtils.warmUpCluster(); // Test wait for one object in multi threads. ObjectRef obj = Ray.task(MultiThreadingTest::echo, 100).remote(); - runTestCaseInMultipleThreads(() -> { - WaitResult result = Ray.wait(ImmutableList.of(obj), 1, 1000); - Assert.assertEquals(1, result.getReady().size()); - }, 1); + runTestCaseInMultipleThreads( + () -> { + WaitResult result = Ray.wait(ImmutableList.of(obj), 1, 1000); + Assert.assertEquals(1, result.getReady().size()); + }, + 1); return "ok"; } @@ -134,21 +147,19 @@ public class MultiThreadingTest extends BaseTest { Assert.assertEquals(actorId, actorIdTester.getId()); } - /** - * Call this method each time to avoid hitting the cache in {@link ObjectRef#get()}. - */ + /** Call this method each time to avoid hitting the cache in {@link ObjectRef#get()}. */ static Runnable[] generateRunnables() { final ObjectRef fooObject = Ray.put(1); final ActorHandle fooActor = Ray.actor(Echo::new).remote(); - return new Runnable[]{ - () -> Ray.put(1), - () -> Ray.get(ImmutableList.of(fooObject)), - fooObject::get, - () -> Ray.wait(ImmutableList.of(fooObject)), - Ray::getRuntimeContext, - () -> Ray.task(MultiThreadingTest::echo, 1).remote(), - () -> Ray.actor(Echo::new).remote(), - () -> fooActor.task(Echo::echo, 1).remote(), + return new Runnable[] { + () -> Ray.put(1), + () -> Ray.get(ImmutableList.of(fooObject)), + fooObject::get, + () -> Ray.wait(ImmutableList.of(fooObject)), + Ray::getRuntimeContext, + () -> Ray.task(MultiThreadingTest::echo, 1).remote(), + () -> Ray.actor(Echo::new).remote(), + () -> fooActor.task(Echo::echo, 1).remote(), }; } @@ -165,16 +176,19 @@ public class MultiThreadingTest extends BaseTest { { Runnable[] runnables = generateRunnables(); - Thread thread = new Thread(Ray.wrapRunnable(() -> { - try { - // It would be OK to run them in another thread if wrapped the runnable. - for (Runnable runnable : runnables) { - runnable.run(); - } - } catch (Throwable ex) { - throwable[0] = ex; - } - })); + Thread thread = + new Thread( + Ray.wrapRunnable( + () -> { + try { + // It would be OK to run them in another thread if wrapped the runnable. + for (Runnable runnable : runnables) { + runnable.run(); + } + } catch (Throwable ex) { + throwable[0] = ex; + } + })); thread.start(); thread.join(); if (throwable[0] != null) { @@ -184,16 +198,18 @@ public class MultiThreadingTest extends BaseTest { { Runnable[] runnables = generateRunnables(); - Thread thread = new Thread(() -> { - try { - // It wouldn't be OK to run them in another thread if not wrapped the runnable. - for (Runnable runnable : runnables) { - Assert.expectThrows(RuntimeException.class, runnable::run); - } - } catch (Throwable ex) { - throwable[0] = ex; - } - }); + Thread thread = + new Thread( + () -> { + try { + // It wouldn't be OK to run them in another thread if not wrapped the runnable. + for (Runnable runnable : runnables) { + Assert.expectThrows(RuntimeException.class, runnable::run); + } + } catch (Throwable ex) { + throwable[0] = ex; + } + }); thread.start(); thread.join(); if (throwable[0] != null) { @@ -230,14 +246,16 @@ public class MultiThreadingTest extends BaseTest { public void testGetAndSetAsyncContext() throws InterruptedException { Object asyncContext = Ray.getAsyncContext(); Throwable[] throwable = new Throwable[1]; - Thread thread = new Thread(() -> { - try { - Ray.setAsyncContext(asyncContext); - Ray.put(1); - } catch (Throwable ex) { - throwable[0] = ex; - } - }); + Thread thread = + new Thread( + () -> { + try { + Ray.setAsyncContext(asyncContext); + Ray.put(1); + } catch (Throwable ex) { + throwable[0] = ex; + } + }); thread.start(); thread.join(); if (throwable[0] != null) { @@ -251,13 +269,15 @@ public class MultiThreadingTest extends BaseTest { try { List> futures = new ArrayList<>(); for (int i = 0; i < NUM_THREADS; i++) { - Callable task = Ray.wrapCallable(() -> { - for (int j = 0; j < numRepeats; j++) { - TimeUnit.MILLISECONDS.sleep(1); - testCase.run(); - } - return "ok"; - }); + Callable task = + Ray.wrapCallable( + () -> { + for (int j = 0; j < numRepeats; j++) { + TimeUnit.MILLISECONDS.sleep(1); + testCase.run(); + } + return "ok"; + }); futures.add(service.submit(task)); } for (Future future : futures) { @@ -275,14 +295,16 @@ public class MultiThreadingTest extends BaseTest { private static boolean testGetAsyncContextAndSetAsyncContext() throws Exception { final Object asyncContext = Ray.getAsyncContext(); final Object[] result = new Object[1]; - Thread thread = new Thread(() -> { - try { - Ray.setAsyncContext(asyncContext); - Ray.put(0); - } catch (Exception e) { - result[0] = e; - } - }); + Thread thread = + new Thread( + () -> { + try { + Ray.setAsyncContext(asyncContext); + Ray.put(0); + } catch (Exception e) { + result[0] = e; + } + }); thread.start(); thread.join(); if (result[0] instanceof Exception) { @@ -296,9 +318,8 @@ public class MultiThreadingTest extends BaseTest { } public void testGetAsyncContextAndSetAsyncContextInWorker() { - ObjectRef obj = Ray.task( - MultiThreadingTest::testGetAsyncContextAndSetAsyncContext).remote(); + ObjectRef obj = + Ray.task(MultiThreadingTest::testGetAsyncContextAndSetAsyncContext).remote(); Assert.assertTrue(obj.get()); } - } diff --git a/java/test/src/main/java/io/ray/test/NamedActorTest.java b/java/test/src/main/java/io/ray/test/NamedActorTest.java index 6b13d4384..febd8234c 100644 --- a/java/test/src/main/java/io/ray/test/NamedActorTest.java +++ b/java/test/src/main/java/io/ray/test/NamedActorTest.java @@ -32,8 +32,8 @@ public class NamedActorTest extends BaseTest { Optional> namedActor = Ray.getActor(name); Assert.assertTrue(namedActor.isPresent()); // Verify that this handle is correct. - Assert.assertEquals(namedActor.get().task(Counter::increment).remote().get(), - Integer.valueOf(2)); + Assert.assertEquals( + namedActor.get().task(Counter::increment).remote().get(), Integer.valueOf(2)); } @Test @@ -49,30 +49,31 @@ public class NamedActorTest extends BaseTest { Optional> namedActor = Ray.getGlobalActor(name); Assert.assertTrue(namedActor.isPresent()); // Verify that this handle is correct. - Assert.assertEquals(namedActor.get().task(Counter::increment).remote().get(), - Integer.valueOf(2)); + Assert.assertEquals( + namedActor.get().task(Counter::increment).remote().get(), Integer.valueOf(2)); if (!TestUtils.isSingleProcessMode()) { // Get the global actor from another driver. RayConfig rayConfig = TestUtils.getRuntime().getRayConfig(); - ProcessBuilder builder = new ProcessBuilder( - "java", - "-cp", - System.getProperty("java.class.path"), - "-Dray.address=" + rayConfig.getRedisAddress(), - "-Dray.object-store.socket-name=" + rayConfig.objectStoreSocketName, - "-Dray.raylet.socket-name=" + rayConfig.rayletSocketName, - "-Dray.raylet.node-manager-port=" + rayConfig.getNodeManagerPort(), - NamedActorTest.class.getName(), - name); + ProcessBuilder builder = + new ProcessBuilder( + "java", + "-cp", + System.getProperty("java.class.path"), + "-Dray.address=" + rayConfig.getRedisAddress(), + "-Dray.object-store.socket-name=" + rayConfig.objectStoreSocketName, + "-Dray.raylet.socket-name=" + rayConfig.rayletSocketName, + "-Dray.raylet.node-manager-port=" + rayConfig.getNodeManagerPort(), + NamedActorTest.class.getName(), + name); builder.redirectError(ProcessBuilder.Redirect.INHERIT); Process driver = builder.start(); Assert.assertTrue(driver.waitFor(60, TimeUnit.SECONDS)); - Assert.assertEquals(driver.exitValue(), 0, - "The driver exited with code " + driver.exitValue()); + Assert.assertEquals( + driver.exitValue(), 0, "The driver exited with code " + driver.exitValue()); - Assert.assertEquals(namedActor.get().task(Counter::increment).remote().get(), - Integer.valueOf(4)); + Assert.assertEquals( + namedActor.get().task(Counter::increment).remote().get(), Integer.valueOf(4)); } } @@ -83,8 +84,8 @@ public class NamedActorTest extends BaseTest { Optional> namedActor = Ray.getGlobalActor(actorName); Assert.assertTrue(namedActor.isPresent()); // Verify that this handle is correct. - Assert.assertEquals(namedActor.get().task(Counter::increment).remote().get(), - Integer.valueOf(3)); + Assert.assertEquals( + namedActor.get().task(Counter::increment).remote().get(), Integer.valueOf(3)); } @Test(expectedExceptions = IllegalArgumentException.class) @@ -97,5 +98,4 @@ public class NamedActorTest extends BaseTest { // Registering with the same name should fail. Ray.actor(Counter::new).setName(name).remote(); } - } diff --git a/java/test/src/main/java/io/ray/test/ObjectStoreTest.java b/java/test/src/main/java/io/ray/test/ObjectStoreTest.java index 7643d8001..8e68a44c1 100644 --- a/java/test/src/main/java/io/ray/test/ObjectStoreTest.java +++ b/java/test/src/main/java/io/ray/test/ObjectStoreTest.java @@ -8,9 +8,7 @@ import java.util.stream.Collectors; import org.testng.Assert; import org.testng.annotations.Test; -/** - * Test putting and getting objects. - */ +/** Test putting and getting objects. */ public class ObjectStoreTest extends BaseTest { @Test @@ -36,8 +34,7 @@ public class ObjectStoreTest extends BaseTest { @Test public void testGetMultipleObjects() { List ints = ImmutableList.of(1, 2, 3, 4, 5); - List> refs = ints.stream().map(Ray::put) - .collect(Collectors.toList()); + List> refs = ints.stream().map(Ray::put).collect(Collectors.toList()); Assert.assertEquals(ints, Ray.get(refs)); } } diff --git a/java/test/src/main/java/io/ray/test/PlacementGroupTest.java b/java/test/src/main/java/io/ray/test/PlacementGroupTest.java index 547d3bae6..14bf0fd6a 100644 --- a/java/test/src/main/java/io/ray/test/PlacementGroupTest.java +++ b/java/test/src/main/java/io/ray/test/PlacementGroupTest.java @@ -32,14 +32,14 @@ public class PlacementGroupTest extends BaseTest { // This test just creates a placement group with one bundle. // It's not comprehensive to test all placement group test cases. public void testCreateAndCallActor() { - PlacementGroupImpl placementGroup = (PlacementGroupImpl)PlacementGroupTestUtils - .createSimpleGroup(); + PlacementGroupImpl placementGroup = + (PlacementGroupImpl) PlacementGroupTestUtils.createSimpleGroup(); Assert.assertTrue(placementGroup.wait(10)); - Assert.assertEquals(placementGroup.getName(),"unnamed_group"); + Assert.assertEquals(placementGroup.getName(), "unnamed_group"); // Test creating an actor from a constructor. - ActorHandle actor = Ray.actor(Counter::new, 1) - .setPlacementGroup(placementGroup, 0).remote(); + ActorHandle actor = + Ray.actor(Counter::new, 1).setPlacementGroup(placementGroup, 0).remote(); Assert.assertNotEquals(actor.getId(), ActorId.NIL); // Test calling an actor. @@ -48,20 +48,22 @@ public class PlacementGroupTest extends BaseTest { @Test(groups = {"cluster"}) public void testGetPlacementGroup() { - PlacementGroupImpl firstPlacementGroup = (PlacementGroupImpl)PlacementGroupTestUtils - .createNameSpecifiedSimpleGroup("CPU", 1, PlacementStrategy.PACK, - 1.0, "first_placement_group"); + PlacementGroupImpl firstPlacementGroup = + (PlacementGroupImpl) + PlacementGroupTestUtils.createNameSpecifiedSimpleGroup( + "CPU", 1, PlacementStrategy.PACK, 1.0, "first_placement_group"); - PlacementGroupImpl secondPlacementGroup = (PlacementGroupImpl)PlacementGroupTestUtils - .createNameSpecifiedSimpleGroup("CPU", 1, PlacementStrategy.PACK, - 1.0, "second_placement_group"); + PlacementGroupImpl secondPlacementGroup = + (PlacementGroupImpl) + PlacementGroupTestUtils.createNameSpecifiedSimpleGroup( + "CPU", 1, PlacementStrategy.PACK, 1.0, "second_placement_group"); Assert.assertTrue(firstPlacementGroup.wait(10)); Assert.assertTrue(secondPlacementGroup.wait(10)); PlacementGroupImpl firstPlacementGroupRes = - (PlacementGroupImpl)Ray.getPlacementGroup((firstPlacementGroup).getId()); + (PlacementGroupImpl) Ray.getPlacementGroup((firstPlacementGroup).getId()); PlacementGroupImpl secondPlacementGroupRes = - (PlacementGroupImpl)Ray.getPlacementGroup((secondPlacementGroup).getId()); + (PlacementGroupImpl) Ray.getPlacementGroup((secondPlacementGroup).getId()); Assert.assertNotNull(firstPlacementGroupRes); Assert.assertNotNull(secondPlacementGroupRes); @@ -74,25 +76,28 @@ public class PlacementGroupTest extends BaseTest { List allPlacementGroup = Ray.getAllPlacementGroups(); Assert.assertEquals(allPlacementGroup.size(), 2); - PlacementGroupImpl placementGroupRes = (PlacementGroupImpl)allPlacementGroup.get(0); + PlacementGroupImpl placementGroupRes = (PlacementGroupImpl) allPlacementGroup.get(0); Assert.assertNotNull(placementGroupRes.getId()); - PlacementGroupImpl expectPlacementGroup = placementGroupRes.getId() - .equals(firstPlacementGroup.getId()) ? firstPlacementGroup : secondPlacementGroup; + PlacementGroupImpl expectPlacementGroup = + placementGroupRes.getId().equals(firstPlacementGroup.getId()) + ? firstPlacementGroup + : secondPlacementGroup; Assert.assertEquals(placementGroupRes.getName(), expectPlacementGroup.getName()); - Assert.assertEquals(placementGroupRes.getBundles().size(), - expectPlacementGroup.getBundles().size()); + Assert.assertEquals( + placementGroupRes.getBundles().size(), expectPlacementGroup.getBundles().size()); Assert.assertEquals(placementGroupRes.getStrategy(), expectPlacementGroup.getStrategy()); } @Test(groups = {"cluster"}) public void testRemovePlacementGroup() { - PlacementGroupTestUtils.createNameSpecifiedSimpleGroup("CPU", - 1, PlacementStrategy.PACK, 1.0, "first_placement_group"); + PlacementGroupTestUtils.createNameSpecifiedSimpleGroup( + "CPU", 1, PlacementStrategy.PACK, 1.0, "first_placement_group"); - PlacementGroupImpl secondPlacementGroup = (PlacementGroupImpl)PlacementGroupTestUtils - .createNameSpecifiedSimpleGroup("CPU", 1, PlacementStrategy.PACK, - 1.0, "second_placement_group"); + PlacementGroupImpl secondPlacementGroup = + (PlacementGroupImpl) + PlacementGroupTestUtils.createNameSpecifiedSimpleGroup( + "CPU", 1, PlacementStrategy.PACK, 1.0, "second_placement_group"); List allPlacementGroup = Ray.getAllPlacementGroups(); Assert.assertEquals(allPlacementGroup.size(), 2); @@ -100,7 +105,7 @@ public class PlacementGroupTest extends BaseTest { Ray.removePlacementGroup(secondPlacementGroup.getId()); PlacementGroupImpl removedPlacementGroup = - (PlacementGroupImpl)Ray.getPlacementGroup((secondPlacementGroup).getId()); + (PlacementGroupImpl) Ray.getPlacementGroup((secondPlacementGroup).getId()); Assert.assertEquals(removedPlacementGroup.getState(), PlacementGroupState.REMOVED); // Wait for placement group after it is removed. @@ -132,12 +137,12 @@ public class PlacementGroupTest extends BaseTest { Assert.assertEquals(exceptionCount, 2); } - @Test (expectedExceptions = { IllegalArgumentException.class }) + @Test(expectedExceptions = {IllegalArgumentException.class}) public void testBundleSizeValidCheckWhenCreate() { PlacementGroupTestUtils.createBundleSizeInvalidGroup(); } - @Test (expectedExceptions = { IllegalArgumentException.class }) + @Test(expectedExceptions = {IllegalArgumentException.class}) public void testBundleResourceValidCheckWhenCreate() { PlacementGroupTestUtils.createBundleResourceInvalidGroup(); } diff --git a/java/test/src/main/java/io/ray/test/PlacementGroupTestUtils.java b/java/test/src/main/java/io/ray/test/PlacementGroupTestUtils.java index 72ebd6629..7edf71324 100644 --- a/java/test/src/main/java/io/ray/test/PlacementGroupTestUtils.java +++ b/java/test/src/main/java/io/ray/test/PlacementGroupTestUtils.java @@ -8,13 +8,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -/** - * A utils class for placement group test. - */ +/** A utils class for placement group test. */ public class PlacementGroupTestUtils { - public static PlacementGroup createNameSpecifiedSimpleGroup(String resourceName, int bundleSize, - PlacementStrategy strategy, Double resourceSize, String groupName) { + public static PlacementGroup createNameSpecifiedSimpleGroup( + String resourceName, + int bundleSize, + PlacementStrategy strategy, + Double resourceSize, + String groupName) { List> bundles = new ArrayList<>(); for (int i = 0; i < bundleSize; i++) { @@ -26,10 +28,10 @@ public class PlacementGroupTestUtils { return Ray.createPlacementGroup(groupName, bundles, strategy); } - public static PlacementGroup createSpecifiedSimpleGroup(String resourceName, int bundleSize, - PlacementStrategy strategy, Double resourceSize) { - return createNameSpecifiedSimpleGroup(resourceName, bundleSize, strategy, - resourceSize, "unnamed_group"); + public static PlacementGroup createSpecifiedSimpleGroup( + String resourceName, int bundleSize, PlacementStrategy strategy, Double resourceSize) { + return createNameSpecifiedSimpleGroup( + resourceName, bundleSize, strategy, resourceSize, "unnamed_group"); } public static PlacementGroup createSimpleGroup() { @@ -43,5 +45,4 @@ public class PlacementGroupTestUtils { public static void createBundleResourceInvalidGroup() { createSpecifiedSimpleGroup("CPU", 1, PlacementStrategy.PACK, 0.0); } - } diff --git a/java/test/src/main/java/io/ray/test/PlasmaFreeTest.java b/java/test/src/main/java/io/ray/test/PlasmaFreeTest.java index 59fafc0f4..3e49ff798 100644 --- a/java/test/src/main/java/io/ray/test/PlasmaFreeTest.java +++ b/java/test/src/main/java/io/ray/test/PlasmaFreeTest.java @@ -20,9 +20,14 @@ public class PlasmaFreeTest extends BaseTest { Assert.assertEquals("hello", helloString); Ray.internal().free(ImmutableList.of(helloId), true); - final boolean result = TestUtils.waitForCondition(() -> - !TestUtils.getRuntime().getObjectStore() - .wait(ImmutableList.of(((ObjectRefImpl) helloId).getId()), 1, 0).get(0), 50); + final boolean result = + TestUtils.waitForCondition( + () -> + !TestUtils.getRuntime() + .getObjectStore() + .wait(ImmutableList.of(((ObjectRefImpl) helloId).getId()), 1, 0) + .get(0), + 50); if (TestUtils.isSingleProcessMode()) { Assert.assertTrue(result); } else { diff --git a/java/test/src/main/java/io/ray/test/RayAlterSuiteListener.java b/java/test/src/main/java/io/ray/test/RayAlterSuiteListener.java index 7a6edbe95..185372bdf 100644 --- a/java/test/src/main/java/io/ray/test/RayAlterSuiteListener.java +++ b/java/test/src/main/java/io/ray/test/RayAlterSuiteListener.java @@ -21,4 +21,4 @@ public class RayAlterSuiteListener implements IAlterSuiteListener { groups.setRun(run); suite.setGroups(groups); } -} \ No newline at end of file +} diff --git a/java/test/src/main/java/io/ray/test/RayCallTest.java b/java/test/src/main/java/io/ray/test/RayCallTest.java index acb4f149e..30b1dbfd3 100644 --- a/java/test/src/main/java/io/ray/test/RayCallTest.java +++ b/java/test/src/main/java/io/ray/test/RayCallTest.java @@ -11,9 +11,7 @@ import java.util.Map; import org.testng.Assert; import org.testng.annotations.Test; -/** - * Test Ray.call API - */ +/** Test Ray.call API */ public class RayCallTest extends BaseTest { private static int testInt(int val) { @@ -69,9 +67,7 @@ public class RayCallTest extends BaseTest { return buffer; } - /** - * Test calling and returning different types. - */ + /** Test calling and returning different types. */ @Test public void testType() { Assert.assertEquals(1, (int) Ray.task(RayCallTest::testInt, 1).remote().get()); @@ -134,16 +130,12 @@ public class RayCallTest extends BaseTest { public void testNumberOfParameters() { Assert.assertEquals(0, (int) Ray.task(RayCallTest::testNoParam).remote().get()); Assert.assertEquals(1, (int) Ray.task(RayCallTest::testOneParam, 1).remote().get()); - Assert.assertEquals(2, (int) Ray.task( - RayCallTest::testTwoParams, 1, 1).remote().get()); - Assert.assertEquals(3, (int) Ray.task( - RayCallTest::testThreeParams, 1, 1, 1).remote().get()); - Assert.assertEquals(4, (int) Ray.task( - RayCallTest::testFourParams, 1, 1, 1, 1).remote().get()); - Assert.assertEquals(5, (int) Ray.task( - RayCallTest::testFiveParams, 1, 1, 1, 1, 1).remote().get()); - Assert.assertEquals(6, (int) Ray.task( - RayCallTest::testSixParams, 1, 1, 1, 1, 1, 1).remote().get()); + Assert.assertEquals(2, (int) Ray.task(RayCallTest::testTwoParams, 1, 1).remote().get()); + Assert.assertEquals(3, (int) Ray.task(RayCallTest::testThreeParams, 1, 1, 1).remote().get()); + Assert.assertEquals(4, (int) Ray.task(RayCallTest::testFourParams, 1, 1, 1, 1).remote().get()); + Assert.assertEquals( + 5, (int) Ray.task(RayCallTest::testFiveParams, 1, 1, 1, 1, 1).remote().get()); + Assert.assertEquals( + 6, (int) Ray.task(RayCallTest::testSixParams, 1, 1, 1, 1, 1, 1).remote().get()); } - } diff --git a/java/test/src/main/java/io/ray/test/RayMethodsTest.java b/java/test/src/main/java/io/ray/test/RayMethodsTest.java index 296a41cef..e5b6dec7c 100644 --- a/java/test/src/main/java/io/ray/test/RayMethodsTest.java +++ b/java/test/src/main/java/io/ray/test/RayMethodsTest.java @@ -9,9 +9,7 @@ import java.util.stream.Collectors; import org.testng.Assert; import org.testng.annotations.Test; -/** - * Integration test for Ray.* - */ +/** Integration test for Ray.* */ public class RayMethodsTest extends BaseTest { @Test @@ -33,6 +31,5 @@ public class RayMethodsTest extends BaseTest { Assert.assertEquals(1, i1); Assert.assertEquals(3.14, f1, Double.MIN_NORMAL); Assert.assertNull(n1); - } } diff --git a/java/test/src/main/java/io/ray/test/RaySerializerTest.java b/java/test/src/main/java/io/ray/test/RaySerializerTest.java index d9d2e335f..8e4ab8d74 100644 --- a/java/test/src/main/java/io/ray/test/RaySerializerTest.java +++ b/java/test/src/main/java/io/ray/test/RaySerializerTest.java @@ -13,14 +13,12 @@ public class RaySerializerTest extends BaseTest { @Test public void testSerializePyActor() { - PyActorHandle pyActor = Ray.actor( - PyActorClass.of("test", "RaySerializerTest")).remote(); + PyActorHandle pyActor = Ray.actor(PyActorClass.of("test", "RaySerializerTest")).remote(); NativeRayObject nativeRayObject = ObjectSerializer.serialize(pyActor); - PyActorHandle result = (PyActorHandle) ObjectSerializer - .deserialize(nativeRayObject, null, Object.class); + PyActorHandle result = + (PyActorHandle) ObjectSerializer.deserialize(nativeRayObject, null, Object.class); Assert.assertEquals(result.getId(), pyActor.getId()); Assert.assertEquals(result.getModuleName(), "test"); Assert.assertEquals(result.getClassName(), "RaySerializerTest"); } - } diff --git a/java/test/src/main/java/io/ray/test/RayletConfigTest.java b/java/test/src/main/java/io/ray/test/RayletConfigTest.java index 81c7ce1d3..ec0a26da7 100644 --- a/java/test/src/main/java/io/ray/test/RayletConfigTest.java +++ b/java/test/src/main/java/io/ray/test/RayletConfigTest.java @@ -25,8 +25,10 @@ public class RayletConfigTest extends BaseTest { public static class TestActor { public String getConfigValue() { - return TestUtils.getRuntime().getRayConfig() - .rayletConfigParameters.get(RAY_CONFIG_KEY) + return TestUtils.getRuntime() + .getRayConfig() + .rayletConfigParameters + .get(RAY_CONFIG_KEY) .toString(); } } diff --git a/java/test/src/main/java/io/ray/test/RedisPasswordTest.java b/java/test/src/main/java/io/ray/test/RedisPasswordTest.java index c9317ed29..165d2ca99 100644 --- a/java/test/src/main/java/io/ray/test/RedisPasswordTest.java +++ b/java/test/src/main/java/io/ray/test/RedisPasswordTest.java @@ -28,5 +28,4 @@ public class RedisPasswordTest extends BaseTest { ObjectRef obj = Ray.task(RedisPasswordTest::echo, "hello").remote(); Assert.assertEquals("hello", obj.get()); } - } diff --git a/java/test/src/main/java/io/ray/test/ReferenceCountingTest.java b/java/test/src/main/java/io/ray/test/ReferenceCountingTest.java index 0d8e40856..aa5658195 100644 --- a/java/test/src/main/java/io/ray/test/ReferenceCountingTest.java +++ b/java/test/src/main/java/io/ray/test/ReferenceCountingTest.java @@ -44,12 +44,12 @@ public class ReferenceCountingTest extends BaseTest { Field referencesField = ObjectRefImpl.class.getDeclaredField("REFERENCES"); referencesField.setAccessible(true); Set references = (Set) referencesField.get(null); - Class referenceClass = Class - .forName("io.ray.runtime.object.ObjectRefImpl$ObjectRefImplReference"); + Class referenceClass = + Class.forName("io.ray.runtime.object.ObjectRefImpl$ObjectRefImplReference"); Method finalizeReferentMethod = referenceClass.getDeclaredMethod("finalizeReferent"); finalizeReferentMethod.setAccessible(true); for (Object reference : references) { - if (obj.equals(((Reference)reference).get())) { + if (obj.equals(((Reference) reference).get())) { finalizeReferentMethod.invoke(reference); break; } @@ -91,18 +91,27 @@ public class ReferenceCountingTest extends BaseTest { checkRefCounts(ImmutableMap.of(objectId, new long[] {localRefCount, submittedTaskRefCount})); } - private void checkRefCounts(ObjectId objectId1, long localRefCount1, long submittedTaskRefCount1, - ObjectId objectId2, long localRefCount2, long submittedTaskRefCount2) { - checkRefCounts(ImmutableMap.of(objectId1, new long[] {localRefCount1, submittedTaskRefCount1}, - objectId2, new long[] {localRefCount2, submittedTaskRefCount2})); + private void checkRefCounts( + ObjectId objectId1, + long localRefCount1, + long submittedTaskRefCount1, + ObjectId objectId2, + long localRefCount2, + long submittedTaskRefCount2) { + checkRefCounts( + ImmutableMap.of( + objectId1, + new long[] {localRefCount1, submittedTaskRefCount1}, + objectId2, + new long[] {localRefCount2, submittedTaskRefCount2})); } private static void fillObjectStoreAndGet(ObjectId objectId, boolean succeed) { fillObjectStoreAndGet(objectId, succeed, 40 * 1024 * 1024, 5); } - private static void fillObjectStoreAndGet(ObjectId objectId, boolean succeed, int objectSize, - int numObjects) { + private static void fillObjectStoreAndGet( + ObjectId objectId, boolean succeed, int objectSize, int numObjects) { for (int i = 0; i < numObjects; i++) { Ray.put(new TestUtils.LargeObject(objectSize)); } @@ -115,9 +124,7 @@ public class ReferenceCountingTest extends BaseTest { } } - /** - * Based on Python test case `test_local_refcounts`. - */ + /** Based on Python test case `test_local_refcounts`. */ public void testLocalRefCounts() { ObjectRefImpl obj1 = (ObjectRefImpl) Ray.put(null); checkRefCounts(obj1.getId(), 1, 0); @@ -163,19 +170,19 @@ public class ReferenceCountingTest extends BaseTest { del(result); } - /** - * Based on Python test case `test_dependency_refcounts`. - */ + /** Based on Python test case `test_dependency_refcounts`. */ public void testDependencyRefCounts() { { // Test that regular plasma dependency refcounts are decremented once the // task finishes. ActorHandle signal = SignalActor.create(); - ObjectRefImpl largeDep = (ObjectRefImpl) Ray - .put(new TestUtils.LargeObject()); - ObjectRefImpl result = (ObjectRefImpl) - Ray., Object>task( - ReferenceCountingTest::oneDep, largeDep, signal).remote(); + ObjectRefImpl largeDep = + (ObjectRefImpl) Ray.put(new TestUtils.LargeObject()); + ObjectRefImpl result = + (ObjectRefImpl) + Ray., Object>task( + ReferenceCountingTest::oneDep, largeDep, signal) + .remote(); checkRefCounts(largeDep.getId(), 1, 1, result.getId(), 1, 0); sendSignal(signal); // Reference count should be removed once the task finishes. @@ -189,12 +196,15 @@ public class ReferenceCountingTest extends BaseTest { // Test that inlined dependency refcounts are decremented once they are // inlined. ActorHandle signal = SignalActor.create(); - ObjectRefImpl dep = (ObjectRefImpl) - Ray., Integer>task(ReferenceCountingTest::oneDep, - Integer.valueOf(1), signal).remote(); + ObjectRefImpl dep = + (ObjectRefImpl) + Ray., Integer>task( + ReferenceCountingTest::oneDep, Integer.valueOf(1), signal) + .remote(); checkRefCounts(dep.getId(), 1, 0); - ObjectRefImpl result = (ObjectRefImpl) - Ray.task(ReferenceCountingTest::oneDep, dep).remote(); + ObjectRefImpl result = + (ObjectRefImpl) + Ray.task(ReferenceCountingTest::oneDep, dep).remote(); checkRefCounts(dep.getId(), 1, 1, result.getId(), 1, 0); sendSignal(signal); // Reference count should be removed as soon as the dependency is inlined. @@ -209,13 +219,17 @@ public class ReferenceCountingTest extends BaseTest { // the task finishes. ActorHandle signal1 = SignalActor.create(); ActorHandle signal2 = SignalActor.create(); - ObjectRefImpl dep = (ObjectRefImpl) - Ray., TestUtils.LargeObject>task( - ReferenceCountingTest::oneDepLarge, (TestUtils.LargeObject) null, signal1).remote(); + ObjectRefImpl dep = + (ObjectRefImpl) + Ray., TestUtils.LargeObject>task( + ReferenceCountingTest::oneDepLarge, (TestUtils.LargeObject) null, signal1) + .remote(); checkRefCounts(dep.getId(), 1, 0); - ObjectRefImpl result = (ObjectRefImpl) - Ray., Integer>task( - ReferenceCountingTest::oneDep, dep, signal2).remote(); + ObjectRefImpl result = + (ObjectRefImpl) + Ray., Integer>task( + ReferenceCountingTest::oneDep, dep, signal2) + .remote(); checkRefCounts(dep.getId(), 1, 1, result.getId(), 1, 0); sendSignal(signal1); dep.get(); // TODO(kfstorm): timeout=10 @@ -233,11 +247,14 @@ public class ReferenceCountingTest extends BaseTest { // Test that regular plasma dependency refcounts are decremented if a task // fails. ActorHandle signal = SignalActor.create(); - ObjectRefImpl largeDep = (ObjectRefImpl) - Ray.put(new TestUtils.LargeObject(10 * 1024 * 1024)); - ObjectRefImpl result = (ObjectRefImpl) - Ray., Boolean, Integer>task( - ReferenceCountingTest::oneDep, largeDep, signal, /* fail= */true).remote(); + ObjectRefImpl largeDep = + (ObjectRefImpl) + Ray.put(new TestUtils.LargeObject(10 * 1024 * 1024)); + ObjectRefImpl result = + (ObjectRefImpl) + Ray., Boolean, Integer>task( + ReferenceCountingTest::oneDep, largeDep, signal, /* fail= */ true) + .remote(); checkRefCounts(largeDep.getId(), 1, 1, result.getId(), 1, 0); sendSignal(signal); // Reference count should be removed once the task finishes. @@ -252,13 +269,17 @@ public class ReferenceCountingTest extends BaseTest { // fails. ActorHandle signal1 = SignalActor.create(); ActorHandle signal2 = SignalActor.create(); - ObjectRefImpl dep = (ObjectRefImpl) - Ray., TestUtils.LargeObject>task( - ReferenceCountingTest::oneDepLarge, (Integer) null, signal1).remote(); + ObjectRefImpl dep = + (ObjectRefImpl) + Ray., TestUtils.LargeObject>task( + ReferenceCountingTest::oneDepLarge, (Integer) null, signal1) + .remote(); checkRefCounts(dep.getId(), 1, 0); - ObjectRefImpl result = (ObjectRefImpl) - Ray., Boolean, Integer>task( - ReferenceCountingTest::oneDep, dep, signal2, /* fail= */true).remote(); + ObjectRefImpl result = + (ObjectRefImpl) + Ray., Boolean, Integer>task( + ReferenceCountingTest::oneDep, dep, signal2, /* fail= */ true) + .remote(); checkRefCounts(dep.getId(), 1, 1, result.getId(), 1, 0); sendSignal(signal1); dep.get(); // TODO(kfstorm): timeout=10 @@ -292,9 +313,7 @@ public class ReferenceCountingTest extends BaseTest { } } - /** - * Based on Python test case `test_basic_pinning`. - */ + /** Based on Python test case `test_basic_pinning`. */ public void testBasicPinning() { ActorHandle actor = Ray.actor(ActorBasicPinning::new).remote(); @@ -302,9 +321,11 @@ public class ReferenceCountingTest extends BaseTest { // evicted before the long-lived object whose reference is held by // the actor. for (int i = 0; i < 10; i++) { - ObjectRef intermediateResult = Ray - .task(ReferenceCountingTest::fooBasicPinning, new TestUtils.LargeObject(10 * 1024 * 1024)) - .remote(); + ObjectRef intermediateResult = + Ray.task( + ReferenceCountingTest::fooBasicPinning, + new TestUtils.LargeObject(10 * 1024 * 1024)) + .remote(); intermediateResult.get(); } // The ray.get below would fail with only LRU eviction, as the object @@ -316,9 +337,7 @@ public class ReferenceCountingTest extends BaseTest { return null; } - /** - * Based on Python test case `test_pending_task_dependency_pinning`. - */ + /** Based on Python test case `test_pending_task_dependency_pinning`. */ public void testPendingTaskDependencyPinning() { // The object that is ray.put here will go out of scope immediately, so if // pending task dependencies aren't considered, it will be evicted before @@ -326,9 +345,12 @@ public class ReferenceCountingTest extends BaseTest { // store. TestUtils.LargeObject input1 = new TestUtils.LargeObject(40 * 1024 * 1024); ActorHandle signal = SignalActor.create(); - ObjectRef result = Ray - .task(ReferenceCountingTest::pending, input1, signal.task(SignalActor::waitSignal).remote()) - .remote(); + ObjectRef result = + Ray.task( + ReferenceCountingTest::pending, + input1, + signal.task(SignalActor::waitSignal).remote()) + .remote(); for (int i = 0; i < 2; i++) { Ray.put(new TestUtils.LargeObject(40 * 1024 * 1024)); diff --git a/java/test/src/main/java/io/ray/test/ResourcesManagementTest.java b/java/test/src/main/java/io/ray/test/ResourcesManagementTest.java index 780f14087..f945363f5 100644 --- a/java/test/src/main/java/io/ray/test/ResourcesManagementTest.java +++ b/java/test/src/main/java/io/ray/test/ResourcesManagementTest.java @@ -12,9 +12,7 @@ import org.testng.annotations.AfterClass; import org.testng.annotations.BeforeClass; import org.testng.annotations.Test; -/** - * Resources Management Test. - */ +/** Resources Management Test. */ @Test(groups = {"cluster"}) public class ResourcesManagementTest extends BaseTest { @@ -44,16 +42,14 @@ public class ResourcesManagementTest extends BaseTest { public void testMethods() { // This is a case that can satisfy required resources. // The static resources for test are "CPU:4,RES-A:4". - ObjectRef result1 = Ray.task(ResourcesManagementTest::echo, 100) - .setResource("CPU", 4.0) - .remote(); + ObjectRef result1 = + Ray.task(ResourcesManagementTest::echo, 100).setResource("CPU", 4.0).remote(); Assert.assertEquals(100, (int) result1.get()); // This is a case that can't satisfy required resources. // The static resources for test are "CPU:4,RES-A:4". - final ObjectRef result2 = Ray.task(ResourcesManagementTest::echo, 200) - .setResource("CPU", 4.0) - .remote(); + final ObjectRef result2 = + Ray.task(ResourcesManagementTest::echo, 200).setResource("CPU", 4.0).remote(); WaitResult waitResult = Ray.wait(ImmutableList.of(result2), 1, 1000); Assert.assertEquals(1, waitResult.getReady().size()); @@ -72,23 +68,17 @@ public class ResourcesManagementTest extends BaseTest { public void testActors() { // This is a case that can satisfy required resources. // The static resources for test are "CPU:4,RES-A:4". - ActorHandle echo1 = Ray.actor(Echo::new) - .setResource("CPU", 2.0) - .remote(); + ActorHandle echo1 = Ray.actor(Echo::new).setResource("CPU", 2.0).remote(); final ObjectRef result1 = echo1.task(Echo::echo, 100).remote(); Assert.assertEquals(100, (int) result1.get()); // This is a case that can't satisfy required resources. // The static resources for test are "CPU:4,RES-A:4". - ActorHandle echo2 = - Ray.actor(Echo::new) - .setResource("CPU", 8.0) - .remote(); + ActorHandle echo2 = Ray.actor(Echo::new).setResource("CPU", 8.0).remote(); final ObjectRef result2 = echo2.task(Echo::echo, 100).remote(); WaitResult waitResult = Ray.wait(ImmutableList.of(result2), 1, 1000); Assert.assertEquals(0, waitResult.getReady().size()); Assert.assertEquals(1, waitResult.getUnready().size()); } - } diff --git a/java/test/src/main/java/io/ray/test/RuntimeContextTest.java b/java/test/src/main/java/io/ray/test/RuntimeContextTest.java index f8efbef04..8807f07c7 100644 --- a/java/test/src/main/java/io/ray/test/RuntimeContextTest.java +++ b/java/test/src/main/java/io/ray/test/RuntimeContextTest.java @@ -49,8 +49,7 @@ public class RuntimeContextTest extends BaseTest { @Test public void testRuntimeContextInActor() { ActorHandle actor = Ray.actor(RuntimeContextTester::new).remote(); - Assert.assertEquals("ok", - actor.task(RuntimeContextTester::testRuntimeContext, actor.getId()).remote().get()); + Assert.assertEquals( + "ok", actor.task(RuntimeContextTester::testRuntimeContext, actor.getId()).remote().get()); } - } diff --git a/java/test/src/main/java/io/ray/test/SignalActor.java b/java/test/src/main/java/io/ray/test/SignalActor.java index 722c5d578..0f74a0cfa 100644 --- a/java/test/src/main/java/io/ray/test/SignalActor.java +++ b/java/test/src/main/java/io/ray/test/SignalActor.java @@ -25,4 +25,4 @@ public class SignalActor { public static ActorHandle create() { return Ray.actor(SignalActor::new).setMaxConcurrency(2).remote(); } -} \ No newline at end of file +} diff --git a/java/test/src/main/java/io/ray/test/SingleProcessModeTest.java b/java/test/src/main/java/io/ray/test/SingleProcessModeTest.java index af9a4f19a..38af97c07 100644 --- a/java/test/src/main/java/io/ray/test/SingleProcessModeTest.java +++ b/java/test/src/main/java/io/ray/test/SingleProcessModeTest.java @@ -18,8 +18,7 @@ public class SingleProcessModeTest extends BaseTest { private static final int TIMES_TO_CALL_PER_ACTOR = 10; static class MyActor { - public MyActor() { - } + public MyActor() {} public long getThreadId() { return Thread.currentThread().getId(); diff --git a/java/test/src/main/java/io/ray/test/TestProgressListener.java b/java/test/src/main/java/io/ray/test/TestProgressListener.java index 50a62d7a0..1fed5ac21 100644 --- a/java/test/src/main/java/io/ray/test/TestProgressListener.java +++ b/java/test/src/main/java/io/ray/test/TestProgressListener.java @@ -10,23 +10,25 @@ import org.testng.ITestResult; public class TestProgressListener implements IInvokedMethodListener, ITestListener { private String getFullTestName(ITestResult testResult) { - return testResult.getTestClass().getName() + "." - + testResult.getMethod().getMethodName(); + return testResult.getTestClass().getName() + "." + testResult.getMethod().getMethodName(); } private void printInfo(String tag, String content) { System.out.println( - "============ [" + LocalDateTime.now().toString() + "] [" + tag + "] " + content + "============ [" + + LocalDateTime.now().toString() + + "] [" + + tag + + "] " + + content + " ============"); } @Override - public void beforeInvocation(IInvokedMethod method, ITestResult testResult) { - } + public void beforeInvocation(IInvokedMethod method, ITestResult testResult) {} @Override - public void afterInvocation(IInvokedMethod method, ITestResult testResult) { - } + public void afterInvocation(IInvokedMethod method, ITestResult testResult) {} @Override public void onTestStart(ITestResult result) { @@ -58,10 +60,8 @@ public class TestProgressListener implements IInvokedMethodListener, ITestListen } @Override - public void onStart(ITestContext context) { - } + public void onStart(ITestContext context) {} @Override - public void onFinish(ITestContext context) { - } + public void onFinish(ITestContext context) {} } diff --git a/java/test/src/main/java/io/ray/test/TestUtils.java b/java/test/src/main/java/io/ray/test/TestUtils.java index 8075a40c2..8fdcec798 100644 --- a/java/test/src/main/java/io/ray/test/TestUtils.java +++ b/java/test/src/main/java/io/ray/test/TestUtils.java @@ -67,12 +67,12 @@ public class TestUtils { /** * Warm up the cluster to make sure there's at least one idle worker. - *

- * This is needed before calling `wait`. Because, in Travis CI, starting a new worker process + * + *

This is needed before calling `wait`. Because, in Travis CI, starting a new worker process * could be slower than the wait timeout. - *

- * TODO(hchen): We should consider supporting always reversing a certain number of idle workers in - * Raylet's worker pool. + * + *

TODO(hchen): We should consider supporting always reversing a certain number of idle workers + * in Raylet's worker pool. */ public static void warmUpCluster() { ObjectRef obj = Ray.task(TestUtils::hi).remote(); @@ -87,8 +87,8 @@ public class TestUtils { if (Ray.internal() instanceof AbstractRayRuntime) { return (RayRuntimeInternal) Ray.internal(); } - RayRuntimeProxy proxy = (RayRuntimeProxy) (java.lang.reflect.Proxy - .getInvocationHandler(Ray.internal())); + RayRuntimeProxy proxy = + (RayRuntimeProxy) (java.lang.reflect.Proxy.getInvocationHandler(Ray.internal())); return proxy.getRuntimeObject(); } diff --git a/java/test/src/main/java/io/ray/test/WaitTest.java b/java/test/src/main/java/io/ray/test/WaitTest.java index 96ad4a700..ba170040d 100644 --- a/java/test/src/main/java/io/ray/test/WaitTest.java +++ b/java/test/src/main/java/io/ray/test/WaitTest.java @@ -70,5 +70,4 @@ public class WaitTest extends BaseTest { Assert.assertTrue(true); } } - } diff --git a/java/test/src/main/java/io/ray/test/WorkerJvmOptionsTest.java b/java/test/src/main/java/io/ray/test/WorkerJvmOptionsTest.java index e3134627b..dc9114d15 100644 --- a/java/test/src/main/java/io/ray/test/WorkerJvmOptionsTest.java +++ b/java/test/src/main/java/io/ray/test/WorkerJvmOptionsTest.java @@ -18,9 +18,10 @@ public class WorkerJvmOptionsTest extends BaseTest { public void testJvmOptions() { // The whitespaces in following argument are intentionally added to test // that raylet can correctly handle dynamic options with whitespaces. - ActorHandle actor = Ray.actor(Echo::new) - .setJvmOptions(" -Dtest.suffix=suffix -Dtest.suffix1=suffix1 ") - .remote(); + ActorHandle actor = + Ray.actor(Echo::new) + .setJvmOptions(" -Dtest.suffix=suffix -Dtest.suffix1=suffix1 ") + .remote(); ObjectRef obj = actor.task(Echo::getOptions).remote(); Assert.assertEquals(obj.get(), "suffix"); } diff --git a/streaming/java/pom.xml b/streaming/java/pom.xml index 976383e52..380e9e369 100644 --- a/streaming/java/pom.xml +++ b/streaming/java/pom.xml @@ -198,7 +198,14 @@ org.apache.maven.plugins maven-checkstyle-plugin - 3.0.0 + 3.1.0 + + + com.puppycrawl.tools + checkstyle + 8.19 + + validate @@ -220,6 +227,22 @@ false + + com.diffplug.spotless + spotless-maven-plugin + 2.5.0 + + + + **/runtime/generated/**/*.* + + + 1.7 + + + + + org.apache.maven.plugins maven-gpg-plugin @@ -251,6 +274,10 @@ org.apache.maven.plugins maven-source-plugin + + com.diffplug.spotless + spotless-maven-plugin + diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/collector/Collector.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/collector/Collector.java index 1fba29053..ef7c16355 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/collector/Collector.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/collector/Collector.java @@ -9,5 +9,4 @@ package io.ray.streaming.api.collector; public interface Collector { void collect(T value); - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/RuntimeContext.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/RuntimeContext.java index 0ff16330c..637e98f0b 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/RuntimeContext.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/RuntimeContext.java @@ -9,9 +9,7 @@ import io.ray.streaming.state.keystate.state.MapState; import io.ray.streaming.state.keystate.state.ValueState; import java.util.Map; -/** - * Encapsulate the runtime information of a streaming task. - */ +/** Encapsulate the runtime information of a streaming task. */ public interface RuntimeContext { int getTaskId(); @@ -20,14 +18,10 @@ public interface RuntimeContext { int getParallelism(); - /** - * @return config of current function - */ + /** Returns config of current function */ Map getConfig(); - /** - * @return config of the job - */ + /** Returns config of the job */ Map getJobConfig(); Long getCheckpointId(); diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/StreamingContext.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/StreamingContext.java index af97167be..caed688d8 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/StreamingContext.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/context/StreamingContext.java @@ -19,28 +19,20 @@ import java.util.concurrent.atomic.AtomicInteger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Encapsulate the context information of a streaming Job. - */ +/** Encapsulate the context information of a streaming Job. */ public class StreamingContext implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(StreamingContext.class); private transient AtomicInteger idGenerator; - /** - * The sinks of this streaming job. - */ + /** The sinks of this streaming job. */ private List streamSinks; - /** - * The user custom streaming job configuration. - */ + /** The user custom streaming job configuration. */ private Map jobConfig; - /** - * The logic plan. - */ + /** The logic plan. */ private JobGraph jobGraph; private StreamingContext() { @@ -53,9 +45,7 @@ public class StreamingContext implements Serializable { return new StreamingContext(); } - /** - * Construct job DAG, and execute the job. - */ + /** Construct job DAG, and execute the job. */ public void execute(String jobName) { JobGraphBuilder jobGraphBuilder = new JobGraphBuilder(this.streamSinks, jobName); JobGraph originalJobGraph = jobGraphBuilder.build(); @@ -78,8 +68,8 @@ public class StreamingContext implements Serializable { ServiceLoader serviceLoader = ServiceLoader.load(JobClient.class); Iterator iterator = serviceLoader.iterator(); - Preconditions.checkArgument(iterator.hasNext(), - "No JobClient implementation has been provided."); + Preconditions.checkArgument( + iterator.hasNext(), "No JobClient implementation has been provided."); JobClient jobClient = iterator.next(); jobClient.submit(jobGraph, jobConfig); } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/Function.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/Function.java index 82791a622..fbfc4736e 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/Function.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/Function.java @@ -2,31 +2,27 @@ package io.ray.streaming.api.function; import java.io.Serializable; -/** - * Interface of streaming functions. - */ +/** Interface of streaming functions. */ public interface Function extends Serializable { /** - * This method will be called periodically by framework, you should return a a serializable - * object which represents function state, framework will help you to serialize this object, save - * it to storage, and load it back when in fail-over through. - * {@link Function#loadCheckpoint(Serializable)}. + * This method will be called periodically by framework, you should return a a serializable object + * which represents function state, framework will help you to serialize this object, save it to + * storage, and load it back when in fail-over through. {@link + * Function#loadCheckpoint(Serializable)}. * - * @return A serializable object which represents function state. + *

Returns A serializable object which represents function state. */ default Serializable saveCheckpoint() { return null; } /** - * This method will be called by framework when a worker died and been restarted. - * We will pass the last object you returned in {@link Function#saveCheckpoint()} when - * doing checkpoint, you are responsible to load this object back to you function. + * This method will be called by framework when a worker died and been restarted. We will pass the + * last object you returned in {@link Function#saveCheckpoint()} when doing checkpoint, you are + * responsible to load this object back to you function. * * @param checkpointObject the last object you returned in {@link Function#saveCheckpoint()} */ - default void loadCheckpoint(Serializable checkpointObject) { - } - + default void loadCheckpoint(Serializable checkpointObject) {} } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/RichFunction.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/RichFunction.java index 32a3cff70..acf09b75e 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/RichFunction.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/RichFunction.java @@ -20,5 +20,4 @@ public interface RichFunction extends Function { * Tear-down method for the user function which called after the last call to the user function. */ void close(); - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/impl/FilterFunction.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/impl/FilterFunction.java index d60e335a9..877a93ae0 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/impl/FilterFunction.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/impl/FilterFunction.java @@ -14,8 +14,8 @@ public interface FilterFunction extends Function { /** * The filter function that evaluates the predicate. * - * @param value The value to be filtered. - * @return True for values that should be retained, false for values to be filtered out. + * @param value The value to be filtered. Returns True for values that should be retained, false + * for values to be filtered out. */ boolean filter(T value) throws Exception; } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/impl/JoinFunction.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/impl/JoinFunction.java index e851da194..a45b6f46c 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/impl/JoinFunction.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/impl/JoinFunction.java @@ -13,5 +13,4 @@ import io.ray.streaming.api.function.Function; public interface JoinFunction extends Function { R join(T left, O right); - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/impl/SourceFunction.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/impl/SourceFunction.java index 96900d841..92c227b7f 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/impl/SourceFunction.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/impl/SourceFunction.java @@ -18,6 +18,5 @@ public interface SourceFunction extends Function { interface SourceContext { void collect(T element) throws Exception; - } } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/internal/CollectionSourceFunction.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/internal/CollectionSourceFunction.java index ec63b7d7e..a1bb38412 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/internal/CollectionSourceFunction.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/internal/CollectionSourceFunction.java @@ -18,8 +18,7 @@ public class CollectionSourceFunction implements SourceFunction { } @Override - public void init(int totalParallel, int currentIndex) { - } + public void init(int totalParallel, int currentIndex) {} @Override public void fetch(SourceContext ctx) throws Exception { @@ -33,7 +32,5 @@ public class CollectionSourceFunction implements SourceFunction { } @Override - public void close() { - } - + public void close() {} } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/internal/Functions.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/internal/Functions.java index 0a8365315..784176fc9 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/internal/Functions.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/function/internal/Functions.java @@ -4,9 +4,7 @@ import io.ray.streaming.api.context.RuntimeContext; import io.ray.streaming.api.function.Function; import io.ray.streaming.api.function.RichFunction; -/** - * A util class for {@link Function} - */ +/** A util class for {@link Function} */ public class Functions { private static class DefaultRichFunction implements RichFunction { @@ -18,12 +16,10 @@ public class Functions { } @Override - public void open(RuntimeContext runtimeContext) { - } + public void open(RuntimeContext runtimeContext) {} @Override - public void close() { - } + public void close() {} public Function getFunction() { return function; @@ -41,5 +37,4 @@ public class Functions { public static RichFunction emptyFunction() { return new DefaultRichFunction(null); } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/partition/Partition.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/partition/Partition.java index 8f9393c3a..527f469c3 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/partition/Partition.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/partition/Partition.java @@ -15,9 +15,8 @@ public interface Partition extends Function { * record. * * @param record The record. - * @param numPartition num of partitions - * @return IDs of the downstream partitions that should receive the record. + * @param numPartition num of partitions Returns IDs of the downstream partitions that should + * receive the record. */ int[] partition(T record, int numPartition); - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/partition/impl/BroadcastPartition.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/partition/impl/BroadcastPartition.java index dd1b38fb2..c34a7fb27 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/partition/impl/BroadcastPartition.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/partition/impl/BroadcastPartition.java @@ -3,15 +3,12 @@ package io.ray.streaming.api.partition.impl; import io.ray.streaming.api.partition.Partition; import java.util.stream.IntStream; -/** - * Broadcast the record to all downstream partitions. - */ +/** Broadcast the record to all downstream partitions. */ public class BroadcastPartition implements Partition { private int[] partitions = new int[0]; - public BroadcastPartition() { - } + public BroadcastPartition() {} @Override public int[] partition(T value, int numPartition) { @@ -20,5 +17,4 @@ public class BroadcastPartition implements Partition { } return partitions; } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/DataStream.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/DataStream.java index 5ca3c7950..698eab29d 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/DataStream.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/DataStream.java @@ -22,6 +22,7 @@ import java.util.List; /** * Represents a stream of data. + * *

This class defines all the streaming operations. * * @param Type of data in the stream. @@ -33,9 +34,7 @@ public class DataStream extends Stream, T> { } public DataStream( - StreamingContext streamingContext, - StreamOperator streamOperator, - Partition partition) { + StreamingContext streamingContext, StreamOperator streamOperator, Partition partition) { super(streamingContext, streamOperator, partition); } @@ -44,9 +43,7 @@ public class DataStream extends Stream, T> { } public DataStream( - DataStream input, - StreamOperator streamOperator, - Partition partition) { + DataStream input, StreamOperator streamOperator, Partition partition) { super(input, streamOperator, partition); } @@ -62,8 +59,7 @@ public class DataStream extends Stream, T> { * Apply a map function to this stream. * * @param mapFunction The map function. - * @param Type of data returned by the map function. - * @return A new DataStream. + * @param Type of data returned by the map function. Returns A new DataStream. */ public DataStream map(MapFunction mapFunction) { return new DataStream<>(this, new MapOperator<>(mapFunction)); @@ -73,8 +69,7 @@ public class DataStream extends Stream, T> { * Apply a flat-map function to this stream. * * @param flatMapFunction The FlatMapFunction - * @param Type of data returned by the flatmap function. - * @return A new DataStream + * @param Type of data returned by the flatmap function. Returns A new DataStream */ public DataStream flatMap(FlatMapFunction flatMapFunction) { return new DataStream<>(this, new FlatMapOperator<>(flatMapFunction)); @@ -89,8 +84,7 @@ public class DataStream extends Stream, T> { * type with each other. * * @param stream The DataStream to union output with. - * @param others The other DataStreams to union output with. - * @return A new UnionStream. + * @param others The other DataStreams to union output with. Returns A new UnionStream. */ @SafeVarargs public final DataStream union(DataStream stream, DataStream... others) { @@ -104,8 +98,7 @@ public class DataStream extends Stream, T> { * Apply union transformations to this stream by merging {@link DataStream} outputs of the same * type with each other. * - * @param streams The DataStreams to union output with. - * @return A new UnionStream. + * @param streams The DataStreams to union output with. Returns A new UnionStream. */ public final DataStream union(List> streams) { if (this instanceof UnionStream) { @@ -122,8 +115,7 @@ public class DataStream extends Stream, T> { * * @param other Another stream. * @param The type of the other stream data. - * @param The type of the data in the joined stream. - * @return A new JoinStream. + * @param The type of the data in the joined stream. Returns A new JoinStream. */ public JoinStream join(DataStream other) { return new JoinStream<>(this, other); @@ -137,8 +129,7 @@ public class DataStream extends Stream, T> { /** * Apply a sink function and get a StreamSink. * - * @param sinkFunction The sink function. - * @return A new StreamSink. + * @param sinkFunction The sink function. Returns A new StreamSink. */ public DataStreamSink sink(SinkFunction sinkFunction) { return new DataStreamSink<>(this, new SinkOperator<>(sinkFunction)); @@ -148,8 +139,7 @@ public class DataStream extends Stream, T> { * Apply a key-by function to this stream. * * @param keyFunction the key function. - * @param The type of the key. - * @return A new KeyDataStream. + * @param The type of the key. Returns A new KeyDataStream. */ public KeyDataStream keyBy(KeyFunction keyFunction) { checkPartitionCall(); @@ -159,7 +149,7 @@ public class DataStream extends Stream, T> { /** * Apply broadcast to this stream. * - * @return This stream. + *

Returns This stream. */ public DataStream broadcast() { checkPartitionCall(); @@ -169,8 +159,7 @@ public class DataStream extends Stream, T> { /** * Apply a partition to this stream. * - * @param partition The partitioning strategy. - * @return This stream. + * @param partition The partitioning strategy. Returns This stream. */ public DataStream partitionBy(Partition partition) { checkPartitionCall(); @@ -183,8 +172,9 @@ public class DataStream extends Stream, T> { */ private void checkPartitionCall() { if (getInputStream() != null && getInputStream().getLanguage() == Language.PYTHON) { - throw new RuntimeException("Partition related methods can't be called on a " + - "java stream if parent stream is a python stream."); + throw new RuntimeException( + "Partition related methods can't be called on a " + + "java stream if parent stream is a python stream."); } } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/DataStreamSource.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/DataStreamSource.java index baecdc339..13de0b33b 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/DataStreamSource.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/DataStreamSource.java @@ -27,12 +27,10 @@ public class DataStreamSource extends DataStream implements StreamSource The type of source data. - * @return A DataStreamSource. + * @param The type of source data. Returns A DataStreamSource. */ public static DataStreamSource fromCollection( StreamingContext context, Collection values) { return new DataStreamSource<>(context, new CollectionSourceFunction<>(values)); } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/JoinStream.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/JoinStream.java index 28d17ccc5..191c0235a 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/JoinStream.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/JoinStream.java @@ -25,9 +25,7 @@ public class JoinStream extends DataStream { return rightStream; } - /** - * Apply key-by to the left join stream. - */ + /** Apply key-by to the left join stream. */ public Where where(KeyFunction keyFunction) { return new Where<>(this, keyFunction); } @@ -64,7 +62,8 @@ public class JoinStream extends DataStream { private KeyFunction rightKeyByFunction; Equal( - JoinStream joinStream, KeyFunction leftKeyByFunction, + JoinStream joinStream, + KeyFunction leftKeyByFunction, KeyFunction rightKeyByFunction) { this.joinStream = joinStream; this.leftKeyByFunction = leftKeyByFunction; @@ -78,5 +77,4 @@ public class JoinStream extends DataStream { return (DataStream) joinStream; } } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/KeyDataStream.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/KeyDataStream.java index cab788ca6..fb6431ef2 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/KeyDataStream.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/KeyDataStream.java @@ -33,8 +33,7 @@ public class KeyDataStream extends DataStream { /** * Apply a reduce function to this stream. * - * @param reduceFunction The reduce function. - * @return A new DataStream. + * @param reduceFunction The reduce function. Returns A new DataStream. */ public DataStream reduce(ReduceFunction reduceFunction) { return new DataStream<>(this, new ReduceOperator(reduceFunction)); @@ -45,8 +44,7 @@ public class KeyDataStream extends DataStream { * * @param aggregateFunction The aggregate function * @param The type of aggregated intermediate data. - * @param The type of result data. - * @return A new DataStream. + * @param The type of result data. Returns A new DataStream. */ public DataStream aggregate(AggregateFunction aggregateFunction) { return new DataStream<>(this, null); @@ -60,5 +58,4 @@ public class KeyDataStream extends DataStream { public PythonKeyDataStream asPythonStream() { return new PythonKeyDataStream(this); } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/Stream.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/Stream.java index ac750b939..cd5d538fd 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/Stream.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/Stream.java @@ -19,8 +19,7 @@ import java.util.Map; * @param Type of stream class * @param Type of the data in the stream. */ -public abstract class Stream, T> - implements Serializable { +public abstract class Stream, T> implements Serializable { private final int id; private final StreamingContext streamingContext; @@ -36,14 +35,15 @@ public abstract class Stream, T> } public Stream( - StreamingContext streamingContext, - StreamOperator streamOperator, - Partition partition) { + StreamingContext streamingContext, StreamOperator streamOperator, Partition partition) { this(streamingContext, null, streamOperator, partition); } public Stream(Stream inputStream, StreamOperator streamOperator) { - this(inputStream.getStreamingContext(), inputStream, streamOperator, + this( + inputStream.getStreamingContext(), + inputStream, + streamOperator, getForwardPartition(streamOperator)); } @@ -87,8 +87,7 @@ public abstract class Stream, T> case JAVA: return new ForwardPartition<>(); default: - throw new UnsupportedOperationException( - "Unsupported language " + operator.getLanguage()); + throw new UnsupportedOperationException("Unsupported language " + operator.getLanguage()); } } @@ -169,18 +168,14 @@ public abstract class Stream, T> return originalStream; } - /** - * Set chain strategy for this stream - */ + /** Set chain strategy for this stream */ public S withChainStrategy(ChainStrategy chainStrategy) { Preconditions.checkArgument(!isProxyStream()); operator.setChainStrategy(chainStrategy); return self(); } - /** - * Disable chain for this stream - */ + /** Disable chain for this stream */ public S disableChain() { return withChainStrategy(ChainStrategy.NEVER); } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/StreamSource.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/StreamSource.java index 692206695..0d21b4f5f 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/StreamSource.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/StreamSource.java @@ -5,6 +5,4 @@ package io.ray.streaming.api.stream; * * @param The type of StreamSource data. */ -public interface StreamSource { - -} +public interface StreamSource {} diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/UnionStream.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/UnionStream.java index 6fd83e29b..d2123e1ef 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/UnionStream.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/api/stream/UnionStream.java @@ -6,6 +6,7 @@ import java.util.List; /** * Represents a union DataStream. + * *

This stream does not create a physical operation, it only affects how upstream data are * connected to downstream data. * diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/client/JobClient.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/client/JobClient.java index 445b4bd02..4a67f8c69 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/client/JobClient.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/client/JobClient.java @@ -3,9 +3,7 @@ package io.ray.streaming.client; import io.ray.streaming.jobgraph.JobGraph; import java.util.Map; -/** - * Interface of the job client. - */ +/** Interface of the job client. */ public interface JobClient { /** diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobEdge.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobEdge.java index 1d3e23b68..186ff3b24 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobEdge.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobEdge.java @@ -3,9 +3,7 @@ package io.ray.streaming.jobgraph; import io.ray.streaming.api.partition.Partition; import java.io.Serializable; -/** - * Job edge is connection and partition rules of upstream and downstream execution nodes. - */ +/** Job edge is connection and partition rules of upstream and downstream execution nodes. */ public class JobEdge implements Serializable { private int srcVertexId; @@ -44,7 +42,13 @@ public class JobEdge implements Serializable { @Override public String toString() { - return "Edge(" + "from:" + srcVertexId + "-" + targetVertexId + "-" + this.partition.getClass() + return "Edge(" + + "from:" + + srcVertexId + + "-" + + targetVertexId + + "-" + + this.partition.getClass() + ")"; } } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobGraph.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobGraph.java index 61722f2f1..6e40ee441 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobGraph.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobGraph.java @@ -9,9 +9,7 @@ import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Job graph, the logical plan of streaming job. - */ +/** Job graph, the logical plan of streaming job. */ public class JobGraph implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(JobGraph.class); @@ -30,8 +28,10 @@ public class JobGraph implements Serializable { } public JobGraph( - String jobName, Map jobConfig, - List jobVertices, List jobEdges) { + String jobName, + Map jobConfig, + List jobVertices, + List jobEdges) { this.jobName = jobName; this.jobConfig = jobConfig; this.jobVertices = jobVertices; @@ -43,7 +43,7 @@ public class JobGraph implements Serializable { * Generate direct-graph(made up of a set of vertices and connected by edges) by current job graph * for simple log printing. * - * @return Digraph in string type. + *

Returns Digraph in string type. */ public String generateDigraph() { StringBuilder digraph = new StringBuilder(); @@ -136,5 +136,4 @@ public class JobGraph implements Serializable { LOG.info(jobEdge.toString()); } } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobGraphBuilder.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobGraphBuilder.java index 86e7d241d..7de388c08 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobGraphBuilder.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobGraphBuilder.java @@ -36,8 +36,7 @@ public class JobGraphBuilder { } public JobGraphBuilder( - List streamSinkList, String jobName, - Map jobConfig) { + List streamSinkList, String jobName, Map jobConfig) { this.jobGraph = new JobGraph(jobName, jobConfig); this.streamSinkList = streamSinkList; this.edgeIdGenerator = new AtomicInteger(0); @@ -60,7 +59,8 @@ public class JobGraphBuilder { stream = stream.getOriginalStream(); } StreamOperator streamOperator = stream.getOperator(); - Preconditions.checkArgument(stream.getLanguage() == streamOperator.getLanguage(), + Preconditions.checkArgument( + stream.getLanguage() == streamOperator.getLanguage(), "Reference stream should be skipped."); int vertexId = stream.getId(); int parallelism = stream.getParallelism(); @@ -76,8 +76,8 @@ public class JobGraphBuilder { } else if (stream instanceof StreamSource) { jobVertex = new JobVertex(vertexId, parallelism, VertexType.SOURCE, streamOperator, config); } else if (stream instanceof DataStream || stream instanceof PythonDataStream) { - jobVertex = new JobVertex( - vertexId, parallelism, VertexType.TRANSFORMATION, streamOperator, config); + jobVertex = + new JobVertex(vertexId, parallelism, VertexType.TRANSFORMATION, streamOperator, config); Stream parentStream = stream.getInputStream(); int inputVertexId = parentStream.getId(); JobEdge jobEdge = new JobEdge(inputVertexId, vertexId, parentStream.getPartition()); @@ -114,5 +114,4 @@ public class JobGraphBuilder { private int getEdgeId() { return this.edgeIdGenerator.incrementAndGet(); } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobGraphOptimizer.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobGraphOptimizer.java index 05407722e..7fbb981ca 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobGraphOptimizer.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobGraphOptimizer.java @@ -36,24 +36,32 @@ public class JobGraphOptimizer { public JobGraphOptimizer(JobGraph jobGraph) { this.jobGraph = jobGraph; - vertexMap = jobGraph.getJobVertices().stream() - .collect(Collectors.toMap(JobVertex::getVertexId, Function.identity())); - outputEdgesMap = vertexMap.keySet().stream().collect(Collectors.toMap( - id -> vertexMap.get(id), id -> new HashSet<>(jobGraph.getVertexOutputEdges(id)))); + vertexMap = + jobGraph.getJobVertices().stream() + .collect(Collectors.toMap(JobVertex::getVertexId, Function.identity())); + outputEdgesMap = + vertexMap.keySet().stream() + .collect( + Collectors.toMap( + id -> vertexMap.get(id), + id -> new HashSet<>(jobGraph.getVertexOutputEdges(id)))); mergedVertexMap = new HashMap<>(); } public JobGraph optimize() { // Deep-first traverse nodes from source to sink to merge vertices that can be chained // together. - jobGraph.getSourceVertices().forEach(vertex -> { - List verticesToMerge = new ArrayList<>(); - verticesToMerge.add(vertex); - mergeVerticesRecursively(vertex, verticesToMerge); - }); + jobGraph + .getSourceVertices() + .forEach( + vertex -> { + List verticesToMerge = new ArrayList<>(); + verticesToMerge.add(vertex); + mergeVerticesRecursively(vertex, verticesToMerge); + }); - List vertices = mergedVertexMap.values().stream() - .map(Pair::getLeft).collect(Collectors.toList()); + List vertices = + mergedVertexMap.values().stream().map(Pair::getLeft).collect(Collectors.toList()); return new JobGraph(jobGraph.getJobName(), jobGraph.getJobConfig(), vertices, createEdges()); } @@ -65,18 +73,19 @@ public class JobGraphOptimizer { if (outputEdges.isEmpty()) { mergeAndAddVertex(verticesToMerge); } else { - outputEdges.forEach(edge -> { - JobVertex succeedingVertex = vertexMap.get(edge.getTargetVertexId()); - if (canBeChained(vertex, succeedingVertex, edge)) { - verticesToMerge.add(succeedingVertex); - mergeVerticesRecursively(succeedingVertex, verticesToMerge); - } else { - mergeAndAddVertex(verticesToMerge); - List newMergedVertices = new ArrayList<>(); - newMergedVertices.add(succeedingVertex); - mergeVerticesRecursively(succeedingVertex, newMergedVertices); - } - }); + outputEdges.forEach( + edge -> { + JobVertex succeedingVertex = vertexMap.get(edge.getTargetVertexId()); + if (canBeChained(vertex, succeedingVertex, edge)) { + verticesToMerge.add(succeedingVertex); + mergeVerticesRecursively(succeedingVertex, verticesToMerge); + } else { + mergeAndAddVertex(verticesToMerge); + List newMergedVertices = new ArrayList<>(); + newMergedVertices.add(succeedingVertex); + mergeVerticesRecursively(succeedingVertex, newMergedVertices); + } + }); } } } @@ -89,25 +98,30 @@ public class JobGraphOptimizer { // no chain mergedVertex = headVertex; } else { - List operators = verticesToMerge.stream() - .map(v -> vertexMap.get(v.getVertexId()) - .getStreamOperator()) - .collect(Collectors.toList()); - List> configs = verticesToMerge.stream() - .map(v -> vertexMap.get(v.getVertexId()).getConfig()) - .collect(Collectors.toList()); + List operators = + verticesToMerge.stream() + .map(v -> vertexMap.get(v.getVertexId()).getStreamOperator()) + .collect(Collectors.toList()); + List> configs = + verticesToMerge.stream() + .map(v -> vertexMap.get(v.getVertexId()).getConfig()) + .collect(Collectors.toList()); StreamOperator operator; if (language == Language.JAVA) { operator = ChainedOperator.newChainedOperator(operators, configs); } else { - List pythonOperators = operators.stream() - .map(o -> (PythonOperator) o) - .collect(Collectors.toList()); + List pythonOperators = + operators.stream().map(o -> (PythonOperator) o).collect(Collectors.toList()); operator = new ChainedPythonOperator(pythonOperators, configs); } // chained operator config is placed into `ChainedOperator`. - mergedVertex = new JobVertex(headVertex.getVertexId(), headVertex.getParallelism(), - headVertex.getVertexType(), operator, new HashMap<>()); + mergedVertex = + new JobVertex( + headVertex.getVertexId(), + headVertex.getParallelism(), + headVertex.getVertexType(), + operator, + new HashMap<>()); } mergedVertexMap.put(mergedVertex.getVertexId(), Pair.of(mergedVertex, verticesToMerge)); @@ -115,37 +129,39 @@ public class JobGraphOptimizer { private List createEdges() { List edges = new ArrayList<>(); - mergedVertexMap.forEach((id, pair) -> { - JobVertex mergedVertex = pair.getLeft(); - List mergedVertices = pair.getRight(); - JobVertex tailVertex = mergedVertices.get(mergedVertices.size() - 1); - // input edge will be set up in input vertices - if (outputEdgesMap.containsKey(tailVertex)) { - outputEdgesMap.get(tailVertex).forEach(edge -> { - Pair> downstreamPair = - mergedVertexMap.get(edge.getTargetVertexId()); - // change ForwardPartition to RoundRobinPartition. - Partition partition = changePartition(edge.getPartition()); - JobEdge newEdge = new JobEdge( - mergedVertex.getVertexId(), - downstreamPair.getLeft().getVertexId(), - partition); - edges.add(newEdge); + mergedVertexMap.forEach( + (id, pair) -> { + JobVertex mergedVertex = pair.getLeft(); + List mergedVertices = pair.getRight(); + JobVertex tailVertex = mergedVertices.get(mergedVertices.size() - 1); + // input edge will be set up in input vertices + if (outputEdgesMap.containsKey(tailVertex)) { + outputEdgesMap + .get(tailVertex) + .forEach( + edge -> { + Pair> downstreamPair = + mergedVertexMap.get(edge.getTargetVertexId()); + // change ForwardPartition to RoundRobinPartition. + Partition partition = changePartition(edge.getPartition()); + JobEdge newEdge = + new JobEdge( + mergedVertex.getVertexId(), + downstreamPair.getLeft().getVertexId(), + partition); + edges.add(newEdge); + }); + } }); - - } - }); return edges; } - /** - * Change ForwardPartition to RoundRobinPartition. - */ + /** Change ForwardPartition to RoundRobinPartition. */ private Partition changePartition(Partition partition) { if (partition instanceof PythonPartition) { PythonPartition pythonPartition = (PythonPartition) partition; - if (!pythonPartition.isConstructedFromBinary() && - pythonPartition.getFunctionName().equals(PythonPartition.FORWARD_PARTITION_CLASS)) { + if (!pythonPartition.isConstructedFromBinary() + && pythonPartition.getFunctionName().equals(PythonPartition.FORWARD_PARTITION_CLASS)) { return PythonPartition.RoundRobinPartition; } else { return partition; @@ -160,11 +176,9 @@ public class JobGraphOptimizer { } private boolean canBeChained( - JobVertex precedingVertex, - JobVertex succeedingVertex, - JobEdge edge) { - if (jobGraph.getVertexOutputEdges(precedingVertex.getVertexId()).size() > 1 || - jobGraph.getVertexInputEdges(succeedingVertex.getVertexId()).size() > 1) { + JobVertex precedingVertex, JobVertex succeedingVertex, JobEdge edge) { + if (jobGraph.getVertexOutputEdges(precedingVertex.getVertexId()).size() > 1 + || jobGraph.getVertexInputEdges(succeedingVertex.getVertexId()).size() > 1) { return false; } if (precedingVertex.getParallelism() != succeedingVertex.getParallelism()) { @@ -183,9 +197,8 @@ public class JobGraphOptimizer { return partition instanceof ForwardPartition; } else { PythonPartition pythonPartition = (PythonPartition) partition; - return !pythonPartition.isConstructedFromBinary() && - pythonPartition.getFunctionName().equals(PythonPartition.FORWARD_PARTITION_CLASS); + return !pythonPartition.isConstructedFromBinary() + && pythonPartition.getFunctionName().equals(PythonPartition.FORWARD_PARTITION_CLASS); } } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobVertex.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobVertex.java index dd59ea36d..a8083246e 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobVertex.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/JobVertex.java @@ -6,9 +6,7 @@ import io.ray.streaming.operator.StreamOperator; import java.io.Serializable; import java.util.Map; -/** - * Job vertex is a cell node where logic is executed. - */ +/** Job vertex is a cell node where logic is executed. */ public class JobVertex implements Serializable { private int vertexId; @@ -71,5 +69,4 @@ public class JobVertex implements Serializable { .add("config", config) .toString(); } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/VertexType.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/VertexType.java index c99af2fcc..90dc36346 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/VertexType.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/jobgraph/VertexType.java @@ -1,8 +1,6 @@ package io.ray.streaming.jobgraph; -/** - * Different roles for a node. - */ +/** Different roles for a node. */ public enum VertexType { SOURCE, TRANSFORMATION, diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/message/Record.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/message/Record.java index fb1ee2636..6f2093205 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/message/Record.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/message/Record.java @@ -37,8 +37,7 @@ public class Record implements Serializable { return false; } Record record = (Record) o; - return Objects.equals(stream, record.stream) && - Objects.equals(value, record.value); + return Objects.equals(stream, record.stream) && Objects.equals(value, record.value); } @Override @@ -50,5 +49,4 @@ public class Record implements Serializable { public String toString() { return value.toString(); } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/ChainStrategy.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/ChainStrategy.java index 9a0084b10..275ebd8fc 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/ChainStrategy.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/ChainStrategy.java @@ -1,20 +1,14 @@ package io.ray.streaming.operator; -/** - * Chain strategy for streaming operators. Chained operators are run in the same thread. - */ +/** Chain strategy for streaming operators. Chained operators are run in the same thread. */ public enum ChainStrategy { /** * The operator won't be chained with preceding operators, but maybe chained with succeeding * operators. */ HEAD, - /** - * Operators will be chained together when possible. - */ + /** Operators will be chained together when possible. */ ALWAYS, - /** - * The operator won't be chained with any operator. - */ + /** The operator won't be chained with any operator. */ NEVER } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/Operator.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/Operator.java index d054b95a7..3754385e7 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/Operator.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/Operator.java @@ -25,13 +25,9 @@ public interface Operator extends Serializable { ChainStrategy getChainStrategy(); - /** - * See {@link Function#saveCheckpoint()}. - */ + /** See {@link Function#saveCheckpoint()}. */ Serializable saveCheckpoint(); - /** - * See {@link Function#loadCheckpoint(Serializable)}. - */ + /** See {@link Function#loadCheckpoint(Serializable)}. */ void loadCheckpoint(Serializable checkpointObject); } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/SourceOperator.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/SourceOperator.java index 11f35f495..04ba1ded7 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/SourceOperator.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/SourceOperator.java @@ -11,4 +11,4 @@ public interface SourceOperator extends Operator { default OperatorType getOpType() { return OperatorType.SOURCE; } -} \ No newline at end of file +} diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/StreamOperator.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/StreamOperator.java index fda6c5d0e..2c7980ab2 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/StreamOperator.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/StreamOperator.java @@ -42,9 +42,7 @@ public abstract class StreamOperator implements Operator { } @Override - public void finish() { - - } + public void finish() {} @Override public void close() { diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/chain/ChainedOperator.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/chain/ChainedOperator.java index 3a4e32cbb..b907896a8 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/chain/ChainedOperator.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/chain/ChainedOperator.java @@ -20,9 +20,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -/** - * Abstract base class for chained operators. - */ +/** Abstract base class for chained operators. */ public abstract class ChainedOperator extends StreamOperator { protected final List operators; @@ -31,9 +29,10 @@ public abstract class ChainedOperator extends StreamOperator { private final List> configs; public ChainedOperator(List operators, List> configs) { - Preconditions.checkArgument(operators.size() >= 2, - "Need at lease two operators to be chained together"); - operators.stream().skip(1) + Preconditions.checkArgument( + operators.size() >= 2, "Need at lease two operators to be chained together"); + operators.stream() + .skip(1) .forEach(operator -> Preconditions.checkArgument(operator instanceof OneInputOperator)); this.operators = operators; this.configs = configs; @@ -44,10 +43,11 @@ public abstract class ChainedOperator extends StreamOperator { @Override public void open(List collectorList, RuntimeContext runtimeContext) { // Dont' call super.open() as we `open` every operator separately. - List succeedingCollectors = operators.stream().skip(1) - .map(operator -> new ForwardCollector( - (OneInputOperator) operator)) - .collect(Collectors.toList()); + List succeedingCollectors = + operators.stream() + .skip(1) + .map(operator -> new ForwardCollector((OneInputOperator) operator)) + .collect(Collectors.toList()); for (int i = 0; i < operators.size() - 1; i++) { StreamOperator operator = operators.get(i); List forwardCollectors = @@ -70,8 +70,7 @@ public abstract class ChainedOperator extends StreamOperator { @Override public String getName() { - return operators.stream().map(Operator::getName) - .collect(Collectors.joining(" -> ", "[", "]")); + return operators.stream().map(Operator::getName).collect(Collectors.joining(" -> ", "[", "]")); } public List getOperators() { @@ -104,20 +103,21 @@ public abstract class ChainedOperator extends StreamOperator { } private RuntimeContext createRuntimeContext(RuntimeContext runtimeContext, int index) { - return (RuntimeContext) Proxy.newProxyInstance(runtimeContext.getClass().getClassLoader(), - new Class[] {RuntimeContext.class}, - (proxy, method, methodArgs) -> { - if (method.getName().equals("getConfig")) { - return configs.get(index); - } else { - return method.invoke(runtimeContext, methodArgs); - } - }); + return (RuntimeContext) + Proxy.newProxyInstance( + runtimeContext.getClass().getClassLoader(), + new Class[] {RuntimeContext.class}, + (proxy, method, methodArgs) -> { + if (method.getName().equals("getConfig")) { + return configs.get(index); + } else { + return method.invoke(runtimeContext, methodArgs); + } + }); } public static ChainedOperator newChainedOperator( - List operators, - List> configs) { + List operators, List> configs) { switch (operators.get(0).getOpType()) { case SOURCE: return new ChainedSourceOperator(operators, configs); @@ -131,8 +131,7 @@ public abstract class ChainedOperator extends StreamOperator { } } - static class ChainedSourceOperator extends ChainedOperator - implements SourceOperator { + static class ChainedSourceOperator extends ChainedOperator implements SourceOperator { private final SourceOperator sourceOperator; @@ -151,11 +150,9 @@ public abstract class ChainedOperator extends StreamOperator { public SourceContext getSourceContext() { return sourceOperator.getSourceContext(); } - } - static class ChainedOneInputOperator extends ChainedOperator - implements OneInputOperator { + static class ChainedOneInputOperator extends ChainedOperator implements OneInputOperator { private final OneInputOperator inputOperator; @@ -169,7 +166,6 @@ public abstract class ChainedOperator extends StreamOperator { public void processElement(Record record) throws Exception { inputOperator.processElement(record); } - } static class ChainedTwoInputOperator extends ChainedOperator @@ -187,6 +183,5 @@ public abstract class ChainedOperator extends StreamOperator { public void processElement(Record record1, Record record2) { inputOperator.processElement(record1, record2); } - } } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/FilterOperator.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/FilterOperator.java index e6aacee59..f4be1b505 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/FilterOperator.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/FilterOperator.java @@ -5,8 +5,8 @@ import io.ray.streaming.message.Record; import io.ray.streaming.operator.OneInputOperator; import io.ray.streaming.operator.StreamOperator; -public class FilterOperator extends StreamOperator> implements - OneInputOperator { +public class FilterOperator extends StreamOperator> + implements OneInputOperator { public FilterOperator(FilterFunction filterFunction) { super(filterFunction); diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/FlatMapOperator.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/FlatMapOperator.java index 39384a548..0fa6e8182 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/FlatMapOperator.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/FlatMapOperator.java @@ -9,8 +9,8 @@ import io.ray.streaming.operator.OneInputOperator; import io.ray.streaming.operator.StreamOperator; import java.util.List; -public class FlatMapOperator extends StreamOperator> implements - OneInputOperator { +public class FlatMapOperator extends StreamOperator> + implements OneInputOperator { private CollectionCollector collectionCollector; diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/JoinOperator.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/JoinOperator.java index 27f03b523..d5cb600e1 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/JoinOperator.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/JoinOperator.java @@ -15,12 +15,10 @@ import io.ray.streaming.operator.TwoInputOperator; * @param Type of the data in the join key. * @param Type of the data in the joined stream. */ -public class JoinOperator extends StreamOperator> implements - TwoInputOperator { +public class JoinOperator extends StreamOperator> + implements TwoInputOperator { - public JoinOperator() { - - } + public JoinOperator() {} public JoinOperator(JoinFunction function) { super(function); @@ -28,13 +26,10 @@ public class JoinOperator extends StreamOperator record1, Record record2) { - - } + public void processElement(Record record1, Record record2) {} @Override public OperatorType getOpType() { return OperatorType.TWO_INPUT; } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/KeyByOperator.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/KeyByOperator.java index f06b5e3f0..0e1828337 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/KeyByOperator.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/KeyByOperator.java @@ -6,8 +6,8 @@ import io.ray.streaming.message.Record; import io.ray.streaming.operator.OneInputOperator; import io.ray.streaming.operator.StreamOperator; -public class KeyByOperator extends StreamOperator> implements - OneInputOperator { +public class KeyByOperator extends StreamOperator> + implements OneInputOperator { public KeyByOperator(KeyFunction keyFunction) { super(keyFunction); @@ -19,4 +19,3 @@ public class KeyByOperator extends StreamOperator> imple collect(new KeyRecord<>(key, record.getValue())); } } - diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/MapOperator.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/MapOperator.java index 6a4dec5b5..7d6a7e3ef 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/MapOperator.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/MapOperator.java @@ -5,8 +5,8 @@ import io.ray.streaming.message.Record; import io.ray.streaming.operator.OneInputOperator; import io.ray.streaming.operator.StreamOperator; -public class MapOperator extends StreamOperator> implements - OneInputOperator { +public class MapOperator extends StreamOperator> + implements OneInputOperator { public MapOperator(MapFunction mapFunction) { super(mapFunction); diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/ReduceOperator.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/ReduceOperator.java index f7c4e7ce3..8ec8a6378 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/ReduceOperator.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/ReduceOperator.java @@ -12,8 +12,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -public class ReduceOperator extends StreamOperator> implements - OneInputOperator { +public class ReduceOperator extends StreamOperator> + implements OneInputOperator { private Map reduceState; @@ -43,5 +43,4 @@ public class ReduceOperator extends StreamOperator> impl collect(record); } } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/SinkOperator.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/SinkOperator.java index 738f75aec..40808e8e5 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/SinkOperator.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/SinkOperator.java @@ -5,8 +5,8 @@ import io.ray.streaming.message.Record; import io.ray.streaming.operator.OneInputOperator; import io.ray.streaming.operator.StreamOperator; -public class SinkOperator extends StreamOperator> implements - OneInputOperator { +public class SinkOperator extends StreamOperator> + implements OneInputOperator { public SinkOperator(SinkFunction sinkFunction) { super(sinkFunction); diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/SourceOperatorImpl.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/SourceOperatorImpl.java index 120701d88..354ada690 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/SourceOperatorImpl.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/SourceOperatorImpl.java @@ -61,7 +61,5 @@ public class SourceOperatorImpl extends StreamOperator> collector.collect(new Record<>(t)); } } - } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/UnionOperator.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/UnionOperator.java index c3467582f..2b916e808 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/UnionOperator.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/operator/impl/UnionOperator.java @@ -6,8 +6,7 @@ import io.ray.streaming.message.Record; import io.ray.streaming.operator.OneInputOperator; import io.ray.streaming.operator.StreamOperator; -public class UnionOperator extends StreamOperator implements - OneInputOperator { +public class UnionOperator extends StreamOperator implements OneInputOperator { public UnionOperator() { super(Functions.emptyFunction()); @@ -17,5 +16,4 @@ public class UnionOperator extends StreamOperator implements public void processElement(Record record) { collect(record); } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/PythonFunction.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/PythonFunction.java index 5aed7d4d3..87ef76d28 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/PythonFunction.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/PythonFunction.java @@ -7,14 +7,18 @@ import org.apache.commons.lang3.StringUtils; /** * Represents a user defined python function. - *

Python worker can use information in this class to create a function object.

- *

If this object is constructed from serialized python function, - * python worker can deserialize it to create python function directly. If this object is - * constructed from moduleName and className/functionName, python worker will use `importlib` to - * load python function.

- *

If the python data stream api is invoked from python, `function` will be not null.

- *

If the python data stream api is invoked from java, `moduleName` and - * `functionName` will be not null.

+ * + *

Python worker can use information in this class to create a function object. + * + *

If this object is constructed from serialized python function, python worker can deserialize + * it to create python function directly. If this object is constructed from moduleName and + * className/functionName, python worker will use `importlib` to load python function. + * + *

If the python data stream api is invoked from python, `function` will be not null. + * + *

If the python data stream api is invoked from java, `moduleName` and `functionName` will be + * not null. + * *

*/ public class PythonFunction implements Function { @@ -30,9 +34,7 @@ public class PythonFunction implements Function { private String functionInterface; - /** - * @param functionInterface function class name in `ray.streaming.function` module. - */ + /** @param functionInterface function class name in `ray.streaming.function` module. */ FunctionInterface(String functionInterface) { this.functionInterface = functionInterface; } @@ -66,13 +68,10 @@ public class PythonFunction implements Function { * Create a {@link PythonFunction} from a moduleName and streaming function name. * * @param moduleName module name of streaming function. - * @param functionName function name of streaming function. {@code functionName} is the name - * of a + * @param functionName function name of streaming function. {@code functionName} is the name of a * python function, or class name of subclass of `ray.streaming.function.` */ - public PythonFunction( - String moduleName, - String functionName) { + public PythonFunction(String moduleName, String functionName) { Preconditions.checkArgument(StringUtils.isNotBlank(moduleName)); Preconditions.checkArgument(StringUtils.isNotBlank(functionName)); this.function = null; @@ -110,12 +109,13 @@ public class PythonFunction implements Function { @Override public String toString() { - StringJoiner stringJoiner = new StringJoiner(", ", - PythonFunction.class.getSimpleName() + "[", "]"); + StringJoiner stringJoiner = + new StringJoiner(", ", PythonFunction.class.getSimpleName() + "[", "]"); if (function != null) { stringJoiner.add("function=binary function"); } else { - stringJoiner.add("moduleName='" + moduleName + "'") + stringJoiner + .add("moduleName='" + moduleName + "'") .add("functionName='" + functionName + "'"); } stringJoiner.add("functionInterface='" + functionInterface + "'"); diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/PythonOperator.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/PythonOperator.java index 185812f99..729803e18 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/PythonOperator.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/PythonOperator.java @@ -12,9 +12,7 @@ import java.util.Map; import java.util.StringJoiner; import java.util.stream.Collectors; -/** - * Represents a {@link StreamOperator} that wraps python {@link PythonFunction}. - */ +/** Represents a {@link StreamOperator} that wraps python {@link PythonFunction}. */ @SuppressWarnings("unchecked") public class PythonOperator extends StreamOperator { @@ -65,8 +63,10 @@ public class PythonOperator extends StreamOperator { StackTraceElement[] trace = Thread.currentThread().getStackTrace(); Preconditions.checkState(trace.length >= 2); StackTraceElement traceElement = trace[2]; - String msg = String.format("Method %s.%s shouldn't be called.", - traceElement.getClassName(), traceElement.getMethodName()); + String msg = + String.format( + "Method %s.%s shouldn't be called.", + traceElement.getClassName(), traceElement.getMethodName()); throw new UnsupportedOperationException(msg); } @@ -90,13 +90,12 @@ public class PythonOperator extends StreamOperator { @Override public String toString() { - StringJoiner stringJoiner = new StringJoiner(", ", - PythonOperator.class.getSimpleName() + "[", "]"); + StringJoiner stringJoiner = + new StringJoiner(", ", PythonOperator.class.getSimpleName() + "[", "]"); if (function != null) { stringJoiner.add("function='" + function + "'"); } else { - stringJoiner.add("moduleName='" + moduleName + "'") - .add("className='" + className + "'"); + stringJoiner.add("moduleName='" + moduleName + "'").add("className='" + className + "'"); } return stringJoiner.toString(); } @@ -130,7 +129,8 @@ public class PythonOperator extends StreamOperator { @Override public String getName() { - return operators.stream().map(Operator::getName) + return operators.stream() + .map(Operator::getName) .collect(Collectors.joining(" -> ", "[", "]")); } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/PythonPartition.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/PythonPartition.java index 1c4970515..e6d80836f 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/PythonPartition.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/PythonPartition.java @@ -7,25 +7,26 @@ import org.apache.commons.lang3.StringUtils; /** * Represents a python partition function. - *

- * Python worker can create a partition object using information in this PythonPartition. - *

- * If this object is constructed from serialized python partition, python worker can deserialize it - * to create python partition directly. If this object is constructed from moduleName and + * + *

Python worker can create a partition object using information in this PythonPartition. + * + *

If this object is constructed from serialized python partition, python worker can deserialize + * it to create python partition directly. If this object is constructed from moduleName and * className/functionName, python worker will use `importlib` to load python partition function. + * *

*/ public class PythonPartition implements Partition { - public static final PythonPartition BroadcastPartition = new PythonPartition( - "ray.streaming.partition", "BroadcastPartition"); - public static final PythonPartition KeyPartition = new PythonPartition( - "ray.streaming.partition", "KeyPartition"); - public static final PythonPartition RoundRobinPartition = new PythonPartition( - "ray.streaming.partition", "RoundRobinPartition"); + public static final PythonPartition BroadcastPartition = + new PythonPartition("ray.streaming.partition", "BroadcastPartition"); + public static final PythonPartition KeyPartition = + new PythonPartition("ray.streaming.partition", "KeyPartition"); + public static final PythonPartition RoundRobinPartition = + new PythonPartition("ray.streaming.partition", "RoundRobinPartition"); public static final String FORWARD_PARTITION_CLASS = "ForwardPartition"; - public static final PythonPartition ForwardPartition = new PythonPartition( - "ray.streaming.partition", FORWARD_PARTITION_CLASS); + public static final PythonPartition ForwardPartition = + new PythonPartition("ray.streaming.partition", FORWARD_PARTITION_CLASS); private byte[] partition; private String moduleName; @@ -51,8 +52,8 @@ public class PythonPartition implements Partition { @Override public int[] partition(Object record, int numPartition) { - String msg = String.format("partition method of %s shouldn't be called.", - getClass().getSimpleName()); + String msg = + String.format("partition method of %s shouldn't be called.", getClass().getSimpleName()); throw new UnsupportedOperationException(msg); } @@ -74,15 +75,15 @@ public class PythonPartition implements Partition { @Override public String toString() { - StringJoiner stringJoiner = new StringJoiner(", ", - PythonPartition.class.getSimpleName() + "[", "]"); + StringJoiner stringJoiner = + new StringJoiner(", ", PythonPartition.class.getSimpleName() + "[", "]"); if (partition != null) { stringJoiner.add("partition=binary partition"); } else { - stringJoiner.add("moduleName='" + moduleName + "'") + stringJoiner + .add("moduleName='" + moduleName + "'") .add("functionName='" + functionName + "'"); } return stringJoiner.toString(); } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonDataStream.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonDataStream.java index 024148c7e..25b587310 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonDataStream.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonDataStream.java @@ -13,14 +13,10 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -/** - * Represents a stream of data whose transformations will be executed in python. - */ +/** Represents a stream of data whose transformations will be executed in python. */ public class PythonDataStream extends Stream implements PythonStream { - protected PythonDataStream( - StreamingContext streamingContext, - PythonOperator pythonOperator) { + protected PythonDataStream(StreamingContext streamingContext, PythonOperator pythonOperator) { super(streamingContext, pythonOperator); } @@ -36,9 +32,7 @@ public class PythonDataStream extends Stream implement } public PythonDataStream( - PythonDataStream input, - PythonOperator pythonOperator, - Partition partition) { + PythonDataStream input, PythonOperator pythonOperator, Partition partition) { super(input, pythonOperator, partition); } @@ -57,8 +51,7 @@ public class PythonDataStream extends Stream implement /** * Apply a map function to this stream. * - * @param func The python MapFunction. - * @return A new PythonDataStream. + * @param func The python MapFunction. Returns A new PythonDataStream. */ public PythonDataStream map(PythonFunction func) { func.setFunctionInterface(FunctionInterface.MAP_FUNCTION); @@ -72,8 +65,7 @@ public class PythonDataStream extends Stream implement /** * Apply a flat-map function to this stream. * - * @param func The python FlapMapFunction. - * @return A new PythonDataStream + * @param func The python FlapMapFunction. Returns A new PythonDataStream */ public PythonDataStream flatMap(PythonFunction func) { func.setFunctionInterface(FunctionInterface.FLAT_MAP_FUNCTION); @@ -87,9 +79,8 @@ public class PythonDataStream extends Stream implement /** * Apply a filter function to this stream. * - * @param func The python FilterFunction. - * @return A new PythonDataStream that contains only the elements satisfying the given filter - * predicate. + * @param func The python FilterFunction. Returns A new PythonDataStream that contains only the + * elements satisfying the given filter predicate. */ public PythonDataStream filter(PythonFunction func) { func.setFunctionInterface(FunctionInterface.FILTER_FUNCTION); @@ -101,8 +92,7 @@ public class PythonDataStream extends Stream implement * same type with each other. * * @param stream The DataStream to union output with. - * @param others The other DataStreams to union output with. - * @return A new UnionStream. + * @param others The other DataStreams to union output with. Returns A new UnionStream. */ public final PythonDataStream union(PythonDataStream stream, PythonDataStream... others) { List streams = new ArrayList<>(); @@ -115,8 +105,7 @@ public class PythonDataStream extends Stream implement * Apply union transformations to this stream by merging {@link PythonDataStream} outputs of the * same type with each other. * - * @param streams The DataStreams to union output with. - * @return A new UnionStream. + * @param streams The DataStreams to union output with. Returns A new UnionStream. */ public final PythonDataStream union(List streams) { if (this instanceof PythonUnionStream) { @@ -135,8 +124,7 @@ public class PythonDataStream extends Stream implement /** * Apply a sink function and get a StreamSink. * - * @param func The python SinkFunction. - * @return A new StreamSink. + * @param func The python SinkFunction. Returns A new StreamSink. */ public PythonStreamSink sink(PythonFunction func) { func.setFunctionInterface(FunctionInterface.SINK_FUNCTION); @@ -150,8 +138,7 @@ public class PythonDataStream extends Stream implement /** * Apply a key-by function to this stream. * - * @param func the python keyFunction. - * @return A new KeyDataStream. + * @param func the python keyFunction. Returns A new KeyDataStream. */ public PythonKeyDataStream keyBy(PythonFunction func) { checkPartitionCall(); @@ -162,7 +149,7 @@ public class PythonDataStream extends Stream implement /** * Apply broadcast to this stream. * - * @return This stream. + *

Returns This stream. */ public PythonDataStream broadcast() { checkPartitionCall(); @@ -172,8 +159,7 @@ public class PythonDataStream extends Stream implement /** * Apply a partition to this stream. * - * @param partition The partitioning strategy. - * @return This stream. + * @param partition The partitioning strategy. Returns This stream. */ public PythonDataStream partitionBy(PythonPartition partition) { checkPartitionCall(); @@ -186,8 +172,9 @@ public class PythonDataStream extends Stream implement */ private void checkPartitionCall() { if (getInputStream() != null && getInputStream().getLanguage() == Language.JAVA) { - throw new RuntimeException("Partition related methods can't be called on a " + - "python stream if parent stream is a java stream."); + throw new RuntimeException( + "Partition related methods can't be called on a " + + "python stream if parent stream is a java stream."); } } @@ -204,5 +191,4 @@ public class PythonDataStream extends Stream implement public Language getLanguage() { return Language.PYTHON; } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonKeyDataStream.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonKeyDataStream.java index a75325818..8116fd392 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonKeyDataStream.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonKeyDataStream.java @@ -8,9 +8,7 @@ import io.ray.streaming.python.PythonFunction.FunctionInterface; import io.ray.streaming.python.PythonOperator; import io.ray.streaming.python.PythonPartition; -/** - * Represents a python DataStream returned by a key-by operation. - */ +/** Represents a python DataStream returned by a key-by operation. */ @SuppressWarnings("unchecked") public class PythonKeyDataStream extends PythonDataStream implements PythonStream { @@ -33,8 +31,7 @@ public class PythonKeyDataStream extends PythonDataStream implements PythonStrea /** * Apply a reduce function to this stream. * - * @param func The reduce function. - * @return A new DataStream. + * @param func The reduce function. Returns A new DataStream. */ public PythonDataStream reduce(PythonFunction func) { func.setFunctionInterface(FunctionInterface.REDUCE_FUNCTION); @@ -51,5 +48,4 @@ public class PythonKeyDataStream extends PythonDataStream implements PythonStrea public KeyDataStream asJavaStream() { return new KeyDataStream(this); } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonStream.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonStream.java index 808516f67..c89d23ca5 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonStream.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonStream.java @@ -1,8 +1,4 @@ package io.ray.streaming.python.stream; -/** - * A marker interface used to identify all python streams. - */ -public interface PythonStream { - -} +/** A marker interface used to identify all python streams. */ +public interface PythonStream {} diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonStreamSink.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonStreamSink.java index 05b6cd37c..6f30d50ad 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonStreamSink.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonStreamSink.java @@ -4,9 +4,7 @@ import io.ray.streaming.api.Language; import io.ray.streaming.api.stream.StreamSink; import io.ray.streaming.python.PythonOperator; -/** - * Represents a sink of the PythonStream. - */ +/** Represents a sink of the PythonStream. */ public class PythonStreamSink extends StreamSink implements PythonStream { public PythonStreamSink(PythonDataStream input, PythonOperator sinkOperator) { @@ -18,5 +16,4 @@ public class PythonStreamSink extends StreamSink implements PythonStream { public Language getLanguage() { return Language.PYTHON; } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonStreamSource.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonStreamSource.java index 5561e4b05..35af00121 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonStreamSource.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonStreamSource.java @@ -7,9 +7,7 @@ import io.ray.streaming.python.PythonFunction; import io.ray.streaming.python.PythonFunction.FunctionInterface; import io.ray.streaming.python.PythonOperator; -/** - * Represents a source of the PythonStream. - */ +/** Represents a source of the PythonStream. */ public class PythonStreamSource extends PythonDataStream implements StreamSource { private PythonStreamSource(StreamingContext streamingContext, PythonFunction sourceFunction) { @@ -18,10 +16,8 @@ public class PythonStreamSource extends PythonDataStream implements StreamSource } public static PythonStreamSource from( - StreamingContext streamingContext, - PythonFunction sourceFunction) { + StreamingContext streamingContext, PythonFunction sourceFunction) { sourceFunction.setFunctionInterface(FunctionInterface.SOURCE_FUNCTION); return new PythonStreamSource(streamingContext, sourceFunction); } - } diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonUnionStream.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonUnionStream.java index 4e115685a..45c16ab62 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonUnionStream.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/python/stream/PythonUnionStream.java @@ -6,6 +6,7 @@ import java.util.List; /** * Represents a union DataStream. + * *

This stream does not create a physical operation, it only affects how upstream data are * connected to downstream data. */ @@ -16,8 +17,7 @@ public class PythonUnionStream extends PythonDataStream { public PythonUnionStream(PythonDataStream input, List others) { // Union stream does not create a physical operation, so we don't have to set partition // function for it. - super(input, new PythonOperator( - "ray.streaming.operator", "UnionOperator")); + super(input, new PythonOperator("ray.streaming.operator", "UnionOperator")); this.unionStreams = new ArrayList<>(); others.forEach(this::addStream); } @@ -33,4 +33,4 @@ public class PythonUnionStream extends PythonDataStream { public List getUnionStreams() { return unionStreams; } -} \ No newline at end of file +} diff --git a/streaming/java/streaming-api/src/main/java/io/ray/streaming/util/Config.java b/streaming/java/streaming-api/src/main/java/io/ray/streaming/util/Config.java index 8d8ae6099..6238384e8 100644 --- a/streaming/java/streaming-api/src/main/java/io/ray/streaming/util/Config.java +++ b/streaming/java/streaming-api/src/main/java/io/ray/streaming/util/Config.java @@ -18,7 +18,6 @@ public class Config { public static final String READ_TIMEOUT_MS = "read_timeout_ms"; public static final String DEFAULT_READ_TIMEOUT_MS = "10"; - public static final String STREAMING_RING_BUFFER_CAPACITY = "streaming.ring_buffer_capacity"; // write an empty message if there is no data to be written in this // interval. @@ -31,6 +30,4 @@ public class Config { public static final String FLOW_CONTROL_TYPE = "streaming.flow_control_type"; public static final String WRITER_CONSUMED_STEP = "streaming.writer.consumed_step"; public static final String READER_CONSUMED_STEP = "streaming.reader.consumed_step"; - - } diff --git a/streaming/java/streaming-api/src/test/java/io/ray/streaming/api/stream/StreamTest.java b/streaming/java/streaming-api/src/test/java/io/ray/streaming/api/stream/StreamTest.java index dbbfe455c..34a9e44d0 100644 --- a/streaming/java/streaming-api/src/test/java/io/ray/streaming/api/stream/StreamTest.java +++ b/streaming/java/streaming-api/src/test/java/io/ray/streaming/api/stream/StreamTest.java @@ -2,7 +2,6 @@ package io.ray.streaming.api.stream; import static org.testng.Assert.assertEquals; - import io.ray.streaming.api.context.StreamingContext; import io.ray.streaming.operator.impl.MapOperator; import io.ray.streaming.python.stream.PythonDataStream; @@ -14,8 +13,8 @@ public class StreamTest { @Test public void testReferencedDataStream() { - DataStream dataStream = new DataStream(StreamingContext.buildContext(), - new MapOperator(value -> null)); + DataStream dataStream = + new DataStream(StreamingContext.buildContext(), new MapOperator(value -> null)); PythonDataStream pythonDataStream = dataStream.asPythonStream(); DataStream javaStream = pythonDataStream.asJavaStream(); assertEquals(dataStream.getId(), pythonDataStream.getId()); @@ -27,8 +26,8 @@ public class StreamTest { @Test public void testReferencedKeyDataStream() { - DataStream dataStream = new DataStream(StreamingContext.buildContext(), - new MapOperator(value -> null)); + DataStream dataStream = + new DataStream(StreamingContext.buildContext(), new MapOperator(value -> null)); KeyDataStream keyDataStream = dataStream.keyBy(value -> null); PythonKeyDataStream pythonKeyDataStream = keyDataStream.asPythonStream(); KeyDataStream javaKeyDataStream = pythonKeyDataStream.asJavaStream(); @@ -38,4 +37,4 @@ public class StreamTest { assertEquals(keyDataStream.getParallelism(), pythonKeyDataStream.getParallelism()); assertEquals(keyDataStream.getParallelism(), javaKeyDataStream.getParallelism()); } -} \ No newline at end of file +} diff --git a/streaming/java/streaming-api/src/test/java/io/ray/streaming/jobgraph/JobGraphBuilderTest.java b/streaming/java/streaming-api/src/test/java/io/ray/streaming/jobgraph/JobGraphBuilderTest.java index 37015585e..d0eec6549 100644 --- a/streaming/java/streaming-api/src/test/java/io/ray/streaming/jobgraph/JobGraphBuilderTest.java +++ b/streaming/java/streaming-api/src/test/java/io/ray/streaming/jobgraph/JobGraphBuilderTest.java @@ -33,13 +33,12 @@ public class JobGraphBuilderTest { JobVertex sourceVertex = jobVertexList.get(0); Assert.assertEquals(sinkVertex.getVertexType(), VertexType.SINK); Assert.assertEquals(sourceVertex.getVertexType(), VertexType.SOURCE); - } public JobGraph buildDataSyncJobGraph() { StreamingContext streamingContext = StreamingContext.buildContext(); - DataStream dataStream = DataStreamSource.fromCollection(streamingContext, - Lists.newArrayList("a", "b", "c")); + DataStream dataStream = + DataStreamSource.fromCollection(streamingContext, Lists.newArrayList("a", "b", "c")); StreamSink streamSink = dataStream.sink(x -> LOG.info(x)); JobGraphBuilder jobGraphBuilder = new JobGraphBuilder(Lists.newArrayList(streamSink)); @@ -73,10 +72,9 @@ public class JobGraphBuilderTest { public JobGraph buildKeyByJobGraph() { StreamingContext streamingContext = StreamingContext.buildContext(); - DataStream dataStream = DataStreamSource.fromCollection(streamingContext, - Lists.newArrayList("1", "2", "3", "4")); - StreamSink streamSink = dataStream.keyBy(x -> x) - .sink(x -> LOG.info(x)); + DataStream dataStream = + DataStreamSource.fromCollection(streamingContext, Lists.newArrayList("1", "2", "3", "4")); + StreamSink streamSink = dataStream.keyBy(x -> x).sink(x -> LOG.info(x)); JobGraphBuilder jobGraphBuilder = new JobGraphBuilder(Lists.newArrayList(streamSink)); JobGraph jobGraph = jobGraphBuilder.build(); @@ -92,4 +90,4 @@ public class JobGraphBuilderTest { Assert.assertTrue(diGraph.contains("\"1-SourceOperatorImpl\" -> \"2-KeyByOperator\"")); Assert.assertTrue(diGraph.contains("\"2-KeyByOperator\" -> \"3-SinkOperator\"")); } -} \ No newline at end of file +} diff --git a/streaming/java/streaming-api/src/test/java/io/ray/streaming/jobgraph/JobGraphOptimizerTest.java b/streaming/java/streaming-api/src/test/java/io/ray/streaming/jobgraph/JobGraphOptimizerTest.java index 69d46accd..bc0854f9b 100644 --- a/streaming/java/streaming-api/src/test/java/io/ray/streaming/jobgraph/JobGraphOptimizerTest.java +++ b/streaming/java/streaming-api/src/test/java/io/ray/streaming/jobgraph/JobGraphOptimizerTest.java @@ -2,7 +2,6 @@ package io.ray.streaming.jobgraph; import static org.testng.Assert.assertEquals; - import com.google.common.collect.Lists; import io.ray.streaming.api.context.StreamingContext; import io.ray.streaming.api.stream.DataStream; @@ -19,13 +18,14 @@ public class JobGraphOptimizerTest { @Test public void testOptimize() { StreamingContext context = StreamingContext.buildContext(); - DataStream source1 = DataStreamSource.fromCollection(context, - Lists.newArrayList(1, 2, 3)); - DataStream source2 = DataStreamSource.fromCollection(context, - Lists.newArrayList("1", "2", "3")); - DataStream source3 = DataStreamSource.fromCollection(context, - Lists.newArrayList("2", "3", "4")); - source1.filter(x -> x > 1) + DataStream source1 = + DataStreamSource.fromCollection(context, Lists.newArrayList(1, 2, 3)); + DataStream source2 = + DataStreamSource.fromCollection(context, Lists.newArrayList("1", "2", "3")); + DataStream source3 = + DataStreamSource.fromCollection(context, Lists.newArrayList("2", "3", "4")); + source1 + .filter(x -> x > 1) .map(String::valueOf) .union(source2) .join(source3) @@ -44,11 +44,12 @@ public class JobGraphOptimizerTest { @Test public void testOptimizeHybridStream() { StreamingContext context = StreamingContext.buildContext(); - DataStream source1 = DataStreamSource.fromCollection(context, - Lists.newArrayList(1, 2, 3)); - DataStream source2 = DataStreamSource.fromCollection(context, - Lists.newArrayList("1", "2", "3")); - source1.asPythonStream() + DataStream source1 = + DataStreamSource.fromCollection(context, Lists.newArrayList(1, 2, 3)); + DataStream source2 = + DataStreamSource.fromCollection(context, Lists.newArrayList("1", "2", "3")); + source1 + .asPythonStream() .map(pyFunc(1)) .filter(pyFunc(2)) .union(source2.asPythonStream().filter(pyFunc(3)).map(pyFunc(4))) @@ -68,5 +69,4 @@ public class JobGraphOptimizerTest { private PythonFunction pyFunc(int number) { return new PythonFunction("module", "func" + number); } - -} \ No newline at end of file +} diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/client/JobClientImpl.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/client/JobClientImpl.java index 099be886e..4e27fdff5 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/client/JobClientImpl.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/client/JobClientImpl.java @@ -12,9 +12,7 @@ import java.util.Map; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Job client: to submit job from api to runtime. - */ +/** Job client: to submit job from api to runtime. */ public class JobClientImpl implements JobClient { public static final Logger LOG = LoggerFactory.getLogger(JobClientImpl.class); @@ -23,8 +21,11 @@ public class JobClientImpl implements JobClient { @Override public void submit(JobGraph jobGraph, Map jobConfig) { - LOG.info("Submitting job [{}] with job graph [{}] and job config [{}].", - jobGraph.getJobName(), jobGraph, jobConfig); + LOG.info( + "Submitting job [{}] with job graph [{}] and job config [{}].", + jobGraph.getJobName(), + jobGraph, + jobConfig); Map resources = new HashMap<>(); // set job name and id at start @@ -34,14 +35,12 @@ public class JobClientImpl implements JobClient { jobGraph.getJobConfig().putAll(jobConfig); // create job master actor - this.jobMasterActor = Ray.actor(JobMaster::new, jobConfig) - .setResources(resources) - .setMaxRestarts(-1) - .remote(); + this.jobMasterActor = + Ray.actor(JobMaster::new, jobConfig).setResources(resources).setMaxRestarts(-1).remote(); try { - ObjectRef submitResult = jobMasterActor.task(JobMaster::submitJob, - jobMasterActor, jobGraph).remote(); + ObjectRef submitResult = + jobMasterActor.task(JobMaster::submitJob, jobMasterActor, jobGraph).remote(); if (submitResult.get()) { LOG.info("Finish submitting job: {}.", jobGraph.getJobName()); diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/Config.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/Config.java index 06ea3c754..d4bc31610 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/Config.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/Config.java @@ -2,9 +2,5 @@ package io.ray.streaming.runtime.config; import org.aeonbits.owner.Accessible; -/** - * Basic config interface. - */ -public interface Config extends org.aeonbits.owner.Config, Accessible { - -} +/** Basic config interface. */ +public interface Config extends org.aeonbits.owner.Config, Accessible {} diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/StreamingConfig.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/StreamingConfig.java index 591e1e852..e300ef7b4 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/StreamingConfig.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/StreamingConfig.java @@ -3,9 +3,7 @@ package io.ray.streaming.runtime.config; import java.io.Serializable; import java.util.Map; -/** - * Streaming config including general, master and worker part. - */ +/** Streaming config including general, master and worker part. */ public class StreamingConfig implements Serializable { public StreamingMasterConfig masterConfig; @@ -21,5 +19,4 @@ public class StreamingConfig implements Serializable { wholeConfigMap.putAll(workerConfigTemplate.configMap); return wholeConfigMap; } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/StreamingGlobalConfig.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/StreamingGlobalConfig.java index 3f1697149..8497ac648 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/StreamingGlobalConfig.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/StreamingGlobalConfig.java @@ -15,9 +15,7 @@ import org.aeonbits.owner.ConfigFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Streaming general config. May used by both JobMaster and JobWorker. - */ +/** Streaming general config. May used by both JobMaster and JobWorker. */ public class StreamingGlobalConfig implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(StreamingGlobalConfig.class); @@ -65,8 +63,7 @@ public class StreamingGlobalConfig implements Serializable { break; } } - Preconditions.checkArgument(configInterface != null, - "Can not get config interface."); + Preconditions.checkArgument(configInterface != null, "Can not get config interface."); Method[] methods = configInterface.getMethods(); for (Method method : methods) { @@ -78,8 +75,10 @@ public class StreamingGlobalConfig implements Serializable { try { value = method.invoke(config); } catch (Exception e) { - LOG.warn("Can not get value by method invoking for config key: {}. " - + "So use default value instead.", ownerKeyAnnotationValue); + LOG.warn( + "Can not get value by method invoking for config key: {}. " + + "So use default value instead.", + ownerKeyAnnotationValue); String defaultValue = method.getAnnotation(DefaultValue.class).value(); value = defaultValue; } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/StreamingMasterConfig.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/StreamingMasterConfig.java index 38f91d80c..cd66c9735 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/StreamingMasterConfig.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/StreamingMasterConfig.java @@ -7,9 +7,7 @@ import org.aeonbits.owner.ConfigFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Streaming job master config. - */ +/** Streaming job master config. */ public class StreamingMasterConfig extends StreamingGlobalConfig { private static final Logger LOG = LoggerFactory.getLogger(StreamingMasterConfig.class); diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/StreamingWorkerConfig.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/StreamingWorkerConfig.java index f682fd3ea..f644cd771 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/StreamingWorkerConfig.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/StreamingWorkerConfig.java @@ -7,9 +7,7 @@ import org.aeonbits.owner.ConfigFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Streaming job worker specified config. - */ +/** Streaming job worker specified config. */ public class StreamingWorkerConfig extends StreamingGlobalConfig { private static final Logger LOG = LoggerFactory.getLogger(StreamingWorkerConfig.class); @@ -33,5 +31,4 @@ public class StreamingWorkerConfig extends StreamingGlobalConfig { } return result; } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/global/CheckpointConfig.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/global/CheckpointConfig.java index b31bc7d8c..0d57febaf 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/global/CheckpointConfig.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/global/CheckpointConfig.java @@ -3,9 +3,7 @@ package io.ray.streaming.runtime.config.global; import io.ray.streaming.runtime.config.Config; import org.aeonbits.owner.Mutable; -/** - * Configurations for checkpointing. - */ +/** Configurations for checkpointing. */ public interface CheckpointConfig extends Config, Mutable { String CP_INTERVAL_SECS = "streaming.checkpoint.interval.secs"; diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/global/CommonConfig.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/global/CommonConfig.java index 61daa787a..0c555e7c5 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/global/CommonConfig.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/global/CommonConfig.java @@ -2,9 +2,7 @@ package io.ray.streaming.runtime.config.global; import io.ray.streaming.runtime.config.Config; -/** - * Job common config. - */ +/** Job common config. */ public interface CommonConfig extends Config { String JOB_ID = "streaming.job.id"; @@ -13,7 +11,7 @@ public interface CommonConfig extends Config { /** * Ray streaming job id. Non-custom. * - * @return Job id with string type. + *

Returns Job id with string type. */ @DefaultValue(value = "default-job-id") @Key(value = JOB_ID) @@ -22,7 +20,7 @@ public interface CommonConfig extends Config { /** * Ray streaming job name. Non-custom. * - * @return Job name with string type. + *

Returns Job name with string type. */ @DefaultValue(value = "default-job-name") @Key(value = JOB_NAME) diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/global/TransferConfig.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/global/TransferConfig.java index e6ea60d7a..eb4978099 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/global/TransferConfig.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/global/TransferConfig.java @@ -3,42 +3,30 @@ package io.ray.streaming.runtime.config.global; import io.ray.streaming.runtime.config.Config; import io.ray.streaming.runtime.config.types.TransferChannelType; -/** - * Job data transfer config. - */ +/** Job data transfer config. */ public interface TransferConfig extends Config { - /** - * Data transfer channel type, support memory queue and native queue. - */ + /** Data transfer channel type, support memory queue and native queue. */ @DefaultValue(value = "NATIVE_CHANNEL") @Key(value = io.ray.streaming.util.Config.CHANNEL_TYPE) TransferChannelType channelType(); - /** - * Queue size. - */ + /** Queue size. */ @DefaultValue(value = "100000000") @Key(value = io.ray.streaming.util.Config.CHANNEL_SIZE) long channelSize(); - /** - * Return from DataReader.getBundle if only empty message read in this interval. - */ + /** Return from DataReader.getBundle if only empty message read in this interval. */ @DefaultValue(value = "-1") @Key(value = io.ray.streaming.util.Config.TIMER_INTERVAL_MS) long readerTimerIntervalMs(); - /** - * Ring capacity. - */ + /** Ring capacity. */ @DefaultValue(value = "-1") @Key(value = io.ray.streaming.util.Config.STREAMING_RING_BUFFER_CAPACITY) int ringBufferCapacity(); - /** - * Write an empty message if there is no data to be written in this interval. - */ + /** Write an empty message if there is no data to be written in this interval. */ @DefaultValue(value = "-1") @Key(value = io.ray.streaming.util.Config.STREAMING_EMPTY_MESSAGE_INTERVAL) int emptyMsgInterval(); diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/master/ResourceConfig.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/master/ResourceConfig.java index d2a6d0859..21a18e292 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/master/ResourceConfig.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/master/ResourceConfig.java @@ -2,81 +2,54 @@ package io.ray.streaming.runtime.config.master; import io.ray.streaming.runtime.config.Config; -/** - * Job resource management config. - */ +/** Job resource management config. */ public interface ResourceConfig extends Config { - /** - * Number of actors per container. - */ + /** Number of actors per container. */ String MAX_ACTOR_NUM_PER_CONTAINER = "streaming.container.per.max.actor"; - /** - * The interval between detecting ray cluster nodes. - */ + /** The interval between detecting ray cluster nodes. */ String CONTAINER_RESOURCE_CHECk_INTERVAL_SECOND = "streaming.resource.check.interval.second"; - /** - * CPU use by per task. - */ + /** CPU use by per task. */ String TASK_RESOURCE_CPU = "streaming.task.resource.cpu"; - /** - * Memory use by each task - */ + /** Memory use by each task */ String TASK_RESOURCE_MEM = "streaming.task.resource.mem"; - /** - * Whether to enable CPU limit in resource control. - */ + /** Whether to enable CPU limit in resource control. */ String TASK_RESOURCE_CPU_LIMIT_ENABLE = "streaming.task.resource.cpu.limitation.enable"; - /** - * Whether to enable memory limit in resource control. - */ + /** Whether to enable memory limit in resource control. */ String TASK_RESOURCE_MEM_LIMIT_ENABLE = "streaming.task.resource.mem.limitation.enable"; - /** - * Number of cpu per task. - */ + /** Number of cpu per task. */ @DefaultValue(value = "1.0") @Key(value = TASK_RESOURCE_CPU) double taskCpuResource(); - /** - * Memory size used by each task. - */ + /** Memory size used by each task. */ @DefaultValue(value = "2.0") @Key(value = TASK_RESOURCE_MEM) double taskMemResource(); - /** - * Whether to enable CPU limit in resource control. - */ + /** Whether to enable CPU limit in resource control. */ @DefaultValue(value = "false") @Key(value = TASK_RESOURCE_CPU_LIMIT_ENABLE) boolean isTaskCpuResourceLimit(); - /** - * Whether to enable memory limit in resource control. - */ + /** Whether to enable memory limit in resource control. */ @DefaultValue(value = "false") @Key(value = TASK_RESOURCE_MEM_LIMIT_ENABLE) boolean isTaskMemResourceLimit(); - /** - * Number of actors per container. - */ + /** Number of actors per container. */ @DefaultValue(value = "500") @Key(MAX_ACTOR_NUM_PER_CONTAINER) int actorNumPerContainer(); - /** - * The interval between detecting ray cluster nodes. - */ + /** The interval between detecting ray cluster nodes. */ @DefaultValue(value = "1") @Key(value = CONTAINER_RESOURCE_CHECk_INTERVAL_SECOND) long resourceCheckIntervalSecond(); - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/master/SchedulerConfig.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/master/SchedulerConfig.java index 3594e3dfc..bc2fc2bd3 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/master/SchedulerConfig.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/master/SchedulerConfig.java @@ -2,9 +2,7 @@ package io.ray.streaming.runtime.config.master; import io.ray.streaming.runtime.config.Config; -/** - * Configuration for job scheduler. - */ +/** Configuration for job scheduler. */ public interface SchedulerConfig extends Config { String WORKER_INITIATION_WAIT_TIMEOUT_MS = "streaming.scheduler.worker.initiation.timeout.ms"; @@ -13,7 +11,7 @@ public interface SchedulerConfig extends Config { /** * The timeout ms of worker initiation. Default is: 10000ms(10s). * - * @return timeout ms + *

Returns timeout ms */ @Key(WORKER_INITIATION_WAIT_TIMEOUT_MS) @DefaultValue(value = "10000") @@ -22,10 +20,9 @@ public interface SchedulerConfig extends Config { /** * The timeout ms of worker starting. Default is: 10000ms(10s). * - * @return timeout ms + *

Returns timeout ms */ @Key(WORKER_STARTING_WAIT_TIMEOUT_MS) @DefaultValue(value = "10000") int workerStartingWaitTimeoutMs(); - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/types/ContextBackendType.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/types/ContextBackendType.java index 329e88c9a..a9856c85c 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/types/ContextBackendType.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/types/ContextBackendType.java @@ -2,14 +2,10 @@ package io.ray.streaming.runtime.config.types; public enum ContextBackendType { - /** - * Memory type - */ + /** Memory type */ MEMORY("memory", 0), - /** - * Local File - */ + /** Local File */ LOCAL_FILE("local_file", 1); private String name; diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/types/ResourceAssignStrategyType.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/types/ResourceAssignStrategyType.java index 32905742f..05c2f6572 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/types/ResourceAssignStrategyType.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/types/ResourceAssignStrategyType.java @@ -2,9 +2,7 @@ package io.ray.streaming.runtime.config.types; public enum ResourceAssignStrategyType { - /** - * Resource scheduling strategy based on FF(First Fit) algorithm and pipeline. - */ + /** Resource scheduling strategy based on FF(First Fit) algorithm and pipeline. */ PIPELINE_FIRST_STRATEGY("pipeline_first_strategy", 0); private String name; diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/types/TransferChannelType.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/types/TransferChannelType.java index f69458a1a..21436790a 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/types/TransferChannelType.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/types/TransferChannelType.java @@ -1,18 +1,12 @@ package io.ray.streaming.runtime.config.types; -/** - * Data transfer channel type. - */ +/** Data transfer channel type. */ public enum TransferChannelType { - /** - * Memory queue. - */ + /** Memory queue. */ MEMORY_CHANNEL("memory_channel", 0), - /** - * Native queue. - */ + /** Native queue. */ NATIVE_CHANNEL("native_channel", 1); private String value; diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/worker/WorkerInternalConfig.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/worker/WorkerInternalConfig.java index 345485cf6..e2eb62cb4 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/worker/WorkerInternalConfig.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/config/worker/WorkerInternalConfig.java @@ -3,24 +3,18 @@ package io.ray.streaming.runtime.config.worker; import io.ray.streaming.runtime.config.Config; import org.aeonbits.owner.Mutable; -/** - * This worker config is used by JobMaster to define the internal configuration of JobWorker. - */ +/** This worker config is used by JobMaster to define the internal configuration of JobWorker. */ public interface WorkerInternalConfig extends Config, Mutable { String WORKER_NAME_INTERNAL = io.ray.streaming.util.Config.STREAMING_WORKER_NAME; String OP_NAME_INTERNAL = io.ray.streaming.util.Config.STREAMING_OP_NAME; - /** - * The name of the worker inside the system. - */ + /** The name of the worker inside the system. */ @DefaultValue(value = "default-worker-name") @Key(value = WORKER_NAME_INTERNAL) String workerName(); - /** - * Operator name corresponding to worker. - */ + /** Operator name corresponding to worker. */ @DefaultValue(value = "default-worker-op-name") @Key(value = OP_NAME_INTERNAL) String workerOperatorName(); diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/ContextBackend.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/ContextBackend.java index b14cdcbb9..faf870390 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/ContextBackend.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/ContextBackend.java @@ -4,23 +4,22 @@ import io.ray.streaming.runtime.master.JobMaster; import io.ray.streaming.runtime.worker.JobWorker; /** - * This interface is used for storing context of {@link JobWorker} and {@link JobMaster}. - * The checkpoint returned by user function is also saved using this interface. + * This interface is used for storing context of {@link JobWorker} and {@link JobMaster}. The + * checkpoint returned by user function is also saved using this interface. */ public interface ContextBackend { /** * check if key exists in state * - * @return true if exists + *

Returns true if exists */ boolean exists(final String key) throws Exception; /** * get content by key * - * @param key key - * @return the StateBackend + * @param key key Returns the StateBackend */ byte[] get(final String key) throws Exception; @@ -38,5 +37,4 @@ public interface ContextBackend { * @param key key */ void remove(final String key) throws Exception; - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/ContextBackendFactory.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/ContextBackendFactory.java index 2ca96b5de..bb0af08a2 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/ContextBackendFactory.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/ContextBackendFactory.java @@ -9,8 +9,8 @@ public class ContextBackendFactory { public static ContextBackend getContextBackend(final StreamingGlobalConfig config) { ContextBackend contextBackend; - ContextBackendType type = ContextBackendType.valueOf( - config.contextBackendConfig.stateBackendType().toUpperCase()); + ContextBackendType type = + ContextBackendType.valueOf(config.contextBackendConfig.stateBackendType().toUpperCase()); switch (type) { case MEMORY: @@ -24,4 +24,4 @@ public class ContextBackendFactory { } return contextBackend; } -} \ No newline at end of file +} diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/OperatorCheckpointInfo.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/OperatorCheckpointInfo.java index 85bceb13c..644e2d332 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/OperatorCheckpointInfo.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/OperatorCheckpointInfo.java @@ -6,21 +6,17 @@ import java.io.Serializable; import java.util.HashMap; import java.util.Map; -/** - * This data structure contains state information of a task. - */ +/** This data structure contains state information of a task. */ public class OperatorCheckpointInfo implements Serializable { - /** - * key: channel ID, value: offset - */ + /** key: channel ID, value: offset */ public Map inputPoints; + public Map outputPoints; - /** - * a serializable checkpoint returned by processor - */ + /** a serializable checkpoint returned by processor */ public Serializable processorCheckpoint; + public long checkpointId; public OperatorCheckpointInfo() { diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/impl/AtomicFsBackend.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/impl/AtomicFsBackend.java index 96288e281..3baf1578b 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/impl/AtomicFsBackend.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/impl/AtomicFsBackend.java @@ -5,8 +5,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * Achieves an atomic `put` method. - * known issue: if you crashed while write a key at first time, this code will not work. + * Achieves an atomic `put` method. known issue: if you crashed while write a key at first time, + * this code will not work. */ public class AtomicFsBackend extends LocalFileContextBackend { diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/impl/LocalFileContextBackend.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/impl/LocalFileContextBackend.java index 41e180462..cd700590f 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/impl/LocalFileContextBackend.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/impl/LocalFileContextBackend.java @@ -6,16 +6,15 @@ import java.io.File; import org.apache.commons.io.FileUtils; /** - * This context backend uses local file system and doesn't supports failover in cluster. - * But it supports failover in single node. - * This is a pure file system backend which doesn't support atomic writing, please don't use this - * class, instead, use {@link AtomicFsBackend} which extends this class. + * This context backend uses local file system and doesn't supports failover in cluster. But it + * supports failover in single node. This is a pure file system backend which doesn't support atomic + * writing, please don't use this class, instead, use {@link AtomicFsBackend} which extends this + * class. */ public class LocalFileContextBackend implements ContextBackend { private final String rootPath; - public LocalFileContextBackend(ContextBackendConfig config) { rootPath = config.fileStateRootPath(); } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/impl/MemoryContextBackend.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/impl/MemoryContextBackend.java index 0a3723e05..eb9ea1539 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/impl/MemoryContextBackend.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/context/impl/MemoryContextBackend.java @@ -8,8 +8,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * This context backend uses memory and doesn't supports failover. - * Data will be lost after worker died. + * This context backend uses memory and doesn't supports failover. Data will be lost after worker + * died. */ public class MemoryContextBackend implements ContextBackend { diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/collector/OutputCollector.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/collector/OutputCollector.java index 90a8e2b18..877f3c5b3 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/collector/OutputCollector.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/collector/OutputCollector.java @@ -36,13 +36,15 @@ public class OutputCollector implements Collector { this.writer = writer; this.outputQueues = outputChannelIds.stream().map(ChannelId::from).toArray(ChannelId[]::new); this.targetActors = targetActors; - this.targetLanguages = targetActors.stream() - .map(actor -> actor instanceof PyActorHandle ? Language.PYTHON : - Language.JAVA) - .toArray(Language[]::new); + this.targetLanguages = + targetActors.stream() + .map(actor -> actor instanceof PyActorHandle ? Language.PYTHON : Language.JAVA) + .toArray(Language[]::new); this.partition = partition; - LOGGER.debug("OutputCollector constructed, outputChannelIds:{}, partition:{}.", - outputChannelIds, this.partition); + LOGGER.debug( + "OutputCollector constructed, outputChannelIds:{}, partition:{}.", + outputChannelIds, + this.partition); } @Override @@ -76,5 +78,4 @@ public class OutputCollector implements Collector { } } } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/common/AbstractId.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/common/AbstractId.java index f61916bfe..d7c7e4658 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/common/AbstractId.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/common/AbstractId.java @@ -5,9 +5,7 @@ import io.ray.streaming.runtime.core.resource.ContainerId; import java.io.Serializable; import java.util.UUID; -/** - * Streaming system unique identity base class. For example, ${@link ContainerId } - */ +/** Streaming system unique identity base class. For example, ${@link ContainerId } */ public class AbstractId implements Serializable { private UUID id; @@ -27,8 +25,6 @@ public class AbstractId implements Serializable { @Override public String toString() { - return MoreObjects.toStringHelper(this) - .add("id", id) - .toString(); + return MoreObjects.toStringHelper(this).add("id", id).toString(); } } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionEdge.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionEdge.java index b86eafb7e..3903f8adb 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionEdge.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionEdge.java @@ -4,29 +4,19 @@ import com.google.common.base.MoreObjects; import io.ray.streaming.api.partition.Partition; import java.io.Serializable; -/** - * An edge that connects two execution vertices. - */ +/** An edge that connects two execution vertices. */ public class ExecutionEdge implements Serializable { - /** - * The source(upstream) execution vertex. - */ + /** The source(upstream) execution vertex. */ private final ExecutionVertex sourceExecutionVertex; - /** - * The target(downstream) execution vertex. - */ + /** The target(downstream) execution vertex. */ private final ExecutionVertex targetExecutionVertex; - /** - * The partition of current execution edge's execution job edge. - */ + /** The partition of current execution edge's execution job edge. */ private final Partition partition; - /** - * An unique id for execution edge. - */ + /** An unique id for execution edge. */ private final String executionEdgeIndex; public ExecutionEdge( @@ -40,7 +30,8 @@ public class ExecutionEdge implements Serializable { } private String generateExecutionEdgeIndex() { - return sourceExecutionVertex.getExecutionVertexId() + "—" + return sourceExecutionVertex.getExecutionVertexId() + + "—" + targetExecutionVertex.getExecutionVertexId(); } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionGraph.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionGraph.java index dcbf6b1ff..b0d3b522e 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionGraph.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionGraph.java @@ -17,62 +17,36 @@ import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Physical plan. - */ +/** Physical plan. */ public class ExecutionGraph implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(ExecutionGraph.class); - /** - * Name of the job. - */ + /** Name of the job. */ private final String jobName; - /** - * Configuration of the job. - */ + /** Configuration of the job. */ private Map jobConfig; - /** - * Data map for execution job vertex. key: job vertex id. value: execution job vertex. - */ + /** Data map for execution job vertex. key: job vertex id. value: execution job vertex. */ private Map executionJobVertexMap; - /** - * Data map for execution vertex. - * key: execution vertex id. - * value: execution vertex. - */ + /** Data map for execution vertex. key: execution vertex id. value: execution vertex. */ private Map executionVertexMap; - /** - * Data map for execution vertex. - * key: actor id. - * value: execution vertex. - */ + /** Data map for execution vertex. key: actor id. value: execution vertex. */ private Map actorIdExecutionVertexMap; - - /** - * key: channel ID - * value: actors in both sides of this channel - */ + /** key: channel ID value: actors in both sides of this channel */ private Map> channelGroupedActors; - /** - * The max parallelism of the whole graph. - */ + /** The max parallelism of the whole graph. */ private int maxParallelism; - /** - * Build time. - */ + /** Build time. */ private long buildTime; - /** - * A monotonic increasing number, used for vertex's id(immutable). - */ + /** A monotonic increasing number, used for vertex's id(immutable). */ private AtomicInteger executionVertexIdGenerator = new AtomicInteger(0); public ExecutionGraph(String jobName) { @@ -96,10 +70,9 @@ public class ExecutionGraph implements Serializable { this.executionJobVertexMap = executionJobVertexMap; } - /** - * generate relation mappings between actors, execution vertices and channels - * this method must be called after worker actor is set. + * generate relation mappings between actors, execution vertices and channels this method must be + * called after worker actor is set. */ public void generateActorMappings() { LOG.info("Setup queue actors relation."); @@ -107,29 +80,33 @@ public class ExecutionGraph implements Serializable { channelGroupedActors = new HashMap<>(); actorIdExecutionVertexMap = new HashMap<>(); - getAllExecutionVertices().forEach(curVertex -> { + getAllExecutionVertices() + .forEach( + curVertex -> { - // current - actorIdExecutionVertexMap.put(curVertex.getActorId(), curVertex); + // current + actorIdExecutionVertexMap.put(curVertex.getActorId(), curVertex); - // input - List inputEdges = curVertex.getInputEdges(); - inputEdges.forEach(inputEdge -> { - ExecutionVertex inputVertex = inputEdge.getSourceExecutionVertex(); - String channelId = curVertex.getChannelIdByPeerVertex(inputVertex); - addActorToChannelGroupedActors(channelGroupedActors, channelId, - inputVertex.getWorkerActor()); - }); + // input + List inputEdges = curVertex.getInputEdges(); + inputEdges.forEach( + inputEdge -> { + ExecutionVertex inputVertex = inputEdge.getSourceExecutionVertex(); + String channelId = curVertex.getChannelIdByPeerVertex(inputVertex); + addActorToChannelGroupedActors( + channelGroupedActors, channelId, inputVertex.getWorkerActor()); + }); - // output - List outputEdges = curVertex.getOutputEdges(); - outputEdges.forEach(outputEdge -> { - ExecutionVertex outputVertex = outputEdge.getTargetExecutionVertex(); - String channelId = curVertex.getChannelIdByPeerVertex(outputVertex); - addActorToChannelGroupedActors(channelGroupedActors, channelId, - outputVertex.getWorkerActor()); - }); - }); + // output + List outputEdges = curVertex.getOutputEdges(); + outputEdges.forEach( + outputEdge -> { + ExecutionVertex outputVertex = outputEdge.getTargetExecutionVertex(); + String channelId = curVertex.getChannelIdByPeerVertex(outputVertex); + addActorToChannelGroupedActors( + channelGroupedActors, channelId, outputVertex.getWorkerActor()); + }); + }); LOG.debug("Channel grouped actors is: {}.", channelGroupedActors); } @@ -179,7 +156,7 @@ public class ExecutionGraph implements Serializable { /** * Get all execution vertices from current execution graph. * - * @return all execution vertices. + *

Returns all execution vertices. */ public List getAllExecutionVertices() { return executionJobVertexMap.values().stream() @@ -191,7 +168,7 @@ public class ExecutionGraph implements Serializable { /** * Get all execution vertices whose status is 'TO_ADD' from current execution graph. * - * @return all added execution vertices. + *

Returns all added execution vertices. */ public List getAllAddedExecutionVertices() { return executionJobVertexMap.values().stream() @@ -204,8 +181,7 @@ public class ExecutionGraph implements Serializable { /** * Get specified execution vertex from current execution graph by execution vertex id. * - * @param executionVertexId execution vertex id. - * @return the specified execution vertex. + * @param executionVertexId execution vertex id. Returns the specified execution vertex. */ public ExecutionVertex getExecutionVertexByExecutionVertexId(int executionVertexId) { if (executionVertexMap.containsKey(executionVertexId)) { @@ -214,53 +190,46 @@ public class ExecutionGraph implements Serializable { throw new RuntimeException("Vertex " + executionVertexId + " does not exist!"); } - /** * Get specified execution vertex from current execution graph by actor id. * - * @param actorId the actor id of execution vertex. - * @return the specified execution vertex. + * @param actorId the actor id of execution vertex. Returns the specified execution vertex. */ public ExecutionVertex getExecutionVertexByActorId(ActorId actorId) { return actorIdExecutionVertexMap.get(actorId); } - /** * Get specified actor by actor id. * - * @param actorId the actor id of execution vertex. - * @return the specified actor handle. + * @param actorId the actor id of execution vertex. Returns the specified actor handle. */ public Optional getActorById(ActorId actorId) { - return getAllActors().stream() - .filter(actor -> actor.getId().equals(actorId)) - .findFirst(); + return getAllActors().stream().filter(actor -> actor.getId().equals(actorId)).findFirst(); } /** * Get the peer actor in the other side of channelName of a given actor * * @param actor actor in this side - * @param channelName the channel name - * @return the peer actor in the other side + * @param channelName the channel name Returns the peer actor in the other side */ public BaseActorHandle getPeerActor(BaseActorHandle actor, String channelName) { Set set = getActorsByChannelId(channelName); final BaseActorHandle[] res = new BaseActorHandle[1]; - set.forEach(anActor -> { - if (!anActor.equals(actor)) { - res[0] = anActor; - } - }); + set.forEach( + anActor -> { + if (!anActor.equals(actor)) { + res[0] = anActor; + } + }); return res[0]; } /** * Get actors in both sides of a channelId * - * @param channelId the channelId - * @return actors in both sides + * @param channelId the channelId Returns actors in both sides */ public Set getActorsByChannelId(String channelId) { return channelGroupedActors.getOrDefault(channelId, Sets.newHashSet()); @@ -269,7 +238,7 @@ public class ExecutionGraph implements Serializable { /** * Get all actors by graph. * - * @return actor list + *

Returns actor list */ public List getAllActors() { return getActorsFromJobVertices(getExecutionJobVertexList()); @@ -278,12 +247,13 @@ public class ExecutionGraph implements Serializable { /** * Get source actors by graph. * - * @return actor list + *

Returns actor list */ public List getSourceActors() { - List executionJobVertices = getExecutionJobVertexList().stream() - .filter(ExecutionJobVertex::isSourceVertex) - .collect(Collectors.toList()); + List executionJobVertices = + getExecutionJobVertexList().stream() + .filter(ExecutionJobVertex::isSourceVertex) + .collect(Collectors.toList()); return getActorsFromJobVertices(executionJobVertices); } @@ -291,16 +261,16 @@ public class ExecutionGraph implements Serializable { /** * Get transformation and sink actors by graph. * - * @return actor list + *

Returns actor list */ public List getNonSourceActors() { - List executionJobVertices = getExecutionJobVertexList().stream() - .filter(executionJobVertex -> - executionJobVertex - .isTransformationVertex() - || executionJobVertex - .isSinkVertex()) - .collect(Collectors.toList()); + List executionJobVertices = + getExecutionJobVertexList().stream() + .filter( + executionJobVertex -> + executionJobVertex.isTransformationVertex() + || executionJobVertex.isSinkVertex()) + .collect(Collectors.toList()); return getActorsFromJobVertices(executionJobVertices); } @@ -308,12 +278,13 @@ public class ExecutionGraph implements Serializable { /** * Get sink actors by graph. * - * @return actor list + *

Returns actor list */ public List getSinkActors() { - List executionJobVertices = getExecutionJobVertexList().stream() - .filter(ExecutionJobVertex::isSinkVertex) - .collect(Collectors.toList()); + List executionJobVertices = + getExecutionJobVertexList().stream() + .filter(ExecutionJobVertex::isSinkVertex) + .collect(Collectors.toList()); return getActorsFromJobVertices(executionJobVertices); } @@ -321,8 +292,7 @@ public class ExecutionGraph implements Serializable { /** * Get actors according to job vertices. * - * @param executionJobVertices specified job vertices - * @return actor list + * @param executionJobVertices specified job vertices Returns actor list */ public List getActorsFromJobVertices( List executionJobVertices) { @@ -351,9 +321,6 @@ public class ExecutionGraph implements Serializable { } public List getAllActorsId() { - return getAllActors().stream() - .map(BaseActorHandle::getId) - .collect(Collectors.toList()); + return getAllActors().stream().map(BaseActorHandle::getId).collect(Collectors.toList()); } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionJobEdge.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionJobEdge.java index 6aa7936b2..72a6a86b6 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionJobEdge.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionJobEdge.java @@ -5,29 +5,19 @@ import io.ray.streaming.api.partition.Partition; import io.ray.streaming.jobgraph.JobEdge; import java.io.Serializable; -/** - * An edge that connects two execution job vertices. - */ +/** An edge that connects two execution job vertices. */ public class ExecutionJobEdge implements Serializable { - /** - * The source(upstream) execution job vertex. - */ + /** The source(upstream) execution job vertex. */ private final ExecutionJobVertex sourceExecutionJobVertex; - /** - * The target(downstream) execution job vertex. - */ + /** The target(downstream) execution job vertex. */ private final ExecutionJobVertex targetExecutionJobVertex; - /** - * The partition of the execution job edge. - */ + /** The partition of the execution job edge. */ private final Partition partition; - /** - * An unique id for execution job edge. - */ + /** An unique id for execution job edge. */ private final String executionJobEdgeIndex; public ExecutionJobEdge( @@ -41,7 +31,8 @@ public class ExecutionJobEdge implements Serializable { } private String generateExecutionJobEdgeIndex() { - return sourceExecutionJobVertex.getExecutionJobVertexId() + "—" + return sourceExecutionJobVertex.getExecutionJobVertexId() + + "—" + targetExecutionJobVertex.getExecutionJobVertexId(); } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionJobVertex.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionJobVertex.java index b617cc053..0aa426672 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionJobVertex.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionJobVertex.java @@ -18,41 +18,35 @@ import org.aeonbits.owner.ConfigFactory; /** * Physical job vertex. - *

Execution job vertex is the physical form of {@link JobVertex} and - * every execution job vertex is corresponding to a group of {@link ExecutionVertex}. + * + *

Execution job vertex is the physical form of {@link JobVertex} and every execution job vertex + * is corresponding to a group of {@link ExecutionVertex}. */ public class ExecutionJobVertex implements Serializable { - /** - * Unique id. Use {@link JobVertex}'s id directly. - */ + /** Unique id. Use {@link JobVertex}'s id directly. */ private final int executionJobVertexId; /** - * Use jobVertex id and operator(use {@link StreamOperator}'s name) as name. e.g. - * 1-SourceOperator + * Use jobVertex id and operator(use {@link StreamOperator}'s name) as name. e.g. 1-SourceOperator */ private final String executionJobVertexName; + private final StreamOperator streamOperator; private final VertexType vertexType; private final Language language; private final Map jobConfig; private final long buildTime; - /** - * Parallelism of current execution job vertex(operator). - */ + /** Parallelism of current execution job vertex(operator). */ private int parallelism; - /** - * Sub execution vertices of current execution job vertex(operator). - */ + /** Sub execution vertices of current execution job vertex(operator). */ private List executionVertices; - /** - * Input and output edges of current execution job vertex. - */ + /** Input and output edges of current execution job vertex. */ private List inputEdges = new ArrayList<>(); + private List outputEdges = new ArrayList<>(); public ExecutionJobVertex( @@ -61,8 +55,9 @@ public class ExecutionJobVertex implements Serializable { AtomicInteger idGenerator, long buildTime) { this.executionJobVertexId = jobVertex.getVertexId(); - this.executionJobVertexName = generateExecutionJobVertexName( - executionJobVertexId, jobVertex.getStreamOperator().getName()); + this.executionJobVertexName = + generateExecutionJobVertexName( + executionJobVertexId, jobVertex.getStreamOperator().getName()); this.streamOperator = jobVertex.getStreamOperator(); this.vertexType = jobVertex.getVertexType(); this.language = jobVertex.getLanguage(); @@ -77,8 +72,8 @@ public class ExecutionJobVertex implements Serializable { ResourceConfig resourceConfig = ConfigFactory.create(ResourceConfig.class, jobConfig); for (int subIndex = 0; subIndex < parallelism; subIndex++) { - executionVertices.add(new ExecutionVertex( - idGenerator.getAndIncrement(), subIndex, this, resourceConfig)); + executionVertices.add( + new ExecutionVertex(idGenerator.getAndIncrement(), subIndex, this, resourceConfig)); } return executionVertices; } @@ -91,14 +86,14 @@ public class ExecutionJobVertex implements Serializable { Map executionVertexWorkersMap = new HashMap<>(); Preconditions.checkArgument( - executionVertices != null && !executionVertices.isEmpty(), - "Empty execution vertex."); - executionVertices.stream().forEach(vertex -> { - Preconditions.checkArgument( - vertex.getWorkerActor() != null, - "Empty execution vertex worker actor."); - executionVertexWorkersMap.put(vertex.getExecutionVertexId(), vertex.getWorkerActor()); - }); + executionVertices != null && !executionVertices.isEmpty(), "Empty execution vertex."); + executionVertices.stream() + .forEach( + vertex -> { + Preconditions.checkArgument( + vertex.getWorkerActor() != null, "Empty execution vertex worker actor."); + executionVertexWorkersMap.put(vertex.getExecutionVertexId(), vertex.getWorkerActor()); + }); return executionVertexWorkersMap; } @@ -114,7 +109,7 @@ public class ExecutionJobVertex implements Serializable { /** * e.g. 1-SourceOperator * - * @return operator name with index + *

Returns operator name with index */ public String getExecutionJobVertexNameWithIndex() { return executionJobVertexId + "-" + executionJobVertexName; @@ -128,8 +123,7 @@ public class ExecutionJobVertex implements Serializable { return executionVertices; } - public void setExecutionVertices( - List executionVertex) { + public void setExecutionVertices(List executionVertex) { this.executionVertices = executionVertex; } @@ -137,8 +131,7 @@ public class ExecutionJobVertex implements Serializable { return outputEdges; } - public void setOutputEdges( - List outputEdges) { + public void setOutputEdges(List outputEdges) { this.outputEdges = outputEdges; } @@ -146,8 +139,7 @@ public class ExecutionJobVertex implements Serializable { return inputEdges; } - public void setInputEdges( - List inputEdges) { + public void setInputEdges(List inputEdges) { this.inputEdges = inputEdges; } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionVertex.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionVertex.java index 5d6a2556c..f53e9e5d2 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionVertex.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionVertex.java @@ -18,34 +18,25 @@ import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; -/** - * Physical vertex, correspond to {@link ExecutionJobVertex}. - */ +/** Physical vertex, correspond to {@link ExecutionJobVertex}. */ public class ExecutionVertex implements Serializable { - /** - * Unique id for execution vertex. - */ + /** Unique id for execution vertex. */ private final int executionVertexId; - /** - * Immutable field inherited from {@link ExecutionJobVertex}. - */ + /** Immutable field inherited from {@link ExecutionJobVertex}. */ private final int executionJobVertexId; + private final String executionJobVertexName; private final StreamOperator streamOperator; private final VertexType vertexType; private final Language language; private final long buildTime; - /** - * Resource used by ExecutionVertex. - */ + /** Resource used by ExecutionVertex. */ private final Map resource; - /** - * Parallelism of current vertex's operator. - */ + /** Parallelism of current vertex's operator. */ private int parallelism; /** @@ -56,21 +47,15 @@ public class ExecutionVertex implements Serializable { private ExecutionVertexState state = ExecutionVertexState.TO_ADD; - /** - * The id of the container which this vertex's worker actor belongs to. - */ + /** The id of the container which this vertex's worker actor belongs to. */ private ContainerId containerId; private String pid; - /** - * Worker actor handle. - */ + /** Worker actor handle. */ private BaseActorHandle workerActor; - /** - * Op config + job config. - */ + /** Op config + job config. */ private Map workerConfig; private List inputEdges = new ArrayList<>(); @@ -83,7 +68,6 @@ public class ExecutionVertex implements Serializable { private transient List inputActorList; private Map exeVertexChannelMap; - public ExecutionVertex( int globalIndex, int index, @@ -182,8 +166,7 @@ public class ExecutionVertex implements Serializable { return inputEdges; } - public void setInputEdges( - List inputEdges) { + public void setInputEdges(List inputEdges) { this.inputEdges = inputEdges; } @@ -191,8 +174,7 @@ public class ExecutionVertex implements Serializable { return outputEdges; } - public void setOutputEdges( - List outputEdges) { + public void setOutputEdges(List outputEdges) { this.outputEdges = outputEdges; } @@ -279,7 +261,6 @@ public class ExecutionVertex implements Serializable { return inputActorList; } - public String getChannelIdByPeerVertex(ExecutionVertex peerVertex) { if (exeVertexChannelMap == null) { generateActorChannelInfo(); @@ -287,7 +268,6 @@ public class ExecutionVertex implements Serializable { return exeVertexChannelMap.get(peerVertex.getExecutionVertexId()); } - private void generateActorChannelInfo() { inputChannelIdList = new ArrayList<>(); inputActorList = new ArrayList<>(); @@ -297,10 +277,11 @@ public class ExecutionVertex implements Serializable { List inputEdges = getInputEdges(); for (ExecutionEdge edge : inputEdges) { - String channelId = ChannelId.genIdStr( - edge.getSourceExecutionVertex().getExecutionVertexId(), - getExecutionVertexId(), - getBuildTime()); + String channelId = + ChannelId.genIdStr( + edge.getSourceExecutionVertex().getExecutionVertexId(), + getExecutionVertexId(), + getBuildTime()); inputChannelIdList.add(channelId); inputActorList.add(edge.getSourceExecutionVertex().getWorkerActor()); exeVertexChannelMap.put(edge.getSourceExecutionVertex().getExecutionVertexId(), channelId); @@ -308,17 +289,17 @@ public class ExecutionVertex implements Serializable { List outputEdges = getOutputEdges(); for (ExecutionEdge edge : outputEdges) { - String channelId = ChannelId.genIdStr( - getExecutionVertexId(), - edge.getTargetExecutionVertex().getExecutionVertexId(), - getBuildTime()); + String channelId = + ChannelId.genIdStr( + getExecutionVertexId(), + edge.getTargetExecutionVertex().getExecutionVertexId(), + getBuildTime()); outputChannelIdList.add(channelId); outputActorList.add(edge.getTargetExecutionVertex().getWorkerActor()); exeVertexChannelMap.put(edge.getTargetExecutionVertex().getExecutionVertexId(), channelId); } } - private Map generateResources(ResourceConfig resourceConfig) { Map resourceMap = new HashMap<>(); if (resourceConfig.isTaskCpuResourceLimit()) { diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionVertexState.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionVertexState.java index ab14891cf..5227c9886 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionVertexState.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/graph/executiongraph/ExecutionVertexState.java @@ -2,29 +2,19 @@ package io.ray.streaming.runtime.core.graph.executiongraph; import java.io.Serializable; -/** - * Vertex state. - */ +/** Vertex state. */ public enum ExecutionVertexState implements Serializable { - /** - * Vertex(Worker) to be added. - */ + /** Vertex(Worker) to be added. */ TO_ADD(1, "TO_ADD"), - /** - * Vertex(Worker) to be deleted. - */ + /** Vertex(Worker) to be deleted. */ TO_DEL(2, "TO_DEL"), - /** - * Vertex(Worker) is running. - */ + /** Vertex(Worker) is running. */ RUNNING(3, "RUNNING"), - /** - * Unknown status, - */ + /** Unknown status, */ UNKNOWN(-1, "UNKNOWN"); public final int code; @@ -34,5 +24,4 @@ public enum ExecutionVertexState implements Serializable { this.code = code; this.msg = msg; } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/processor/ProcessBuilder.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/processor/ProcessBuilder.java index d189c42c1..971e73243 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/processor/ProcessBuilder.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/processor/ProcessBuilder.java @@ -14,7 +14,9 @@ public class ProcessBuilder { public static StreamProcessor buildProcessor(StreamOperator streamOperator) { OperatorType type = streamOperator.getOpType(); - LOGGER.info("Building StreamProcessor, operator type = {}, operator = {}.", type, + LOGGER.info( + "Building StreamProcessor, operator type = {}, operator = {}.", + type, streamOperator.getClass().getSimpleName()); switch (type) { case SOURCE: diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/processor/Processor.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/processor/Processor.java index 54fe76cd8..c323b968d 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/processor/Processor.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/processor/Processor.java @@ -12,14 +12,10 @@ public interface Processor extends Serializable { void process(T t); - /** - * See {@link Function#saveCheckpoint()}. - */ + /** See {@link Function#saveCheckpoint()}. */ Serializable saveCheckpoint(); - /** - * See {@link Function#loadCheckpoint(Serializable)}. - */ + /** See {@link Function#loadCheckpoint(Serializable)}. */ void loadCheckpoint(Serializable checkpointObject); void close(); diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/processor/SourceProcessor.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/processor/SourceProcessor.java index 1cc721a2a..802ef122b 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/processor/SourceProcessor.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/processor/SourceProcessor.java @@ -24,7 +24,5 @@ public class SourceProcessor extends StreamProcessor availableResources = new HashMap<>(); - /** - * List of {@link ExecutionVertex} ids belong to the container. - */ + /** List of {@link ExecutionVertex} ids belong to the container. */ private List executionVertexIds = new ArrayList<>(); - /** - * Capacity is max actor number could be allocated in the container - */ + /** Capacity is max actor number could be allocated in the container */ private int capacity = 0; - public Container() { - } + public Container() {} public Container( - String address, - UniqueId nodeId, String hostname, - Map availableResources) { + String address, UniqueId nodeId, String hostname, Map availableResources) { this.id = new ContainerId(); this.address = address; @@ -73,11 +56,7 @@ public class Container implements Serializable { public static Container from(NodeInfo nodeInfo) { return new Container( - nodeInfo.nodeAddress, - nodeInfo.nodeId, - nodeInfo.nodeHostname, - nodeInfo.resources - ); + nodeInfo.nodeAddress, nodeInfo.nodeId, nodeInfo.nodeHostname, nodeInfo.resources); } public ContainerId getId() { @@ -112,7 +91,6 @@ public class Container implements Serializable { return capacity; } - public void updateCapacity(int capacity) { LOG.info("Update container capacity, old value: {}, new value: {}.", this.capacity, capacity); this.capacity = capacity; @@ -150,8 +128,10 @@ public class Container implements Serializable { executionVertexIds.removeIf(id -> id == vertex.getExecutionVertexId()); reclaimResource(vertex.getResource()); } else { - throw new RuntimeException(String.format("Current container [%s] not found vertex [%s].", - this, vertex.getExecutionJobVertexName())); + throw new RuntimeException( + String.format( + "Current container [%s] not found vertex [%s].", + this, vertex.getExecutionJobVertexName())); } } @@ -160,24 +140,36 @@ public class Container implements Serializable { } private void decreaseResource(Map allocatedResource) { - allocatedResource.forEach((k, v) -> { - Preconditions.checkArgument(this.availableResources.get(k) >= v, - String.format("Available resource %s not >= decreased resource %s", - this.availableResources.get(k), v)); - Double newValue = this.availableResources.get(k) - v; - LOG.info("Decrease container {} resource [{}], from {} to {}.", - this.address, k, this.availableResources.get(k), newValue); - this.availableResources.put(k, newValue); - }); + allocatedResource.forEach( + (k, v) -> { + Preconditions.checkArgument( + this.availableResources.get(k) >= v, + String.format( + "Available resource %s not >= decreased resource %s", + this.availableResources.get(k), v)); + Double newValue = this.availableResources.get(k) - v; + LOG.info( + "Decrease container {} resource [{}], from {} to {}.", + this.address, + k, + this.availableResources.get(k), + newValue); + this.availableResources.put(k, newValue); + }); } private void reclaimResource(Map allocatedResource) { - allocatedResource.forEach((k, v) -> { - Double newValue = this.availableResources.get(k) + v; - LOG.info("Reclaim container {} resource [{}], from {} to {}.", - this.address, k, this.availableResources.get(k), newValue); - this.availableResources.put(k, newValue); - }); + allocatedResource.forEach( + (k, v) -> { + Double newValue = this.availableResources.get(k) + v; + LOG.info( + "Reclaim container {} resource [{}], from {} to {}.", + this.address, + k, + this.availableResources.get(k), + newValue); + this.availableResources.put(k, newValue); + }); } @Override @@ -192,4 +184,4 @@ public class Container implements Serializable { .add("capacity", capacity) .toString(); } -} \ No newline at end of file +} diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/resource/ContainerId.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/resource/ContainerId.java index 81c2a3f84..a0b08ad2b 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/resource/ContainerId.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/resource/ContainerId.java @@ -2,9 +2,5 @@ package io.ray.streaming.runtime.core.resource; import io.ray.streaming.runtime.core.common.AbstractId; -/** - * Container unique identifier. - */ -public class ContainerId extends AbstractId { - -} +/** Container unique identifier. */ +public class ContainerId extends AbstractId {} diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/resource/ResourceType.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/resource/ResourceType.java index 496266cf2..e6b5d0efc 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/resource/ResourceType.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/resource/ResourceType.java @@ -1,23 +1,15 @@ package io.ray.streaming.runtime.core.resource; -/** - * Key for different type of resources. - */ +/** Key for different type of resources. */ public enum ResourceType { - /** - * Cpu resource key. - */ + /** Cpu resource key. */ CPU("CPU"), - /** - * Gpu resource key. - */ + /** Gpu resource key. */ GPU("GPU"), - /** - * Memory resource key. - */ + /** Memory resource key. */ MEM("MEM"); private String value; @@ -29,5 +21,4 @@ public enum ResourceType { public String getValue() { return value; } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/resource/Resources.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/resource/Resources.java index 3f7a1b6d7..b0dec4aef 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/resource/Resources.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/core/resource/Resources.java @@ -11,25 +11,20 @@ import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * Resource description of ResourceManager. - */ +/** Resource description of ResourceManager. */ public class Resources implements Serializable { private static final Logger LOG = LoggerFactory.getLogger(Resources.class); - /** - * Available containers registered to ResourceManager. - */ + /** Available containers registered to ResourceManager. */ private List registerContainers = new ArrayList<>(); - public Resources() { - } + public Resources() {} /** * Get registered containers, the container list is read-only. * - * @return container list. + *

Returns container list. */ public ImmutableList getRegisteredContainers() { return ImmutableList.copyOf(registerContainers); @@ -52,9 +47,9 @@ public class Resources implements Serializable { } public ImmutableMap getRegisteredContainerMap() { - return ImmutableMap.copyOf(registerContainers.stream() - .collect(java.util.stream.Collectors - .toMap(Container::getNodeId, c -> c))); + return ImmutableMap.copyOf( + registerContainers.stream() + .collect(java.util.stream.Collectors.toMap(Container::getNodeId, c -> c))); } @Override diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/JobMaster.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/JobMaster.java index 87124afdb..2106b4d01 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/JobMaster.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/JobMaster.java @@ -67,8 +67,8 @@ public class JobMaster { runtimeContext = new JobMasterRuntimeContext(streamingConfig); // load checkpoint if is recover - if (!Ray.getRuntimeContext().isSingleProcess() && Ray.getRuntimeContext() - .wasCurrentActorRestarted()) { + if (!Ray.getRuntimeContext().isSingleProcess() + && Ray.getRuntimeContext().wasCurrentActorRestarted()) { loadMasterCheckpoint(); } @@ -101,7 +101,7 @@ public class JobMaster { /** * Init JobMaster. To initiate or recover other components(like metrics and extra coordinators). * - * @return init result + *

Returns init result */ public Boolean init(boolean isRecover) { LOG.info("Initializing job master, isRecover={}.", isRecover); @@ -128,15 +128,15 @@ public class JobMaster { /** * Submit job to run: + * *

    - *
  1. Using GraphManager to build physical plan according to the logical plan.
  2. - *
  3. Using ResourceManager to manage and allocate the resources.
  4. - *
  5. Using JobScheduler to schedule the job to run.
  6. + *
  7. Using GraphManager to build physical plan according to the logical plan. + *
  8. Using ResourceManager to manage and allocate the resources. + *
  9. Using JobScheduler to schedule the job to run. *
* * @param jobMasterActor JobMaster actor - * @param jobGraph logical plan - * @return submit result + * @param jobGraph logical plan Returns submit result */ public boolean submitJob(ActorHandle jobMasterActor, JobGraph jobGraph) { LOG.info("Begin submitting job using logical plan: {}.", jobGraph); @@ -168,8 +168,8 @@ public class JobMaster { LOG.debug("Save JobMaster context."); byte[] contextBytes = Serializer.encode(runtimeContext); - CheckpointStateUtil - .put(contextBackend, getJobMasterRuntimeContextKey(getConf()), contextBytes); + CheckpointStateUtil.put( + contextBackend, getJobMasterRuntimeContextKey(getConf()), contextBytes); } } @@ -180,8 +180,11 @@ public class JobMaster { reportPb = RemoteCall.BaseWorkerCmd.parseFrom(reportBytes); ActorId actorId = ActorId.fromBytes(reportPb.getActorId().toByteArray()); long remoteCallCost = System.currentTimeMillis() - reportPb.getTimestamp(); - LOG.info("Vertex {}, request job worker commit cost {}ms, actorId={}.", - getExecutionVertex(actorId), remoteCallCost, actorId); + LOG.info( + "Vertex {}, request job worker commit cost {}ms, actorId={}.", + getExecutionVertex(actorId), + remoteCallCost, + actorId); RemoteCall.WorkerCommitReport commit = reportPb.getDetail().unpack(RemoteCall.WorkerCommitReport.class); WorkerCommitReport report = new WorkerCommitReport(actorId, commit.getCommitCheckpointId()); @@ -206,27 +209,31 @@ public class JobMaster { return RemoteCall.BoolResult.newBuilder().setBoolRes(false).build().toByteArray(); } ExecutionVertex exeVertex = getExecutionVertex(actorId); - LOG.info("Vertex {}, request job worker rollback cost {}ms, actorId={}.", - exeVertex, remoteCallCost, actorId); - RemoteCall.WorkerRollbackRequest rollbackPb - = RemoteCall.WorkerRollbackRequest.parseFrom(requestPb.getDetail().getValue()); + LOG.info( + "Vertex {}, request job worker rollback cost {}ms, actorId={}.", + exeVertex, + remoteCallCost, + actorId); + RemoteCall.WorkerRollbackRequest rollbackPb = + RemoteCall.WorkerRollbackRequest.parseFrom(requestPb.getDetail().getValue()); exeVertex.setPid(rollbackPb.getWorkerPid()); // To find old container where slot is located in. String hostname = ""; - Optional container = ResourceUtil.getContainerById( - resourceManager.getRegisteredContainers(), - exeVertex.getContainerId() - ); + Optional container = + ResourceUtil.getContainerById( + resourceManager.getRegisteredContainers(), exeVertex.getContainerId()); if (container.isPresent()) { hostname = container.get().getHostname(); } - WorkerRollbackRequest request = new WorkerRollbackRequest( - actorId, rollbackPb.getExceptionMsg(), hostname, exeVertex.getPid() - ); + WorkerRollbackRequest request = + new WorkerRollbackRequest( + actorId, rollbackPb.getExceptionMsg(), hostname, exeVertex.getPid()); ret = failoverCoordinator.requestJobWorkerRollback(request); - LOG.info("Vertex {} request rollback, exception msg : {}.", - exeVertex, rollbackPb.getExceptionMsg()); + LOG.info( + "Vertex {} request rollback, exception msg : {}.", + exeVertex, + rollbackPb.getExceptionMsg()); } catch (Throwable e) { LOG.error("Parse job worker rollback has exception.", e); @@ -257,5 +264,4 @@ public class JobMaster { public StreamingMasterConfig getConf() { return conf; } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/JobRuntimeContext.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/JobRuntimeContext.java index ec10c69ae..a271ff2d1 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/JobRuntimeContext.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/JobRuntimeContext.java @@ -8,6 +8,7 @@ import java.io.Serializable; /** * Runtime context for job master. + * *

Including: graph, resource, checkpoint info, etc. */ public class JobRuntimeContext implements Serializable { @@ -52,5 +53,4 @@ public class JobRuntimeContext implements Serializable { .add("conf", conf.getMap()) .toString(); } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/context/JobMasterRuntimeContext.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/context/JobMasterRuntimeContext.java index c9e6e8f57..1765519ed 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/context/JobMasterRuntimeContext.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/context/JobMasterRuntimeContext.java @@ -77,5 +77,4 @@ public class JobMasterRuntimeContext implements Serializable { .add("conf", conf.getMap()) .toString(); } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/BaseCoordinator.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/BaseCoordinator.java index ece4de4b7..73323da49 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/BaseCoordinator.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/BaseCoordinator.java @@ -25,8 +25,9 @@ public abstract class BaseCoordinator implements Runnable { } public void start() { - thread = new Thread(Ray.wrapRunnable(this), - this.getClass().getName() + "-" + System.currentTimeMillis()); + thread = + new Thread( + Ray.wrapRunnable(this), this.getClass().getName() + "-" + System.currentTimeMillis()); thread.start(); } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/CheckpointCoordinator.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/CheckpointCoordinator.java index 862528776..05064c543 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/CheckpointCoordinator.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/CheckpointCoordinator.java @@ -20,8 +20,8 @@ import org.slf4j.LoggerFactory; /** * CheckpointCoordinator is the controller of checkpoint, responsible for triggering checkpoint, - * collecting {@link JobWorker}'s reports and calling {@link JobWorker} to clear expired - * checkpoints when new checkpoint finished. + * collecting {@link JobWorker}'s reports and calling {@link JobWorker} to clear expired checkpoints + * when new checkpoint finished. */ public class CheckpointCoordinator extends BaseCoordinator { @@ -58,7 +58,8 @@ public class CheckpointCoordinator extends BaseCoordinator { if (!pendingCheckpointActors.isEmpty()) { // if wait commit report timeout, this cp fail, and restart next cp if (timeoutOnWaitCheckpoint()) { - LOG.warn("Waiting for checkpoint {} timeout, pending cp actors is {}.", + LOG.warn( + "Waiting for checkpoint {} timeout, pending cp actors is {}.", runtimeContext.lastCheckpointId, graphManager.getExecutionGraph().getActorName(pendingCheckpointActors)); @@ -90,14 +91,17 @@ public class CheckpointCoordinator extends BaseCoordinator { } private void processCommitReport(WorkerCommitReport commitReport) { - LOG.info("Start process commit report {}, from actor name={}.", commitReport, + LOG.info( + "Start process commit report {}, from actor name={}.", + commitReport, graphManager.getExecutionGraph().getActorName(commitReport.fromActorId)); try { Preconditions.checkArgument( commitReport.commitCheckpointId == runtimeContext.lastCheckpointId, "expect checkpointId %s, but got %s", - runtimeContext.lastCheckpointId, commitReport); + runtimeContext.lastCheckpointId, + commitReport); if (!pendingCheckpointActors.contains(commitReport.fromActorId)) { LOG.warn("Invalid commit report, skipped."); @@ -105,7 +109,8 @@ public class CheckpointCoordinator extends BaseCoordinator { } pendingCheckpointActors.remove(commitReport.fromActorId); - LOG.info("Pending actors after this commit: {}.", + LOG.info( + "Pending actors after this commit: {}.", graphManager.getExecutionGraph().getActorName(pendingCheckpointActors)); // checkpoint finish @@ -144,10 +149,14 @@ public class CheckpointCoordinator extends BaseCoordinator { final List sourcesRet = new ArrayList<>(); - graphManager.getExecutionGraph().getSourceActors().forEach(actor -> { - sourcesRet.add(RemoteCallWorker.triggerCheckpoint( - actor, runtimeContext.lastCheckpointId)); - }); + graphManager + .getExecutionGraph() + .getSourceActors() + .forEach( + actor -> { + sourcesRet.add( + RemoteCallWorker.triggerCheckpoint(actor, runtimeContext.lastCheckpointId)); + }); for (ObjectRef rayObject : sourcesRet) { if (rayObject.get() instanceof RayException) { @@ -171,8 +180,7 @@ public class CheckpointCoordinator extends BaseCoordinator { List allActor = graphManager.getExecutionGraph().getAllActors(); if (runtimeContext.lastCheckpointId > runtimeContext.getLastValidCheckpointId()) { - RemoteCallWorker - .notifyCheckpointTimeoutParallel(allActor, runtimeContext.lastCheckpointId); + RemoteCallWorker.notifyCheckpointTimeoutParallel(allActor, runtimeContext.lastCheckpointId); } if (!pendingCheckpointActors.isEmpty()) { @@ -198,15 +206,14 @@ public class CheckpointCoordinator extends BaseCoordinator { if (runtimeContext.checkpointIds.size() > 1) { Long stateExpiredCpId = runtimeContext.checkpointIds.remove(0); Long msgExpiredCheckpointId = runtimeContext.checkpointIds.get(0); - RemoteCallWorker - .clearExpiredCheckpointParallel(allActor, stateExpiredCpId, msgExpiredCheckpointId); + RemoteCallWorker.clearExpiredCheckpointParallel( + allActor, stateExpiredCpId, msgExpiredCheckpointId); } return true; } private boolean readyToTrigger() { - return (System.currentTimeMillis() - runtimeContext.lastCpTimestamp) >= - cpIntervalSecs * 1000; + return (System.currentTimeMillis() - runtimeContext.lastCpTimestamp) >= cpIntervalSecs * 1000; } private boolean timeoutOnWaitCheckpoint() { diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/FailoverCoordinator.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/FailoverCoordinator.java index c58c84d6a..5a3f235d4 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/FailoverCoordinator.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/FailoverCoordinator.java @@ -39,8 +39,7 @@ public class FailoverCoordinator extends BaseCoordinator { } public FailoverCoordinator( - JobMaster jobMaster, AsyncRemoteCaller asyncRemoteCaller, - boolean isRecover) { + JobMaster jobMaster, AsyncRemoteCaller asyncRemoteCaller, boolean isRecover) { super(jobMaster); this.asyncRemoteCaller = asyncRemoteCaller; @@ -111,8 +110,8 @@ public class FailoverCoordinator extends BaseCoordinator { ExecutionVertex exeVertex = getExeVertexFromRequest(rollbackRequest); // Reset pid for new-rollback actor. - if (null != rollbackRequest.getPid() && - !rollbackRequest.getPid().equals(WorkerRollbackRequest.DEFAULT_PID)) { + if (null != rollbackRequest.getPid() + && !rollbackRequest.getPid().equals(WorkerRollbackRequest.DEFAULT_PID)) { exeVertex.setPid(rollbackRequest.getPid()); } @@ -122,10 +121,9 @@ public class FailoverCoordinator extends BaseCoordinator { } String hostname = ""; - Optional container = ResourceUtil.getContainerById( - jobMaster.getResourceManager().getRegisteredContainers(), - exeVertex.getContainerId() - ); + Optional container = + ResourceUtil.getContainerById( + jobMaster.getResourceManager().getRegisteredContainers(), exeVertex.getContainerId()); if (container.isPresent()) { hostname = container.get().getHostname(); } @@ -133,16 +131,22 @@ public class FailoverCoordinator extends BaseCoordinator { if (rollbackRequest.isForcedRollback) { interruptCheckpointAndRollback(rollbackRequest); } else { - asyncRemoteCaller.checkIfNeedRollbackAsync(exeVertex.getWorkerActor(), res -> { - if (!res) { - LOG.info("Vertex {} doesn't need to rollback, skip it.", exeVertex); - return; - } - interruptCheckpointAndRollback(rollbackRequest); - }, throwable -> { - LOG.error("Exception when calling checkIfNeedRollbackAsync, maybe vertex is dead" + - ", ignore this request, vertex={}.", exeVertex, throwable); - }); + asyncRemoteCaller.checkIfNeedRollbackAsync( + exeVertex.getWorkerActor(), + res -> { + if (!res) { + LOG.info("Vertex {} doesn't need to rollback, skip it.", exeVertex); + return; + } + interruptCheckpointAndRollback(rollbackRequest); + }, + throwable -> { + LOG.error( + "Exception when calling checkIfNeedRollbackAsync, maybe vertex is dead" + + ", ignore this request, vertex={}.", + exeVertex, + throwable); + }); } LOG.info("Deal with rollback request {} success.", rollbackRequest); @@ -154,7 +158,9 @@ public class FailoverCoordinator extends BaseCoordinator { rollbackRequest.cascadingGroupId = currentCascadingGroupId++; } // get last valid checkpoint id then call worker rollback - rollback(jobMaster.getRuntimeContext().getLastValidCheckpointId(), rollbackRequest, + rollback( + jobMaster.getRuntimeContext().getLastValidCheckpointId(), + rollbackRequest, currentCascadingGroupId); // we interrupt current checkpoint for 2 considerations: // 1. current checkpoint might be timeout, because barrier might be lost after failover. so we @@ -165,66 +171,83 @@ public class FailoverCoordinator extends BaseCoordinator { } /** - * call worker rollback, and deal with it's reports. callback won't be finished until - * the entire DAG back to normal. + * call worker rollback, and deal with it's reports. callback won't be finished until the entire + * DAG back to normal. * * @param checkpointId checkpointId to be rollback * @param rollbackRequest worker rollback request * @param cascadingGroupId all rollback of a cascading group should have same ID */ private void rollback( - long checkpointId, WorkerRollbackRequest rollbackRequest, - long cascadingGroupId) { + long checkpointId, WorkerRollbackRequest rollbackRequest, long cascadingGroupId) { ExecutionVertex exeVertex = getExeVertexFromRequest(rollbackRequest); - LOG.info("Call vertex {} to rollback, checkpoint id is {}, cascadingGroupId={}.", - exeVertex, checkpointId, cascadingGroupId); + LOG.info( + "Call vertex {} to rollback, checkpoint id is {}, cascadingGroupId={}.", + exeVertex, + checkpointId, + cascadingGroupId); isRollbacking.put(exeVertex, true); - asyncRemoteCaller.rollback(exeVertex.getWorkerActor(), checkpointId, result -> { - List newRollbackRequests = new ArrayList<>(); - switch (result.getResultEnum()) { - case SUCCESS: - ChannelRecoverInfo recoverInfo = result.getResultObj(); - LOG.info("Vertex {} rollback done, dataLostQueues={}, msg={}, cascadingGroupId={}.", - exeVertex, recoverInfo.getDataLostQueues(), result.getResultMsg(), cascadingGroupId); - // rollback upstream if vertex reports abnormal input queues - newRollbackRequests = - cascadeUpstreamActors(recoverInfo.getDataLostQueues(), exeVertex, cascadingGroupId); - break; - case SKIPPED: - LOG.info("Vertex skip rollback, result = {}, cascadingGroupId={}.", result, - cascadingGroupId); - break; - default: - LOG.error( - "Rollback vertex {} failed, result={}, cascadingGroupId={}," + - " rollback this worker again after {} ms.", - exeVertex, result, cascadingGroupId, ROLLBACK_RETRY_TIME_MS); - Thread.sleep(ROLLBACK_RETRY_TIME_MS); - LOG.info("Add rollback request for {} again, cascadingGroupId={}.", exeVertex, - cascadingGroupId); - newRollbackRequests.add( - new WorkerRollbackRequest(exeVertex, "", "Rollback failed, try again.", false) - ); - break; - } + asyncRemoteCaller.rollback( + exeVertex.getWorkerActor(), + checkpointId, + result -> { + List newRollbackRequests = new ArrayList<>(); + switch (result.getResultEnum()) { + case SUCCESS: + ChannelRecoverInfo recoverInfo = result.getResultObj(); + LOG.info( + "Vertex {} rollback done, dataLostQueues={}, msg={}, cascadingGroupId={}.", + exeVertex, + recoverInfo.getDataLostQueues(), + result.getResultMsg(), + cascadingGroupId); + // rollback upstream if vertex reports abnormal input queues + newRollbackRequests = + cascadeUpstreamActors( + recoverInfo.getDataLostQueues(), exeVertex, cascadingGroupId); + break; + case SKIPPED: + LOG.info( + "Vertex skip rollback, result = {}, cascadingGroupId={}.", + result, + cascadingGroupId); + break; + default: + LOG.error( + "Rollback vertex {} failed, result={}, cascadingGroupId={}," + + " rollback this worker again after {} ms.", + exeVertex, + result, + cascadingGroupId, + ROLLBACK_RETRY_TIME_MS); + Thread.sleep(ROLLBACK_RETRY_TIME_MS); + LOG.info( + "Add rollback request for {} again, cascadingGroupId={}.", + exeVertex, + cascadingGroupId); + newRollbackRequests.add( + new WorkerRollbackRequest(exeVertex, "", "Rollback failed, try again.", false)); + break; + } - // lock to avoid executing new rollback requests added. - // consider such a case: A->B->C, C cascade B, and B cascade A - // if B is rollback before B's rollback request is saved, and then JobMaster crashed, - // then A will never be rollback. - synchronized (cmdLock) { - jobMaster.getRuntimeContext().foCmds.addAll(newRollbackRequests); - // this rollback request is finished, remove it. - jobMaster.getRuntimeContext().unfinishedFoCmds.remove(rollbackRequest); - jobMaster.saveContext(); - } - isRollbacking.put(exeVertex, false); - }, throwable -> { - LOG.error("Exception when calling vertex to rollback, vertex={}.", exeVertex, throwable); - isRollbacking.put(exeVertex, false); - }); + // lock to avoid executing new rollback requests added. + // consider such a case: A->B->C, C cascade B, and B cascade A + // if B is rollback before B's rollback request is saved, and then JobMaster crashed, + // then A will never be rollback. + synchronized (cmdLock) { + jobMaster.getRuntimeContext().foCmds.addAll(newRollbackRequests); + // this rollback request is finished, remove it. + jobMaster.getRuntimeContext().unfinishedFoCmds.remove(rollbackRequest); + jobMaster.saveContext(); + } + isRollbacking.put(exeVertex, false); + }, + throwable -> { + LOG.error("Exception when calling vertex to rollback, vertex={}.", exeVertex, throwable); + isRollbacking.put(exeVertex, false); + }); LOG.info("Finish rollback vertex {}, checkpoint id is {}.", exeVertex, checkpointId); } @@ -233,32 +256,39 @@ public class FailoverCoordinator extends BaseCoordinator { Set dataLostQueues, ExecutionVertex fromVertex, long cascadingGroupId) { List cascadedRollbackRequest = new ArrayList<>(); // rollback upstream if vertex reports abnormal input queues - dataLostQueues.forEach(q -> { - BaseActorHandle upstreamActor = - graphManager.getExecutionGraph().getPeerActor(fromVertex.getWorkerActor(), q); - ExecutionVertex upstreamExeVertex = getExecutionVertex(upstreamActor); - // vertexes that has already cascaded by other vertex in the same level - // of graph should be ignored. - if (isRollbacking.get(upstreamExeVertex)) { - return; - } - LOG.info("Call upstream vertex {} of vertex {} to rollback, cascadingGroupId={}.", - upstreamExeVertex, fromVertex, cascadingGroupId); - String hostname = ""; - Optional container = ResourceUtil.getContainerById( - jobMaster.getResourceManager().getRegisteredContainers(), - upstreamExeVertex.getContainerId() - ); - if (container.isPresent()) { - hostname = container.get().getHostname(); - } - // force upstream vertexes to rollback - WorkerRollbackRequest upstreamRequest = new WorkerRollbackRequest( - upstreamExeVertex, hostname, String.format("Cascading rollback from %s", fromVertex), true - ); - upstreamRequest.cascadingGroupId = cascadingGroupId; - cascadedRollbackRequest.add(upstreamRequest); - }); + dataLostQueues.forEach( + q -> { + BaseActorHandle upstreamActor = + graphManager.getExecutionGraph().getPeerActor(fromVertex.getWorkerActor(), q); + ExecutionVertex upstreamExeVertex = getExecutionVertex(upstreamActor); + // vertexes that has already cascaded by other vertex in the same level + // of graph should be ignored. + if (isRollbacking.get(upstreamExeVertex)) { + return; + } + LOG.info( + "Call upstream vertex {} of vertex {} to rollback, cascadingGroupId={}.", + upstreamExeVertex, + fromVertex, + cascadingGroupId); + String hostname = ""; + Optional container = + ResourceUtil.getContainerById( + jobMaster.getResourceManager().getRegisteredContainers(), + upstreamExeVertex.getContainerId()); + if (container.isPresent()) { + hostname = container.get().getHostname(); + } + // force upstream vertexes to rollback + WorkerRollbackRequest upstreamRequest = + new WorkerRollbackRequest( + upstreamExeVertex, + hostname, + String.format("Cascading rollback from %s", fromVertex), + true); + upstreamRequest.cascadingGroupId = cascadingGroupId; + cascadedRollbackRequest.add(upstreamRequest); + }); return cascadedRollbackRequest; } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/command/BaseWorkerCmd.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/command/BaseWorkerCmd.java index 2c6a9322d..b222b3b5f 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/command/BaseWorkerCmd.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/command/BaseWorkerCmd.java @@ -7,11 +7,9 @@ public abstract class BaseWorkerCmd implements Serializable { public ActorId fromActorId; - public BaseWorkerCmd() { - } + public BaseWorkerCmd() {} protected BaseWorkerCmd(ActorId actorId) { this.fromActorId = actorId; } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/command/InterruptCheckpointRequest.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/command/InterruptCheckpointRequest.java index 29a46ab10..28843296d 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/command/InterruptCheckpointRequest.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/command/InterruptCheckpointRequest.java @@ -1,5 +1,3 @@ package io.ray.streaming.runtime.master.coordinator.command; -public final class InterruptCheckpointRequest extends BaseWorkerCmd { - -} +public final class InterruptCheckpointRequest extends BaseWorkerCmd {} diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/command/WorkerRollbackRequest.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/command/WorkerRollbackRequest.java index e56518382..df7596304 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/command/WorkerRollbackRequest.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/coordinator/command/WorkerRollbackRequest.java @@ -23,10 +23,7 @@ public final class WorkerRollbackRequest extends BaseWorkerCmd { } public WorkerRollbackRequest( - ExecutionVertex executionVertex, - String hostname, - String msg, - boolean isForcedRollback) { + ExecutionVertex executionVertex, String hostname, String msg, boolean isForcedRollback) { super(executionVertex.getWorkerActorId()); @@ -56,8 +53,6 @@ public final class WorkerRollbackRequest extends BaseWorkerCmd { @Override public String toString() { - return MoreObjects.toStringHelper(this) - .add("fromActorId", fromActorId) - .toString(); + return MoreObjects.toStringHelper(this).add("fromActorId", fromActorId).toString(); } } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/graphmanager/GraphManager.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/graphmanager/GraphManager.java index 14598ff45..ce8dd4741 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/graphmanager/GraphManager.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/graphmanager/GraphManager.java @@ -5,37 +5,35 @@ import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph; /** * Graph manager is one of the important roles of JobMaster. It mainly focuses on graph management. - *

- * Such as: + * + *

Such as: + * *

    - *
  1. Build execution graph from job graph.
  2. - *
  3. Do modifications or operations on graph.
  4. - *
  5. Query vertex info from graph.
  6. + *
  7. Build execution graph from job graph. + *
  8. Do modifications or operations on graph. + *
  9. Query vertex info from graph. *
- *

*/ public interface GraphManager { /** * Build execution graph from job graph. * - * @param jobGraph logical plan of streaming job. - * @return physical plan of streaming job. + * @param jobGraph logical plan of streaming job. Returns physical plan of streaming job. */ ExecutionGraph buildExecutionGraph(JobGraph jobGraph); /** * Get job graph. * - * @return the job graph. + *

Returns the job graph. */ JobGraph getJobGraph(); /** * Get execution graph. * - * @return the execution graph. + *

Returns the execution graph. */ ExecutionGraph getExecutionGraph(); - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/graphmanager/GraphManagerImpl.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/graphmanager/GraphManagerImpl.java index a977967ff..b33b95a3e 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/graphmanager/GraphManagerImpl.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/graphmanager/GraphManagerImpl.java @@ -35,9 +35,11 @@ public class GraphManagerImpl implements GraphManager { ExecutionGraph executionGraph = setupStructure(jobGraph); // set max parallelism - int maxParallelism = jobGraph.getJobVertices().stream() - .map(JobVertex::getParallelism) - .max(Integer::compareTo).get(); + int maxParallelism = + jobGraph.getJobVertices().stream() + .map(JobVertex::getParallelism) + .max(Integer::compareTo) + .get(); executionGraph.setMaxParallelism(maxParallelism); // set job config @@ -57,37 +59,47 @@ public class GraphManagerImpl implements GraphManager { long buildTime = executionGraph.getBuildTime(); for (JobVertex jobVertex : jobGraph.getJobVertices()) { int jobVertexId = jobVertex.getVertexId(); - exeJobVertexMap.put(jobVertexId, + exeJobVertexMap.put( + jobVertexId, new ExecutionJobVertex( - jobVertex, - jobConfig, - executionGraph.getExecutionVertexIdGenerator(), - buildTime)); + jobVertex, jobConfig, executionGraph.getExecutionVertexIdGenerator(), buildTime)); } // for each job edge, connect all source exeVertices and target exeVertices - jobGraph.getJobEdges().forEach(jobEdge -> { - ExecutionJobVertex source = exeJobVertexMap.get(jobEdge.getSrcVertexId()); - ExecutionJobVertex target = exeJobVertexMap.get(jobEdge.getTargetVertexId()); + jobGraph + .getJobEdges() + .forEach( + jobEdge -> { + ExecutionJobVertex source = exeJobVertexMap.get(jobEdge.getSrcVertexId()); + ExecutionJobVertex target = exeJobVertexMap.get(jobEdge.getTargetVertexId()); - ExecutionJobEdge executionJobEdge = new ExecutionJobEdge(source, target, jobEdge); + ExecutionJobEdge executionJobEdge = new ExecutionJobEdge(source, target, jobEdge); - source.getOutputEdges().add(executionJobEdge); - target.getInputEdges().add(executionJobEdge); + source.getOutputEdges().add(executionJobEdge); + target.getInputEdges().add(executionJobEdge); - source.getExecutionVertices().forEach(sourceExeVertex -> { - target.getExecutionVertices().forEach(targetExeVertex -> { - // pre-process some mappings - executionVertexMap.put(targetExeVertex.getExecutionVertexId(), targetExeVertex); - executionVertexMap.put(sourceExeVertex.getExecutionVertexId(), sourceExeVertex); - // build execution edge - ExecutionEdge executionEdge = - new ExecutionEdge(sourceExeVertex, targetExeVertex, executionJobEdge); - sourceExeVertex.getOutputEdges().add(executionEdge); - targetExeVertex.getInputEdges().add(executionEdge); - }); - }); - }); + source + .getExecutionVertices() + .forEach( + sourceExeVertex -> { + target + .getExecutionVertices() + .forEach( + targetExeVertex -> { + // pre-process some mappings + executionVertexMap.put( + targetExeVertex.getExecutionVertexId(), targetExeVertex); + executionVertexMap.put( + sourceExeVertex.getExecutionVertexId(), sourceExeVertex); + // build execution edge + ExecutionEdge executionEdge = + new ExecutionEdge( + sourceExeVertex, targetExeVertex, executionJobEdge); + sourceExeVertex.getOutputEdges().add(executionEdge); + targetExeVertex.getInputEdges().add(executionEdge); + }); + }); + }); // set execution job vertex into execution graph executionGraph.setExecutionJobVertexMap(exeJobVertexMap); @@ -115,5 +127,4 @@ public class GraphManagerImpl implements GraphManager { public ExecutionGraph getExecutionGraph() { return runtimeContext.getExecutionGraph(); } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/ResourceAssignmentView.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/ResourceAssignmentView.java index 9ee228eaf..ea6fb0986 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/ResourceAssignmentView.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/ResourceAssignmentView.java @@ -5,9 +5,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -/** - * Cluster resource allocation view, used to statically view cluster resource information. - */ +/** Cluster resource allocation view, used to statically view cluster resource information. */ public class ResourceAssignmentView extends HashMap> { public static ResourceAssignmentView of(Map> assignmentView) { diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/ResourceManager.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/ResourceManager.java index c885ffcea..43671eea1 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/ResourceManager.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/ResourceManager.java @@ -4,16 +4,13 @@ import com.google.common.collect.ImmutableList; import io.ray.streaming.runtime.core.resource.Container; import io.ray.streaming.runtime.master.resourcemanager.strategy.ResourceAssignStrategy; -/** - * ResourceManager(RM) is responsible for resource de-/allocation and monitoring ray cluster. - */ +/** ResourceManager(RM) is responsible for resource de-/allocation and monitoring ray cluster. */ public interface ResourceManager extends ResourceAssignStrategy { /** * Get registered containers, the container list is read-only. * - * @return the registered container list + *

Returns the registered container list */ ImmutableList getRegisteredContainers(); - -} \ No newline at end of file +} diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/ResourceManagerImpl.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/ResourceManagerImpl.java index 2e59fed09..1a20c4ae3 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/ResourceManagerImpl.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/ResourceManagerImpl.java @@ -28,32 +28,21 @@ public class ResourceManagerImpl implements ResourceManager { private static final Logger LOG = LoggerFactory.getLogger(ResourceManagerImpl.class); - //Container used tag + // Container used tag private static final String CONTAINER_ENGAGED_KEY = "CONTAINER_ENGAGED_KEY"; - /** - * Resource description information. - */ + /** Resource description information. */ private final Resources resources; - /** - * Timing resource updating thread - */ - private final ScheduledExecutorService resourceUpdater = new ScheduledThreadPoolExecutor(1, - new ThreadFactoryBuilder().setNameFormat("resource-update-thread").build()); - /** - * Job runtime context. - */ + /** Timing resource updating thread */ + private final ScheduledExecutorService resourceUpdater = + new ScheduledThreadPoolExecutor( + 1, new ThreadFactoryBuilder().setNameFormat("resource-update-thread").build()); + /** Job runtime context. */ private JobMasterRuntimeContext runtimeContext; - /** - * Resource related configuration. - */ + /** Resource related configuration. */ private ResourceConfig resourceConfig; - /** - * Slot assign strategy. - */ + /** Slot assign strategy. */ private ResourceAssignStrategy resourceAssignStrategy; - /** - * Customized actor number for each container - */ + /** Customized actor number for each container */ private int actorNumPerContainer; public ResourceManagerImpl(JobMasterRuntimeContext runtimeContext) { @@ -62,19 +51,19 @@ public class ResourceManagerImpl implements ResourceManager { this.resourceConfig = masterConfig.resourceConfig; this.resources = new Resources(); - LOG.info("ResourceManagerImpl begin init, conf is {}, resources are {}.", - resourceConfig, resources); + LOG.info( + "ResourceManagerImpl begin init, conf is {}, resources are {}.", resourceConfig, resources); // Init custom resource configurations this.actorNumPerContainer = resourceConfig.actorNumPerContainer(); ResourceAssignStrategyType resourceAssignStrategyType = ResourceAssignStrategyType.PIPELINE_FIRST_STRATEGY; - this.resourceAssignStrategy = ResourceAssignStrategyFactory.getStrategy( - resourceAssignStrategyType); + this.resourceAssignStrategy = + ResourceAssignStrategyFactory.getStrategy(resourceAssignStrategyType); LOG.info("Slot assign strategy: {}.", resourceAssignStrategy.getName()); - //Init resource + // Init resource initResource(); checkAndUpdateResourcePeriodically(); @@ -84,8 +73,7 @@ public class ResourceManagerImpl implements ResourceManager { @Override public ResourceAssignmentView assignResource( - List containers, - ExecutionGraph executionGraph) { + List containers, ExecutionGraph executionGraph) { return resourceAssignStrategy.assignResource(containers, executionGraph); } @@ -105,17 +93,22 @@ public class ResourceManagerImpl implements ResourceManager { * system. */ private void checkAndUpdateResource() { - //Get add&del nodes(node -> container) + // Get add&del nodes(node -> container) Map latestNodeInfos = RayUtils.getAliveNodeInfoMap(); - List addNodes = latestNodeInfos.keySet().stream() - .filter(this::isAddedNode).collect(Collectors.toList()); + List addNodes = + latestNodeInfos.keySet().stream().filter(this::isAddedNode).collect(Collectors.toList()); - List deleteNodes = resources.getRegisteredContainerMap().keySet().stream() - .filter(nodeId -> !latestNodeInfos.containsKey(nodeId)) - .collect(Collectors.toList()); - LOG.info("Latest node infos: {}, current containers: {}, add nodes: {}, delete nodes: {}.", - latestNodeInfos, resources.getRegisteredContainers(), addNodes, deleteNodes); + List deleteNodes = + resources.getRegisteredContainerMap().keySet().stream() + .filter(nodeId -> !latestNodeInfos.containsKey(nodeId)) + .collect(Collectors.toList()); + LOG.info( + "Latest node infos: {}, current containers: {}, add nodes: {}, delete nodes: {}.", + latestNodeInfos, + resources.getRegisteredContainers(), + addNodes, + deleteNodes); if (!addNodes.isEmpty() || !deleteNodes.isEmpty()) { LOG.info("Latest node infos from GCS: {}", latestNodeInfos); @@ -126,8 +119,8 @@ public class ResourceManagerImpl implements ResourceManager { unregisterDeletedContainer(deleteNodes); // register containers - registerNewContainers(addNodes.stream().map(latestNodeInfos::get) - .collect(Collectors.toList())); + registerNewContainers( + addNodes.stream().map(latestNodeInfos::get).collect(Collectors.toList())); } } @@ -152,14 +145,13 @@ public class ResourceManagerImpl implements ResourceManager { // failover case: container has already allocated actors double availableCapacity = actorNumPerContainer - container.getAllocatedActorNum(); - //Create ray resource. + // Create ray resource. Ray.setResource(container.getNodeId(), container.getName(), availableCapacity); - //Mark container is already registered. + // Mark container is already registered. Ray.setResource(container.getNodeId(), CONTAINER_ENGAGED_KEY, 1); // update container's available dynamic resources - container.getAvailableResources() - .put(container.getName(), availableCapacity); + container.getAvailableResources().put(container.getName(), availableCapacity); // update register container list resources.registerContainer(container); @@ -187,5 +179,4 @@ public class ResourceManagerImpl implements ResourceManager { private boolean isAddedNode(UniqueId uniqueId) { return !resources.getRegisteredContainerMap().containsKey(uniqueId); } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/ViewBuilder.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/ViewBuilder.java index 8f005fa47..76cf221fd 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/ViewBuilder.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/ViewBuilder.java @@ -5,19 +5,18 @@ import io.ray.streaming.runtime.core.resource.ContainerId; import java.util.List; import java.util.Map; -/** - * ViewBuilder describes current cluster's resource allocation detail information - */ +/** ViewBuilder describes current cluster's resource allocation detail information */ public class ViewBuilder { // Default constructor for serialization. - public ViewBuilder() { - } + public ViewBuilder() {} public static ResourceAssignmentView buildResourceAssignmentView(List containers) { Map> assignmentView = - containers.stream().collect(java.util.stream.Collectors.toMap(Container::getId, - Container::getExecutionVertexIds)); + containers.stream() + .collect( + java.util.stream.Collectors.toMap( + Container::getId, Container::getExecutionVertexIds)); return ResourceAssignmentView.of(assignmentView); } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/strategy/ResourceAssignStrategy.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/strategy/ResourceAssignStrategy.java index c4ec37b4a..8df20790c 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/strategy/ResourceAssignStrategy.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/strategy/ResourceAssignStrategy.java @@ -6,23 +6,17 @@ import io.ray.streaming.runtime.core.resource.Container; import io.ray.streaming.runtime.master.resourcemanager.ResourceAssignmentView; import java.util.List; -/** - * The ResourceAssignStrategy responsible assign {@link Container} to {@link ExecutionVertex}. - */ +/** The ResourceAssignStrategy responsible assign {@link Container} to {@link ExecutionVertex}. */ public interface ResourceAssignStrategy { /** * Assign {@link Container} for {@link ExecutionVertex} * * @param containers registered container - * @param executionGraph execution graph - * @return allocating view + * @param executionGraph execution graph Returns allocating view */ ResourceAssignmentView assignResource(List containers, ExecutionGraph executionGraph); - /** - * Get container assign strategy name - */ + /** Get container assign strategy name */ String getName(); - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/strategy/impl/PipelineFirstStrategy.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/strategy/impl/PipelineFirstStrategy.java index bc65f8d35..74b646c67 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/strategy/impl/PipelineFirstStrategy.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/resourcemanager/strategy/impl/PipelineFirstStrategy.java @@ -22,6 +22,7 @@ import org.slf4j.LoggerFactory; * balanced and controllable scheduling. Assume that we have 2 containers and have a DAG graph * composed of a source node with parallelism of 2 and a sink node with parallelism of 2, the * structure will be like: + * *

  *   container_0
  *             |- source_1
@@ -41,24 +42,23 @@ public class PipelineFirstStrategy implements ResourceAssignStrategy {
    * Assign resource to each execution vertex in the given execution graph.
    *
    * @param containers registered containers
-   * @param executionGraph execution graph
-   * @return allocating map, key is container ID, value is list of vertextId, and contains vertices
+   * @param executionGraph execution graph Returns allocating map, key is container ID, value is
+   *     list of vertextId, and contains vertices
    */
   @Override
   public ResourceAssignmentView assignResource(
-      List containers,
-      ExecutionGraph executionGraph) {
+      List containers, ExecutionGraph executionGraph) {
 
     Map vertices = executionGraph.getExecutionJobVertexMap();
     Map vertexRemainingNum = new HashMap<>();
 
-    vertices.forEach((k, v) -> {
-      int size = v.getExecutionVertices().size();
-      vertexRemainingNum.put(k, size);
-    });
-    int totalExecutionVerticesNum = vertexRemainingNum.values().stream()
-        .mapToInt(Integer::intValue)
-        .sum();
+    vertices.forEach(
+        (k, v) -> {
+          int size = v.getExecutionVertices().size();
+          vertexRemainingNum.put(k, size);
+        });
+    int totalExecutionVerticesNum =
+        vertexRemainingNum.values().stream().mapToInt(Integer::intValue).sum();
     int containerNum = containers.size();
     int capacityPerContainer = Math.max(totalExecutionVerticesNum / containerNum, 1);
 
@@ -70,8 +70,11 @@ public class PipelineFirstStrategy implements ResourceAssignStrategy {
       enlargeCapacityThreshold = capacityPerContainer * containerNum;
       LOG.info("Need to enlarge capacity per container, threshold: {}.", enlargeCapacityThreshold);
     }
-    LOG.info("Total execution vertices num: {}, container num: {}, capacity per container: {}.",
-        totalExecutionVerticesNum, containerNum, capacityPerContainer);
+    LOG.info(
+        "Total execution vertices num: {}, container num: {}, capacity per container: {}.",
+        totalExecutionVerticesNum,
+        containerNum,
+        capacityPerContainer);
 
     int maxParallelism = executionGraph.getMaxParallelism();
 
@@ -86,8 +89,10 @@ public class PipelineFirstStrategy implements ResourceAssignStrategy {
         ExecutionVertex executionVertex = exeVertices.get(i);
         Map requiredResource = executionVertex.getResource();
         if (requiredResource.containsKey(ResourceType.CPU.getValue())) {
-          LOG.info("Required resource contain {} value : {}, no limitation by default.",
-              ResourceType.CPU, requiredResource.get(ResourceType.CPU.getValue()));
+          LOG.info(
+              "Required resource contain {} value : {}, no limitation by default.",
+              ResourceType.CPU,
+              requiredResource.get(ResourceType.CPU.getValue()));
           requiredResource.remove(ResourceType.CPU.getValue());
         }
 
@@ -96,7 +101,8 @@ public class PipelineFirstStrategy implements ResourceAssignStrategy {
         targetContainer.allocateActor(executionVertex);
         allocatedVertexCount++;
         // Once allocatedVertexCount reaches threshold, we should enlarge capacity
-        if (!enlarged && enlargeCapacityThreshold > 0
+        if (!enlarged
+            && enlargeCapacityThreshold > 0
             && allocatedVertexCount >= enlargeCapacityThreshold) {
           updateContainerCapacity(containers, capacityPerContainer + 1);
           enlarged = true;
@@ -127,12 +133,10 @@ public class PipelineFirstStrategy implements ResourceAssignStrategy {
    * Find a container which matches required resource
    *
    * @param requiredResource required resource
-   * @param containers registered containers
-   * @return container that matches the required resource
+   * @param containers registered containers Returns container that matches the required resource
    */
   private Container findMatchedContainer(
-      Map requiredResource,
-      List containers) {
+      Map requiredResource, List containers) {
 
     LOG.info("Check resource, required: {}.", requiredResource);
 
@@ -143,7 +147,8 @@ public class PipelineFirstStrategy implements ResourceAssignStrategy {
       forwardToNextContainer(containers);
       if (checkedNum >= containers.size()) {
         throw new ScheduleException(
-            String.format("No enough resource left, required resource: %s, available resource: %s.",
+            String.format(
+                "No enough resource left, required resource: %s, available resource: %s.",
                 requiredResource, containers));
       }
     }
@@ -154,8 +159,7 @@ public class PipelineFirstStrategy implements ResourceAssignStrategy {
    * Check if current container has enough resource
    *
    * @param requiredResource required resource
-   * @param container container
-   * @return true if matches, false else
+   * @param container container Returns true if matches, false else
    */
   private boolean hasEnoughResource(Map requiredResource, Container container) {
     LOG.info("Check resource for index: {}, container: {}", currentContainerIndex, container);
@@ -173,13 +177,19 @@ public class PipelineFirstStrategy implements ResourceAssignStrategy {
     for (Map.Entry entry : requiredResource.entrySet()) {
       if (availableResource.containsKey(entry.getKey())) {
         if (availableResource.get(entry.getKey()) < entry.getValue()) {
-          LOG.warn("No enough resource for container {}. required: {}, available: {}.",
-              container.getAddress(), requiredResource, availableResource);
+          LOG.warn(
+              "No enough resource for container {}. required: {}, available: {}.",
+              container.getAddress(),
+              requiredResource,
+              availableResource);
           return false;
         }
       } else {
-        LOG.warn("No enough resource for container {}. required: {}, available: {}.",
-            container.getAddress(), requiredResource, availableResource);
+        LOG.warn(
+            "No enough resource for container {}. required: {}, available: {}.",
+            container.getAddress(),
+            requiredResource,
+            availableResource);
         return false;
       }
     }
@@ -190,8 +200,7 @@ public class PipelineFirstStrategy implements ResourceAssignStrategy {
   /**
    * Forward to next container
    *
-   * @param containers registered container list
-   * @return next container in the list
+   * @param containers registered container list Returns next container in the list
    */
   private Container forwardToNextContainer(List containers) {
     this.currentContainerIndex = (this.currentContainerIndex + 1) % containers.size();
@@ -201,8 +210,7 @@ public class PipelineFirstStrategy implements ResourceAssignStrategy {
   /**
    * Get current container
    *
-   * @param containers registered container
-   * @return current container to allocate actor
+   * @param containers registered container Returns current container to allocate actor
    */
   private Container getCurrentContainer(List containers) {
     return containers.get(currentContainerIndex);
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/scheduler/JobScheduler.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/scheduler/JobScheduler.java
index 3863a1709..962c0bdfa 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/scheduler/JobScheduler.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/scheduler/JobScheduler.java
@@ -2,16 +2,13 @@ package io.ray.streaming.runtime.master.scheduler;
 
 import io.ray.streaming.runtime.core.graph.executiongraph.ExecutionGraph;
 
-/**
- * Job scheduler is used to do the scheduling in JobMaster.
- */
+/** Job scheduler is used to do the scheduling in JobMaster. */
 public interface JobScheduler {
 
   /**
    * Schedule streaming job using the physical plan.
    *
-   * @param executionGraph physical plan
-   * @return scheduling result
+   * @param executionGraph physical plan Returns scheduling result
    */
   boolean scheduleJob(ExecutionGraph executionGraph);
 }
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/scheduler/JobSchedulerImpl.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/scheduler/JobSchedulerImpl.java
index 238fdf6f7..05fdb2d2c 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/scheduler/JobSchedulerImpl.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/scheduler/JobSchedulerImpl.java
@@ -17,9 +17,7 @@ import java.util.Map;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-/**
- * Job scheduler implementation.
- */
+/** Job scheduler implementation. */
 public class JobSchedulerImpl implements JobScheduler {
 
   private static final Logger LOG = LoggerFactory.getLogger(JobSchedulerImpl.class);
@@ -96,16 +94,15 @@ public class JobSchedulerImpl implements JobScheduler {
   /**
    * Create JobWorker actors according to the physical plan.
    *
-   * @param executionGraph physical plan
-   * @return actor creation result
+   * @param executionGraph physical plan Returns actor creation result
    */
   public boolean createWorkers(ExecutionGraph executionGraph) {
     LOG.info("Begin creating workers.");
     long startTs = System.currentTimeMillis();
 
     // create JobWorker actors
-    boolean createResult = workerLifecycleController
-        .createWorkers(executionGraph.getAllAddedExecutionVertices());
+    boolean createResult =
+        workerLifecycleController.createWorkers(executionGraph.getAllAddedExecutionVertices());
 
     if (createResult) {
       LOG.info("Finished creating workers. Cost {} ms.", System.currentTimeMillis() - startTs);
@@ -124,8 +121,10 @@ public class JobSchedulerImpl implements JobScheduler {
   protected boolean initWorkers(Map vertexToContextMap) {
     boolean result;
     try {
-      result = workerLifecycleController.initWorkers(vertexToContextMap,
-          jobConfig.masterConfig.schedulerConfig.workerInitiationWaitTimeoutMs());
+      result =
+          workerLifecycleController.initWorkers(
+              vertexToContextMap,
+              jobConfig.masterConfig.schedulerConfig.workerInitiationWaitTimeoutMs());
     } catch (Exception e) {
       LOG.error("Failed to initiate workers.", e);
       return false;
@@ -133,15 +132,15 @@ public class JobSchedulerImpl implements JobScheduler {
     return result;
   }
 
-  /**
-   * Start JobWorkers according to the physical plan.
-   */
+  /** Start JobWorkers according to the physical plan. */
   public boolean startWorkers(ExecutionGraph executionGraph, long checkpointId) {
     boolean result;
     try {
-      result = workerLifecycleController.startWorkers(
-          executionGraph, checkpointId,
-          jobConfig.masterConfig.schedulerConfig.workerStartingWaitTimeoutMs());
+      result =
+          workerLifecycleController.startWorkers(
+              executionGraph,
+              checkpointId,
+              jobConfig.masterConfig.schedulerConfig.workerStartingWaitTimeoutMs());
     } catch (Exception e) {
       LOG.error("Failed to start workers.", e);
       return false;
@@ -152,8 +151,7 @@ public class JobSchedulerImpl implements JobScheduler {
   /**
    * Build workers context.
    *
-   * @param executionGraph execution graph
-   * @return vertex to worker context map
+   * @param executionGraph execution graph Returns vertex to worker context map
    */
   protected Map buildWorkersContext(
       ExecutionGraph executionGraph) {
@@ -161,22 +159,21 @@ public class JobSchedulerImpl implements JobScheduler {
 
     // build workers' context
     Map vertexToContextMap = new HashMap<>();
-    executionGraph.getAllExecutionVertices().forEach(vertex -> {
-      JobWorkerContext context = buildJobWorkerContext(vertex, masterActor);
-      vertexToContextMap.put(vertex, context);
-    });
+    executionGraph
+        .getAllExecutionVertices()
+        .forEach(
+            vertex -> {
+              JobWorkerContext context = buildJobWorkerContext(vertex, masterActor);
+              vertexToContextMap.put(vertex, context);
+            });
     return vertexToContextMap;
   }
 
   private JobWorkerContext buildJobWorkerContext(
-      ExecutionVertex executionVertex,
-      ActorHandle masterActor) {
+      ExecutionVertex executionVertex, ActorHandle masterActor) {
 
     // create java worker context
-    JobWorkerContext context = new JobWorkerContext(
-        masterActor,
-        executionVertex
-    );
+    JobWorkerContext context = new JobWorkerContext(masterActor, executionVertex);
 
     return context;
   }
@@ -200,5 +197,4 @@ public class JobSchedulerImpl implements JobScheduler {
   private void initMaster() {
     jobMaster.init(false);
   }
-
 }
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/scheduler/ScheduleException.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/scheduler/ScheduleException.java
index af6b81d4e..9841e0365 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/scheduler/ScheduleException.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/scheduler/ScheduleException.java
@@ -19,8 +19,7 @@ public class ScheduleException extends RuntimeException {
   }
 
   protected ScheduleException(
-      String message, Throwable cause, boolean enableSuppression,
-      boolean writableStackTrace) {
+      String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) {
     super(message, cause, enableSuppression, writableStackTrace);
   }
 }
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/scheduler/controller/WorkerLifecycleController.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/scheduler/controller/WorkerLifecycleController.java
index bc8b462c7..f5c4be5f7 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/scheduler/controller/WorkerLifecycleController.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/master/scheduler/controller/WorkerLifecycleController.java
@@ -24,9 +24,7 @@ import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-/**
- * Worker lifecycle controller is used to control JobWorker's creation, initiation and so on.
- */
+/** Worker lifecycle controller is used to control JobWorker's creation, initiation and so on. */
 public class WorkerLifecycleController {
 
   private static final Logger LOG = LoggerFactory.getLogger(WorkerLifecycleController.class);
@@ -38,30 +36,34 @@ public class WorkerLifecycleController {
   /**
    * Create JobWorker actor according to the execution vertex.
    *
-   * @param executionVertex target execution vertex
-   * @return creation result
+   * @param executionVertex target execution vertex Returns creation result
    */
   private boolean createWorker(ExecutionVertex executionVertex) {
-    LOG.info("Start to create worker actor for vertex: {} with resource: {}, workeConfig: {}.",
-        executionVertex.getExecutionVertexName(), executionVertex.getResource(),
+    LOG.info(
+        "Start to create worker actor for vertex: {} with resource: {}, workeConfig: {}.",
+        executionVertex.getExecutionVertexName(),
+        executionVertex.getResource(),
         executionVertex.getWorkerConfig());
 
     Language language = executionVertex.getLanguage();
 
     BaseActorHandle actor;
     if (Language.JAVA == language) {
-      actor = Ray.actor(JobWorker::new, executionVertex)
-          .setResources(executionVertex.getResource())
-          .setMaxRestarts(-1)
-          .remote();
+      actor =
+          Ray.actor(JobWorker::new, executionVertex)
+              .setResources(executionVertex.getResource())
+              .setMaxRestarts(-1)
+              .remote();
     } else {
-      RemoteCall.ExecutionVertexContext.ExecutionVertex vertexPb
-          = new GraphPbBuilder().buildVertex(executionVertex);
-      actor = Ray.actor(
-          PyActorClass.of("ray.streaming.runtime.worker", "JobWorker"), vertexPb.toByteArray())
-          .setResources(executionVertex.getResource())
-          .setMaxRestarts(-1)
-          .remote();
+      RemoteCall.ExecutionVertexContext.ExecutionVertex vertexPb =
+          new GraphPbBuilder().buildVertex(executionVertex);
+      actor =
+          Ray.actor(
+                  PyActorClass.of("ray.streaming.runtime.worker", "JobWorker"),
+                  vertexPb.toByteArray())
+              .setResources(executionVertex.getResource())
+              .setMaxRestarts(-1)
+              .remote();
     }
 
     if (null == actor) {
@@ -71,8 +73,10 @@ public class WorkerLifecycleController {
 
     executionVertex.setWorkerActor(actor);
 
-    LOG.info("Worker actor created, actor: {}, vertex: {}.",
-        executionVertex.getWorkerActorId(), executionVertex.getExecutionVertexName());
+    LOG.info(
+        "Worker actor created, actor: {}, vertex: {}.",
+        executionVertex.getWorkerActorId(),
+        executionVertex.getExecutionVertexName());
     return true;
   }
 
@@ -80,8 +84,7 @@ public class WorkerLifecycleController {
    * Using context to init JobWorker.
    *
    * @param vertexToContextMap target JobWorker actor
-   * @param timeout timeout for waiting, unit: ms
-   * @return initiation result
+   * @param timeout timeout for waiting, unit: ms Returns initiation result
    */
   public boolean initWorkers(
       Map vertexToContextMap, int timeout) {
@@ -89,11 +92,15 @@ public class WorkerLifecycleController {
     long startTime = System.currentTimeMillis();
 
     Map, ActorId> rayObjects = new HashMap<>();
-    vertexToContextMap.entrySet().forEach((entry -> {
-      ExecutionVertex vertex = entry.getKey();
-      rayObjects.put(RemoteCallWorker.initWorker(vertex.getWorkerActor(), entry.getValue()),
-          vertex.getWorkerActorId());
-    }));
+    vertexToContextMap
+        .entrySet()
+        .forEach(
+            (entry -> {
+              ExecutionVertex vertex = entry.getKey();
+              rayObjects.put(
+                  RemoteCallWorker.initWorker(vertex.getWorkerActor(), entry.getValue()),
+                  vertex.getWorkerActorId());
+            }));
 
     List> objectRefList = new ArrayList<>(rayObjects.keySet());
 
@@ -113,8 +120,7 @@ public class WorkerLifecycleController {
    * Start JobWorkers to run task.
    *
    * @param executionGraph physical plan
-   * @param timeout timeout for waiting, unit: ms
-   * @return starting result
+   * @param timeout timeout for waiting, unit: ms Returns starting result
    */
   public boolean startWorkers(ExecutionGraph executionGraph, long lastCheckpointId, int timeout) {
     LOG.info("Begin starting workers.");
@@ -122,11 +128,13 @@ public class WorkerLifecycleController {
     List> objectRefs = new ArrayList<>();
 
     // start source actors 1st
-    executionGraph.getSourceActors()
+    executionGraph
+        .getSourceActors()
         .forEach(actor -> objectRefs.add(RemoteCallWorker.rollback(actor, lastCheckpointId)));
 
     // then start non-source actors
-    executionGraph.getNonSourceActors()
+    executionGraph
+        .getNonSourceActors()
         .forEach(actor -> objectRefs.add(RemoteCallWorker.rollback(actor, lastCheckpointId)));
 
     WaitResult result = Ray.wait(objectRefs, objectRefs.size(), timeout);
@@ -142,8 +150,7 @@ public class WorkerLifecycleController {
   /**
    * Stop and destroy JobWorkers' actor.
    *
-   * @param executionVertices target vertices
-   * @return destroy result
+   * @param executionVertices target vertices Returns destroy result
    */
   public boolean destroyWorkers(List executionVertices) {
     return asyncBatchExecute(this::destroyWorker, executionVertices);
@@ -151,14 +158,18 @@ public class WorkerLifecycleController {
 
   private boolean destroyWorker(ExecutionVertex executionVertex) {
     BaseActorHandle rayActor = executionVertex.getWorkerActor();
-    LOG.info("Begin destroying worker[vertex={}, actor={}].",
-        executionVertex.getExecutionVertexName(), rayActor.getId());
+    LOG.info(
+        "Begin destroying worker[vertex={}, actor={}].",
+        executionVertex.getExecutionVertexName(),
+        rayActor.getId());
 
     boolean destroyResult = RemoteCallWorker.shutdownWithoutReconstruction(rayActor);
 
     if (!destroyResult) {
-      LOG.error("Failed to destroy JobWorker[{}]'s actor: {}.",
-          executionVertex.getExecutionVertexName(), rayActor);
+      LOG.error(
+          "Failed to destroy JobWorker[{}]'s actor: {}.",
+          executionVertex.getExecutionVertexName(),
+          rayActor);
       return false;
     }
 
@@ -172,18 +183,22 @@ public class WorkerLifecycleController {
    * @param operation the function to be executed
    */
   private boolean asyncBatchExecute(
-      Function operation,
-      List executionVertices) {
+      Function operation, List executionVertices) {
     final Object asyncContext = Ray.getAsyncContext();
 
     List> futureResults =
-        executionVertices.stream().map(vertex -> CompletableFuture.supplyAsync(() -> {
-          Ray.setAsyncContext(asyncContext);
-          return operation.apply(vertex);
-        })).collect(Collectors.toList());
+        executionVertices.stream()
+            .map(
+                vertex ->
+                    CompletableFuture.supplyAsync(
+                        () -> {
+                          Ray.setAsyncContext(asyncContext);
+                          return operation.apply(vertex);
+                        }))
+            .collect(Collectors.toList());
 
-    List succeeded = futureResults.stream().map(CompletableFuture::join)
-        .collect(Collectors.toList());
+    List succeeded =
+        futureResults.stream().map(CompletableFuture::join).collect(Collectors.toList());
 
     if (succeeded.stream().anyMatch(x -> !x)) {
       LOG.error("Not all futures return true, check ResourceManager'log the detail.");
@@ -191,5 +206,4 @@ public class WorkerLifecycleController {
     }
     return true;
   }
-
 }
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/message/CallResult.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/message/CallResult.java
index 5cdba0b0a..d1459c40d 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/message/CallResult.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/message/CallResult.java
@@ -10,8 +10,7 @@ public class CallResult implements Serializable {
   private int resultCode;
   private String resultMsg;
 
-  public CallResult() {
-  }
+  public CallResult() {}
 
   public CallResult(boolean success, int resultCode, String resultMsg, T resultObj) {
     this.success = success;
@@ -95,9 +94,7 @@ public class CallResult implements Serializable {
   }
 
   public enum CallResultEnum implements Serializable {
-    /**
-     * call result enum
-     */
+    /** call result enum */
     SUCCESS(0, "SUCCESS"),
     FAILED(1, "FAILED"),
     SKIPPED(2, "SKIPPED");
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/python/GraphPbBuilder.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/python/GraphPbBuilder.java
index f7bbc6278..2e3aa3cce 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/python/GraphPbBuilder.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/python/GraphPbBuilder.java
@@ -33,33 +33,25 @@ public class GraphPbBuilder {
     // build upstream vertices
     List upstreamVertices = executionVertex.getInputVertices();
     List upstreamVertexPbs =
-        upstreamVertices.stream()
-            .map(this::buildVertex)
-            .collect(Collectors.toList());
+        upstreamVertices.stream().map(this::buildVertex).collect(Collectors.toList());
     builder.addAllUpstreamExecutionVertices(upstreamVertexPbs);
 
     // build downstream vertices
     List downstreamVertices = executionVertex.getOutputVertices();
     List downstreamVertexPbs =
-        downstreamVertices.stream()
-            .map(this::buildVertex)
-            .collect(Collectors.toList());
+        downstreamVertices.stream().map(this::buildVertex).collect(Collectors.toList());
     builder.addAllDownstreamExecutionVertices(downstreamVertexPbs);
 
     // build input edges
     List inputEdges = executionVertex.getInputEdges();
     List inputEdgesPbs =
-        inputEdges.stream()
-            .map(this::buildEdge)
-            .collect(Collectors.toList());
+        inputEdges.stream().map(this::buildEdge).collect(Collectors.toList());
     builder.addAllInputExecutionEdges(inputEdgesPbs);
 
     // build output edges
     List outputEdges = executionVertex.getOutputEdges();
     List outputEdgesPbs =
-        outputEdges.stream()
-            .map(this::buildEdge)
-            .collect(Collectors.toList());
+        outputEdges.stream().map(this::buildEdge).collect(Collectors.toList());
     builder.addAllOutputExecutionEdges(outputEdgesPbs);
 
     return builder.build();
@@ -76,13 +68,11 @@ public class GraphPbBuilder {
     executionVertexBuilder.setExecutionVertexIndex(executionVertex.getExecutionVertexIndex());
     executionVertexBuilder.setParallelism(executionVertex.getParallelism());
     executionVertexBuilder.setOperator(
-        ByteString.copyFrom(
-            serializeOperator(executionVertex.getStreamOperator())));
+        ByteString.copyFrom(serializeOperator(executionVertex.getStreamOperator())));
     executionVertexBuilder.setChained(isPythonChainedOperator(executionVertex.getStreamOperator()));
     if (executionVertex.getWorkerActor() != null) {
       executionVertexBuilder.setWorkerActor(
-          ByteString.copyFrom(
-              ((NativeActorHandle) (executionVertex.getWorkerActor())).toBytes()));
+          ByteString.copyFrom(((NativeActorHandle) (executionVertex.getWorkerActor())).toBytes()));
     }
     executionVertexBuilder.setContainerId(executionVertex.getContainerId().toString());
     executionVertexBuilder.setBuildTime(executionVertex.getBuildTime());
@@ -112,11 +102,11 @@ public class GraphPbBuilder {
         return serializePythonChainedOperator((ChainedPythonOperator) operator);
       } else {
         PythonOperator pythonOperator = (PythonOperator) operator;
-        return serializer.serialize(Arrays.asList(
-            serializeFunction(pythonOperator.getFunction()),
-            pythonOperator.getModuleName(),
-            pythonOperator.getClassName()
-        ));
+        return serializer.serialize(
+            Arrays.asList(
+                serializeFunction(pythonOperator.getFunction()),
+                pythonOperator.getModuleName(),
+                pythonOperator.getClassName()));
       }
     } else {
       return new byte[0];
@@ -128,24 +118,19 @@ public class GraphPbBuilder {
   }
 
   private byte[] serializePythonChainedOperator(ChainedPythonOperator operator) {
-    List serializedOperators = operator.getOperators().stream()
-        .map(this::serializeOperator)
-        .collect(Collectors.toList());
-    return serializer.serialize(Arrays.asList(
-        serializedOperators,
-        operator.getConfigs()
-    ));
+    List serializedOperators =
+        operator.getOperators().stream().map(this::serializeOperator).collect(Collectors.toList());
+    return serializer.serialize(Arrays.asList(serializedOperators, operator.getConfigs()));
   }
 
-
   private byte[] serializeFunction(Function function) {
     if (function instanceof PythonFunction) {
       PythonFunction pyFunc = (PythonFunction) function;
       // function_bytes, module_name, function_name, function_interface
-      return serializer.serialize(Arrays.asList(
-          pyFunc.getFunction(), pyFunc.getModuleName(),
-          pyFunc.getFunctionName(), pyFunc.getFunctionInterface()
-      ));
+      return serializer.serialize(
+          Arrays.asList(
+              pyFunc.getFunction(), pyFunc.getModuleName(),
+              pyFunc.getFunctionName(), pyFunc.getFunctionInterface()));
     } else {
       return new byte[0];
     }
@@ -155,13 +140,13 @@ public class GraphPbBuilder {
     if (partition instanceof PythonPartition) {
       PythonPartition pythonPartition = (PythonPartition) partition;
       // partition_bytes, module_name, function_name
-      return serializer.serialize(Arrays.asList(
-          pythonPartition.getPartition(), pythonPartition.getModuleName(),
-          pythonPartition.getFunctionName()
-      ));
+      return serializer.serialize(
+          Arrays.asList(
+              pythonPartition.getPartition(),
+              pythonPartition.getModuleName(),
+              pythonPartition.getFunctionName()));
     } else {
       return new byte[0];
     }
   }
-
 }
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/python/PythonGateway.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/python/PythonGateway.java
index e67adb898..fd59a0ffd 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/python/PythonGateway.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/python/PythonGateway.java
@@ -72,8 +72,8 @@ public class PythonGateway {
   public byte[] createPythonStreamSource(byte[] pySourceFunc) {
     Preconditions.checkNotNull(streamingContext);
     try {
-      PythonStreamSource pythonStreamSource = PythonStreamSource.from(
-          streamingContext, new PythonFunction(pySourceFunc));
+      PythonStreamSource pythonStreamSource =
+          PythonStreamSource.from(streamingContext, new PythonFunction(pySourceFunc));
       referenceMap.put(getReferenceId(pythonStreamSource), pythonStreamSource);
       return serializer.serialize(getReferenceId(pythonStreamSource));
     } catch (Exception e) {
@@ -104,8 +104,7 @@ public class PythonGateway {
     List streams = (List) serializer.deserialize(paramsBytes);
     streams = processParameters(streams);
     LOG.info("Call union with streams {}", streams);
-    Preconditions.checkArgument(streams.size() >= 2,
-        "Union needs at least two streams");
+    Preconditions.checkArgument(streams.size() >= 2, "Union needs at least two streams");
     Stream unionStream;
     Stream stream1 = (Stream) streams.get(0);
     List otherStreams = streams.subList(1, streams.size());
@@ -128,8 +127,8 @@ public class PythonGateway {
       String className = (String) params.get(0);
       String funcName = (String) params.get(1);
       Class clz = Class.forName(className, true, this.getClass().getClassLoader());
-      Class[] paramsTypes = params.subList(2, params.size()).stream()
-          .map(Object::getClass).toArray(Class[]::new);
+      Class[] paramsTypes =
+          params.subList(2, params.size()).stream().map(Object::getClass).toArray(Class[]::new);
       Method method = findMethod(clz, funcName, paramsTypes);
       Object result = method.invoke(null, params.subList(2, params.size()).toArray());
       return serialize(result);
@@ -146,8 +145,8 @@ public class PythonGateway {
       Object obj = params.get(0);
       String methodName = (String) params.get(1);
       Class clz = obj.getClass();
-      Class[] paramsTypes = params.subList(2, params.size()).stream()
-          .map(Object::getClass).toArray(Class[]::new);
+      Class[] paramsTypes =
+          params.subList(2, params.size()).stream().map(Object::getClass).toArray(Class[]::new);
       Method method = findMethod(clz, methodName, paramsTypes);
       Object result = method.invoke(obj, params.subList(2, params.size()).toArray());
       return serialize(result);
@@ -162,31 +161,36 @@ public class PythonGateway {
       return methods.get(0);
     }
     // Convert all params types to primitive types if it's boxed type
-    Class[] unwrappedTypes = Arrays.stream(paramsTypes)
-        .map((Function) Primitives::unwrap)
-        .toArray(Class[]::new);
-    Optional any = methods.stream()
-        .filter(m -> {
-          boolean exactMatch =
-              Arrays.equals(m.getParameterTypes(), paramsTypes) ||
-                  Arrays.equals(m.getParameterTypes(), unwrappedTypes);
-          if (exactMatch) {
-            return true;
-          } else if (paramsTypes.length == m.getParameterTypes().length) {
-            for (int i = 0; i < m.getParameterTypes().length; i++) {
-              Class parameterType = m.getParameterTypes()[i];
-              if (!parameterType.isAssignableFrom(paramsTypes[i])) {
-                return false;
-              }
-            }
-            return true;
-          } else {
-            return false;
-          }
-        })
-        .findAny();
-    Preconditions.checkArgument(any.isPresent(),
-        String.format("Method %s with type %s doesn't exist on class %s",
+    Class[] unwrappedTypes =
+        Arrays.stream(paramsTypes)
+            .map((Function) Primitives::unwrap)
+            .toArray(Class[]::new);
+    Optional any =
+        methods.stream()
+            .filter(
+                m -> {
+                  boolean exactMatch =
+                      Arrays.equals(m.getParameterTypes(), paramsTypes)
+                          || Arrays.equals(m.getParameterTypes(), unwrappedTypes);
+                  if (exactMatch) {
+                    return true;
+                  } else if (paramsTypes.length == m.getParameterTypes().length) {
+                    for (int i = 0; i < m.getParameterTypes().length; i++) {
+                      Class parameterType = m.getParameterTypes()[i];
+                      if (!parameterType.isAssignableFrom(paramsTypes[i])) {
+                        return false;
+                      }
+                    }
+                    return true;
+                  } else {
+                    return false;
+                  }
+                })
+            .findAny();
+    Preconditions.checkArgument(
+        any.isPresent(),
+        String.format(
+            "Method %s with type %s doesn't exist on class %s",
             methodName, Arrays.toString(paramsTypes), cls));
     return any.get();
   }
@@ -214,8 +218,11 @@ public class PythonGateway {
   }
 
   private static boolean isBasic(Object value) {
-    return value == null || (value instanceof Boolean) || (value instanceof Number) ||
-        (value instanceof String) || (value instanceof byte[]);
+    return value == null
+        || (value instanceof Boolean)
+        || (value instanceof Number)
+        || (value instanceof String)
+        || (value instanceof byte[]);
   }
 
   public byte[] newInstance(byte[] classNameBytes) {
@@ -232,8 +239,7 @@ public class PythonGateway {
   }
 
   private List processParameters(List params) {
-    return params.stream().map(this::processParameter)
-        .collect(Collectors.toList());
+    return params.stream().map(this::processParameter).collect(Collectors.toList());
   }
 
   private Object processParameter(Object o) {
@@ -253,5 +259,4 @@ public class PythonGateway {
   private String getReferenceId(Object o) {
     return REFERENCE_ID_PREFIX + System.identityHashCode(o);
   }
-
 }
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/PbResultParser.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/PbResultParser.java
index c0bbcc2c2..48ba7aa73 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/PbResultParser.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/PbResultParser.java
@@ -45,9 +45,13 @@ public class PbResultParser {
     callResult.setResultMsg(callResultPb.getResultMsg());
     RemoteCall.QueueRecoverInfo recoverInfo = callResultPb.getResultObj();
     Map creationStatusMap = new HashMap<>();
-    recoverInfo.getCreationStatusMap().forEach((k, v) -> {
-      creationStatusMap.put(k, ChannelRecoverInfo.ChannelCreationStatus.fromInt(v.getNumber()));
-    });
+    recoverInfo
+        .getCreationStatusMap()
+        .forEach(
+            (k, v) -> {
+              creationStatusMap.put(
+                  k, ChannelRecoverInfo.ChannelCreationStatus.fromInt(v.getNumber()));
+            });
     callResult.setResultObj(new ChannelRecoverInfo(creationStatusMap));
     return callResult;
   }
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/RemoteCallMaster.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/RemoteCallMaster.java
index fe25002bf..1dc69751d 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/RemoteCallMaster.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/RemoteCallMaster.java
@@ -12,34 +12,39 @@ import io.ray.streaming.runtime.master.coordinator.command.WorkerRollbackRequest
 public class RemoteCallMaster {
 
   public static ObjectRef reportJobWorkerCommitAsync(
-      ActorHandle actor,
-      WorkerCommitReport commitReport) {
-    RemoteCall.WorkerCommitReport commit = RemoteCall.WorkerCommitReport.newBuilder()
-        .setCommitCheckpointId(commitReport.commitCheckpointId)
-        .build();
+      ActorHandle actor, WorkerCommitReport commitReport) {
+    RemoteCall.WorkerCommitReport commit =
+        RemoteCall.WorkerCommitReport.newBuilder()
+            .setCommitCheckpointId(commitReport.commitCheckpointId)
+            .build();
     Any detail = Any.pack(commit);
-    RemoteCall.BaseWorkerCmd cmd = RemoteCall.BaseWorkerCmd.newBuilder()
-        .setActorId(ByteString.copyFrom(commitReport.fromActorId.getBytes()))
-        .setTimestamp(System.currentTimeMillis())
-        .setDetail(detail).build();
+    RemoteCall.BaseWorkerCmd cmd =
+        RemoteCall.BaseWorkerCmd.newBuilder()
+            .setActorId(ByteString.copyFrom(commitReport.fromActorId.getBytes()))
+            .setTimestamp(System.currentTimeMillis())
+            .setDetail(detail)
+            .build();
 
     return actor.task(JobMaster::reportJobWorkerCommit, cmd.toByteArray()).remote();
   }
 
   public static Boolean requestJobWorkerRollback(
-      ActorHandle actor,
-      WorkerRollbackRequest rollbackRequest) {
-    RemoteCall.WorkerRollbackRequest request = RemoteCall.WorkerRollbackRequest.newBuilder()
-        .setExceptionMsg(rollbackRequest.getRollbackExceptionMsg())
-        .setWorkerHostname(rollbackRequest.getHostname())
-        .setWorkerPid(rollbackRequest.getPid()).build();
+      ActorHandle actor, WorkerRollbackRequest rollbackRequest) {
+    RemoteCall.WorkerRollbackRequest request =
+        RemoteCall.WorkerRollbackRequest.newBuilder()
+            .setExceptionMsg(rollbackRequest.getRollbackExceptionMsg())
+            .setWorkerHostname(rollbackRequest.getHostname())
+            .setWorkerPid(rollbackRequest.getPid())
+            .build();
     Any detail = Any.pack(request);
-    RemoteCall.BaseWorkerCmd cmd = RemoteCall.BaseWorkerCmd.newBuilder()
-        .setActorId(ByteString.copyFrom(rollbackRequest.fromActorId.getBytes()))
-        .setTimestamp(System.currentTimeMillis())
-        .setDetail(detail).build();
-    ObjectRef ret = actor.task(
-        JobMaster::requestJobWorkerRollback, cmd.toByteArray()).remote();
+    RemoteCall.BaseWorkerCmd cmd =
+        RemoteCall.BaseWorkerCmd.newBuilder()
+            .setActorId(ByteString.copyFrom(rollbackRequest.fromActorId.getBytes()))
+            .setTimestamp(System.currentTimeMillis())
+            .setDetail(detail)
+            .build();
+    ObjectRef ret =
+        actor.task(JobMaster::requestJobWorkerRollback, cmd.toByteArray()).remote();
     byte[] res = ret.get();
     return PbResultParser.parseBoolResult(res);
   }
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/RemoteCallWorker.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/RemoteCallWorker.java
index d9b373370..5a5475350 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/RemoteCallWorker.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/RemoteCallWorker.java
@@ -16,9 +16,7 @@ import java.util.List;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-/**
- * Ray call worker. It takes the communication job from {@link JobMaster} to {@link JobWorker}.
- */
+/** Ray call worker. It takes the communication job from {@link JobMaster} to {@link JobWorker}. */
 public class RemoteCallWorker {
 
   private static final Logger LOG = LoggerFactory.getLogger(RemoteCallWorker.class);
@@ -27,8 +25,7 @@ public class RemoteCallWorker {
    * Call JobWorker actor to init.
    *
    * @param actor target JobWorker actor
-   * @param context JobWorker's context
-   * @return init result
+   * @param context JobWorker's context Returns init result
    */
   public static ObjectRef initWorker(BaseActorHandle actor, JobWorkerContext context) {
     LOG.info("Call worker to initiate, actor: {}, context: {}.", actor.getId(), context);
@@ -36,8 +33,10 @@ public class RemoteCallWorker {
 
     // python
     if (actor instanceof PyActorHandle) {
-      result = ((PyActorHandle) actor).task(PyActorMethod.of("init", Boolean.class),
-          context.getPythonWorkerContextBytes()).remote();
+      result =
+          ((PyActorHandle) actor)
+              .task(PyActorMethod.of("init", Boolean.class), context.getPythonWorkerContextBytes())
+              .remote();
     } else {
       // java
       result = ((ActorHandle) actor).task(JobWorker::init, context).remote();
@@ -51,8 +50,7 @@ public class RemoteCallWorker {
    * Call JobWorker actor to start.
    *
    * @param actor target JobWorker actor
-   * @param checkpointId checkpoint ID to be rollback
-   * @return start result
+   * @param checkpointId checkpoint ID to be rollback Returns start result
    */
   public static ObjectRef rollback(BaseActorHandle actor, final Long checkpointId) {
     LOG.info("Call worker to start, actor: {}.", actor.getId());
@@ -60,17 +58,18 @@ public class RemoteCallWorker {
 
     // python
     if (actor instanceof PyActorHandle) {
-      RemoteCall.CheckpointId checkpointIdPb = RemoteCall.CheckpointId.newBuilder()
-          .setCheckpointId(checkpointId)
-          .build();
-      result = ((PyActorHandle) actor)
-          .task(PyActorMethod.of("rollback"),
-              checkpointIdPb.toByteArray()
-          ).remote();
+      RemoteCall.CheckpointId checkpointIdPb =
+          RemoteCall.CheckpointId.newBuilder().setCheckpointId(checkpointId).build();
+      result =
+          ((PyActorHandle) actor)
+              .task(PyActorMethod.of("rollback"), checkpointIdPb.toByteArray())
+              .remote();
     } else {
       // java
-      result = ((ActorHandle) actor)
-          .task(JobWorker::rollback, checkpointId, System.currentTimeMillis()).remote();
+      result =
+          ((ActorHandle) actor)
+              .task(JobWorker::rollback, checkpointId, System.currentTimeMillis())
+              .remote();
     }
 
     LOG.info("Finished calling worker to start.");
@@ -80,12 +79,10 @@ public class RemoteCallWorker {
   /**
    * Call JobWorker actor to destroy without reconstruction.
    *
-   * @param actor target JobWorker actor
-   * @return destroy result
+   * @param actor target JobWorker actor Returns destroy result
    */
   public static Boolean shutdownWithoutReconstruction(BaseActorHandle actor) {
-    LOG.info("Call worker to shutdown without reconstruction, actor is {}.",
-        actor.getId());
+    LOG.info("Call worker to shutdown without reconstruction, actor is {}.", actor.getId());
     Boolean result = false;
 
     // TODO (datayjz): ray call worker to destroy
@@ -98,26 +95,34 @@ public class RemoteCallWorker {
     // python
     if (actor instanceof PyActorHandle) {
       RemoteCall.Barrier barrierPb = RemoteCall.Barrier.newBuilder().setId(barrierId).build();
-      return ((PyActorHandle) actor).task(
-          PyActorMethod.of("commit"), barrierPb.toByteArray()).remote();
+      return ((PyActorHandle) actor)
+          .task(PyActorMethod.of("commit"), barrierPb.toByteArray())
+          .remote();
     } else {
       // java
-      return ((ActorHandle) actor).task(JobWorker::triggerCheckpoint, barrierId)
+      return ((ActorHandle) actor)
+          .task(JobWorker::triggerCheckpoint, barrierId)
           .remote();
     }
   }
 
   public static void clearExpiredCheckpointParallel(
-      List actors, Long stateCheckpointId,
-      Long queueCheckpointId) {
+      List actors, Long stateCheckpointId, Long queueCheckpointId) {
     if (LOG.isInfoEnabled()) {
-      LOG.info("Call worker clearExpiredCheckpoint, state checkpoint id is {}," +
-          " queue checkpoint id is {}.", stateCheckpointId, queueCheckpointId);
+      LOG.info(
+          "Call worker clearExpiredCheckpoint, state checkpoint id is {},"
+              + " queue checkpoint id is {}.",
+          stateCheckpointId,
+          queueCheckpointId);
     }
 
     List result =
-        checkpointCompleteCommonCallTwoWay(actors, stateCheckpointId, queueCheckpointId,
-            "clear_expired_cp", JobWorker::clearExpiredCheckpoint);
+        checkpointCompleteCommonCallTwoWay(
+            actors,
+            stateCheckpointId,
+            queueCheckpointId,
+            "clear_expired_cp",
+            JobWorker::clearExpiredCheckpoint);
 
     if (LOG.isInfoEnabled()) {
       result.forEach(
@@ -126,60 +131,68 @@ public class RemoteCallWorker {
   }
 
   public static void notifyCheckpointTimeoutParallel(
-      List actors,
-      Long checkpointId) {
+      List actors, Long checkpointId) {
     LOG.info("Call worker notifyCheckpointTimeoutParallel, checkpoint id is {}", checkpointId);
 
-    actors.forEach(actor -> {
-      if (actor instanceof PyActorHandle) {
-        RemoteCall.CheckpointId checkpointIdPb = RemoteCall.CheckpointId.newBuilder()
-            .setCheckpointId(checkpointId)
-            .build();
-        ((PyActorHandle) actor).task(PyActorMethod.of("notify_checkpoint_timeout"),
-            checkpointIdPb.toByteArray()).remote();
-      } else {
-        ((ActorHandle) actor).task(JobWorker::notifyCheckpointTimeout, checkpointId)
-            .remote();
-      }
-    });
+    actors.forEach(
+        actor -> {
+          if (actor instanceof PyActorHandle) {
+            RemoteCall.CheckpointId checkpointIdPb =
+                RemoteCall.CheckpointId.newBuilder().setCheckpointId(checkpointId).build();
+            ((PyActorHandle) actor)
+                .task(PyActorMethod.of("notify_checkpoint_timeout"), checkpointIdPb.toByteArray())
+                .remote();
+          } else {
+            ((ActorHandle) actor)
+                .task(JobWorker::notifyCheckpointTimeout, checkpointId)
+                .remote();
+          }
+        });
 
     LOG.info("Finish call worker notifyCheckpointTimeoutParallel.");
   }
 
   private static List checkpointCompleteCommonCallTwoWay(
-      List actors, Long stateCheckpointId, Long queueCheckpointId,
-      String pyFuncName, RayFunc3 rayFunc) {
+      List actors,
+      Long stateCheckpointId,
+      Long queueCheckpointId,
+      String pyFuncName,
+      RayFunc3 rayFunc) {
     List> waitFor =
-        checkpointCompleteCommonCall(actors, stateCheckpointId, queueCheckpointId,
-            pyFuncName, rayFunc);
+        checkpointCompleteCommonCall(
+            actors, stateCheckpointId, queueCheckpointId, pyFuncName, rayFunc);
     return Ray.get(waitFor);
   }
 
   private static List> checkpointCompleteCommonCall(
       List actors,
-      Long stateCheckpointId, Long queueCheckpointId,
+      Long stateCheckpointId,
+      Long queueCheckpointId,
       String pyFuncName,
       RayFunc3 rayFunc) {
     List> waitFor = new ArrayList<>();
-    actors.forEach(actor -> {
-      // python
-      if (actor instanceof PyActorHandle) {
-        RemoteCall.CheckpointId stateCheckpointIdPb = RemoteCall.CheckpointId.newBuilder()
-            .setCheckpointId(stateCheckpointId)
-            .build();
+    actors.forEach(
+        actor -> {
+          // python
+          if (actor instanceof PyActorHandle) {
+            RemoteCall.CheckpointId stateCheckpointIdPb =
+                RemoteCall.CheckpointId.newBuilder().setCheckpointId(stateCheckpointId).build();
 
-        RemoteCall.CheckpointId queueCheckpointIdPb = RemoteCall.CheckpointId.newBuilder()
-            .setCheckpointId(queueCheckpointId)
-            .build();
-        waitFor.add(((PyActorHandle) actor).task(PyActorMethod.of(pyFuncName),
-            stateCheckpointIdPb.toByteArray(), queueCheckpointIdPb.toByteArray()).remote());
-      } else {
-        // java
-        waitFor.add(((ActorHandle) actor).task(rayFunc, stateCheckpointId, queueCheckpointId)
-            .remote());
-      }
-    });
+            RemoteCall.CheckpointId queueCheckpointIdPb =
+                RemoteCall.CheckpointId.newBuilder().setCheckpointId(queueCheckpointId).build();
+            waitFor.add(
+                ((PyActorHandle) actor)
+                    .task(
+                        PyActorMethod.of(pyFuncName),
+                        stateCheckpointIdPb.toByteArray(),
+                        queueCheckpointIdPb.toByteArray())
+                    .remote());
+          } else {
+            // java
+            waitFor.add(
+                ((ActorHandle) actor).task(rayFunc, stateCheckpointId, queueCheckpointId).remote());
+          }
+        });
     return waitFor;
   }
-
 }
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/async/AsyncRemoteCaller.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/async/AsyncRemoteCaller.java
index db7937159..360c1b23f 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/async/AsyncRemoteCaller.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/async/AsyncRemoteCaller.java
@@ -34,8 +34,7 @@ public class AsyncRemoteCaller {
    * @param onException callback function on exception
    */
   public void checkIfNeedRollbackAsync(
-      BaseActorHandle actor, Callback callback,
-      ExceptionHandler onException) {
+      BaseActorHandle actor, Callback callback, ExceptionHandler onException) {
     if (actor instanceof PyActorHandle) {
       // python
       remoteCallPool.bindCallback(
@@ -43,12 +42,16 @@ public class AsyncRemoteCaller {
           (obj) -> {
             byte[] res = (byte[]) obj;
             callback.handle(PbResultParser.parseBoolResult(res));
-          }, onException);
+          },
+          onException);
     } else {
       // java
       remoteCallPool.bindCallback(
-          ((ActorHandle) actor).task(JobWorker::checkIfNeedRollback,
-              System.currentTimeMillis()).remote(), callback, onException);
+          ((ActorHandle) actor)
+              .task(JobWorker::checkIfNeedRollback, System.currentTimeMillis())
+              .remote(),
+          callback,
+          onException);
     }
   }
 
@@ -66,21 +69,29 @@ public class AsyncRemoteCaller {
       ExceptionHandler onException) {
     // python
     if (actor instanceof PyActorHandle) {
-      RemoteCall.CheckpointId checkpointIdPb = RemoteCall.CheckpointId.newBuilder()
-          .setCheckpointId(checkpointId)
-          .build();
-      ObjectRef call = ((PyActorHandle) actor).task(PyActorMethod.of("rollback"),
-          checkpointIdPb.toByteArray()).remote();
-      remoteCallPool.bindCallback(call, obj ->
-          callback.handle(PbResultParser.parseRollbackResult((byte[]) obj)), onException);
+      RemoteCall.CheckpointId checkpointIdPb =
+          RemoteCall.CheckpointId.newBuilder().setCheckpointId(checkpointId).build();
+      ObjectRef call =
+          ((PyActorHandle) actor)
+              .task(PyActorMethod.of("rollback"), checkpointIdPb.toByteArray())
+              .remote();
+      remoteCallPool.bindCallback(
+          call,
+          obj -> callback.handle(PbResultParser.parseRollbackResult((byte[]) obj)),
+          onException);
     } else {
       // java
-      ObjectRef call = ((ActorHandle) actor).task(
-          JobWorker::rollback, checkpointId, System.currentTimeMillis()).remote();
-      remoteCallPool.bindCallback(call, obj -> {
-        CallResult res = (CallResult) obj;
-        callback.handle(res);
-      }, onException);
+      ObjectRef call =
+          ((ActorHandle) actor)
+              .task(JobWorker::rollback, checkpointId, System.currentTimeMillis())
+              .remote();
+      remoteCallPool.bindCallback(
+          call,
+          obj -> {
+            CallResult res = (CallResult) obj;
+            callback.handle(res);
+          },
+          onException);
     }
   }
 
@@ -92,7 +103,8 @@ public class AsyncRemoteCaller {
    * @param onException callback function on exception
    */
   public void batchRollback(
-      List actors, final Long checkpointId,
+      List actors,
+      final Long checkpointId,
       Collection abnormalQueues,
       Callback>> callback,
       ExceptionHandler onException) {
@@ -103,29 +115,35 @@ public class AsyncRemoteCaller {
       ObjectRef call;
       if (actor instanceof PyActorHandle) {
         isPyActor.put(i, true);
-        RemoteCall.CheckpointId checkpointIdPb = RemoteCall.CheckpointId.newBuilder()
-            .setCheckpointId(checkpointId)
-            .build();
-        call = ((PyActorHandle) actor).task(PyActorMethod.of("rollback"),
-            checkpointIdPb.toByteArray()).remote();
+        RemoteCall.CheckpointId checkpointIdPb =
+            RemoteCall.CheckpointId.newBuilder().setCheckpointId(checkpointId).build();
+        call =
+            ((PyActorHandle) actor)
+                .task(PyActorMethod.of("rollback"), checkpointIdPb.toByteArray())
+                .remote();
       } else {
         // java
-        call = ((ActorHandle) actor).task(JobWorker::rollback, checkpointId,
-            System.currentTimeMillis()).remote();
+        call =
+            ((ActorHandle) actor)
+                .task(JobWorker::rollback, checkpointId, System.currentTimeMillis())
+                .remote();
       }
       rayCallList.add(call);
     }
-    remoteCallPool.bindCallback(rayCallList, objList -> {
-      List> results = new ArrayList<>();
-      for (int i = 0; i < objList.size(); ++i) {
-        Object obj = objList.get(i);
-        if (isPyActor.getOrDefault(i, false)) {
-          results.add(PbResultParser.parseRollbackResult((byte[]) obj));
-        } else {
-          results.add((CallResult) obj);
-        }
-      }
-      callback.handle(results);
-    }, onException);
+    remoteCallPool.bindCallback(
+        rayCallList,
+        objList -> {
+          List> results = new ArrayList<>();
+          for (int i = 0; i < objList.size(); ++i) {
+            Object obj = objList.get(i);
+            if (isPyActor.getOrDefault(i, false)) {
+              results.add(PbResultParser.parseRollbackResult((byte[]) obj));
+            } else {
+              results.add((CallResult) obj);
+            }
+          }
+          callback.handle(results);
+        },
+        onException);
   }
 }
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/async/RemoteCallPool.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/async/RemoteCallPool.java
index 52e9e5651..a60d916f2 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/async/RemoteCallPool.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/rpc/async/RemoteCallPool.java
@@ -18,7 +18,6 @@ import java.util.stream.Collectors;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-
 public class RemoteCallPool implements Runnable {
 
   private static final Logger LOG = LoggerFactory.getLogger(RemoteCallPool.class);
@@ -30,27 +29,28 @@ public class RemoteCallPool implements Runnable {
       new ConcurrentHashMap<>();
   private Map> bundleExceptionHandlerMap =
       new ConcurrentHashMap<>();
-  private ThreadPoolExecutor callBackPool = new ThreadPoolExecutor(
-      2, Runtime.getRuntime().availableProcessors(),
-      1, TimeUnit.MINUTES, new LinkedBlockingQueue<>(),
-      new CallbackThreadFactory());
+  private ThreadPoolExecutor callBackPool =
+      new ThreadPoolExecutor(
+          2,
+          Runtime.getRuntime().availableProcessors(),
+          1,
+          TimeUnit.MINUTES,
+          new LinkedBlockingQueue<>(),
+          new CallbackThreadFactory());
   private volatile boolean stop = false;
 
   public RemoteCallPool() {
     Thread t = new Thread(Ray.wrapRunnable(this), "remote-pool-loop");
-    t.setUncaughtExceptionHandler((thread, throwable) ->
-        LOG.error("Error in remote call pool thread.", throwable)
-    );
+    t.setUncaughtExceptionHandler(
+        (thread, throwable) -> LOG.error("Error in remote call pool thread.", throwable));
     t.start();
   }
 
   @SuppressWarnings("unchecked")
   public  void bindCallback(
-      ObjectRef obj, Callback callback,
-      ExceptionHandler onException) {
+      ObjectRef obj, Callback callback, ExceptionHandler onException) {
     List objectRefList = Collections.singletonList(obj);
-    RemoteCallBundle bundle = new RemoteCallBundle(objectRefList,
-        true);
+    RemoteCallBundle bundle = new RemoteCallBundle(objectRefList, true);
     singletonHandlerMap.put(bundle, (Callback) callback);
     bundleExceptionHandlerMap.put(bundle, onException);
     synchronized (pendingObjectBundles) {
@@ -59,7 +59,8 @@ public class RemoteCallPool implements Runnable {
   }
 
   public void bindCallback(
-      List> objectBundle, Callback> callback,
+      List> objectBundle,
+      Callback> callback,
       ExceptionHandler onException) {
     RemoteCallBundle bundle = new RemoteCallBundle(objectBundle, false);
     bundleHandlerMap.put(bundle, callback);
@@ -99,34 +100,40 @@ public class RemoteCallPool implements Runnable {
 
             ExceptionHandler exceptionHandler = bundleExceptionHandlerMap.get(bundle);
             if (bundle.isSingletonBundle) {
-              callBackPool.execute(Ray.wrapRunnable(() -> {
-                try {
-                  singletonHandlerMap.get(bundle).handle(readyObjs.get(0).get());
-                  singletonHandlerMap.remove(bundle);
-                } catch (Throwable th) {
-                  LOG.error("Error when get object, objectId = {}.", readyObjs.get(0).toString(),
-                      th);
-                  if (exceptionHandler != null) {
-                    exceptionHandler.handle(th);
-                  }
-                }
-              }));
+              callBackPool.execute(
+                  Ray.wrapRunnable(
+                      () -> {
+                        try {
+                          singletonHandlerMap.get(bundle).handle(readyObjs.get(0).get());
+                          singletonHandlerMap.remove(bundle);
+                        } catch (Throwable th) {
+                          LOG.error(
+                              "Error when get object, objectId = {}.",
+                              readyObjs.get(0).toString(),
+                              th);
+                          if (exceptionHandler != null) {
+                            exceptionHandler.handle(th);
+                          }
+                        }
+                      }));
             } else {
               List results =
                   readyObjs.stream().map(ObjectRef::get).collect(Collectors.toList());
               List resultIds =
                   readyObjs.stream().map(ObjectRef::toString).collect(Collectors.toList());
-              callBackPool.execute(Ray.wrapRunnable(() -> {
-                try {
-                  bundleHandlerMap.get(bundle).handle(results);
-                  bundleHandlerMap.remove(bundle);
-                } catch (Throwable th) {
-                  LOG.error("Error when get object, objectIds = {}.", resultIds, th);
-                  if (exceptionHandler != null) {
-                    exceptionHandler.handle(th);
-                  }
-                }
-              }));
+              callBackPool.execute(
+                  Ray.wrapRunnable(
+                      () -> {
+                        try {
+                          bundleHandlerMap.get(bundle).handle(results);
+                          bundleHandlerMap.remove(bundle);
+                        } catch (Throwable th) {
+                          LOG.error("Error when get object, objectIds = {}.", resultIds, th);
+                          if (exceptionHandler != null) {
+                            exceptionHandler.handle(th);
+                          }
+                        }
+                      }));
             }
             itr.remove();
           }
@@ -185,5 +192,4 @@ public class RemoteCallPool implements Runnable {
       return t;
     }
   }
-
 }
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/serialization/CrossLangSerializer.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/serialization/CrossLangSerializer.java
index 56ff42832..ee987146a 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/serialization/CrossLangSerializer.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/serialization/CrossLangSerializer.java
@@ -21,13 +21,12 @@ public class CrossLangSerializer implements Serializer {
     Object value = record.getValue();
     Class clz = record.getClass();
     if (clz == Record.class) {
-      return msgPackSerializer.serialize(Arrays.asList(
-          RECORD_TYPE_ID, record.getStream(), value));
+      return msgPackSerializer.serialize(Arrays.asList(RECORD_TYPE_ID, record.getStream(), value));
     } else if (clz == KeyRecord.class) {
       KeyRecord keyRecord = (KeyRecord) record;
       Object key = keyRecord.getKey();
-      return msgPackSerializer.serialize(Arrays.asList(
-          KEY_RECORD_TYPE_ID, keyRecord.getStream(), key, value));
+      return msgPackSerializer.serialize(
+          Arrays.asList(KEY_RECORD_TYPE_ID, keyRecord.getStream(), key, value));
     } else {
       throw new UnsupportedOperationException(
           String.format("Serialize %s is unsupported.", record));
@@ -39,25 +38,25 @@ public class CrossLangSerializer implements Serializer {
     List list = (List) msgPackSerializer.deserialize(bytes);
     Byte typeId = (Byte) list.get(0);
     switch (typeId) {
-      case RECORD_TYPE_ID: {
-        String stream = (String) list.get(1);
-        Object value = list.get(2);
-        Record record = new Record(value);
-        record.setStream(stream);
-        return record;
-      }
-      case KEY_RECORD_TYPE_ID: {
-        String stream = (String) list.get(1);
-        Object key = list.get(2);
-        Object value = list.get(3);
-        KeyRecord keyRecord = new KeyRecord(key, value);
-        keyRecord.setStream(stream);
-        return keyRecord;
-      }
+      case RECORD_TYPE_ID:
+        {
+          String stream = (String) list.get(1);
+          Object value = list.get(2);
+          Record record = new Record(value);
+          record.setStream(stream);
+          return record;
+        }
+      case KEY_RECORD_TYPE_ID:
+        {
+          String stream = (String) list.get(1);
+          Object key = list.get(2);
+          Object value = list.get(3);
+          KeyRecord keyRecord = new KeyRecord(key, value);
+          keyRecord.setStream(stream);
+          return keyRecord;
+        }
       default:
         throw new UnsupportedOperationException("Unsupported type " + typeId);
-
     }
   }
-
 }
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/serialization/Serializer.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/serialization/Serializer.java
index 92449a736..0be25b9a9 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/serialization/Serializer.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/serialization/Serializer.java
@@ -9,5 +9,4 @@ public interface Serializer {
   byte[] serialize(Object object);
 
    T deserialize(byte[] bytes);
-
 }
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/ChannelCreationParametersBuilder.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/ChannelCreationParametersBuilder.java
index ff8b92c5d..2e964cb51 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/ChannelCreationParametersBuilder.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/ChannelCreationParametersBuilder.java
@@ -13,9 +13,7 @@ import io.ray.streaming.runtime.worker.JobWorker;
 import java.util.ArrayList;
 import java.util.List;
 
-/**
- * Save channel initial parameters needed by DataWriter/DataReader.
- */
+/** Save channel initial parameters needed by DataWriter/DataReader. */
 public class ChannelCreationParametersBuilder {
 
   public static class Parameter {
@@ -28,20 +26,22 @@ public class ChannelCreationParametersBuilder {
       this.actorId = actorId;
     }
 
-    public void setAsyncFunctionDescriptor(
-        FunctionDescriptor asyncFunctionDescriptor) {
+    public void setAsyncFunctionDescriptor(FunctionDescriptor asyncFunctionDescriptor) {
       this.asyncFunctionDescriptor = asyncFunctionDescriptor;
     }
 
-    public void setSyncFunctionDescriptor(
-        FunctionDescriptor syncFunctionDescriptor) {
+    public void setSyncFunctionDescriptor(FunctionDescriptor syncFunctionDescriptor) {
       this.syncFunctionDescriptor = syncFunctionDescriptor;
     }
 
     public String toString() {
       String language =
           asyncFunctionDescriptor instanceof JavaFunctionDescriptor ? "Java" : "Python";
-      return "Language: " + language + " Desc: " + asyncFunctionDescriptor.toList() + " "
+      return "Language: "
+          + language
+          + " Desc: "
+          + asyncFunctionDescriptor.toList()
+          + " "
           + syncFunctionDescriptor.toList();
     }
 
@@ -64,61 +64,60 @@ public class ChannelCreationParametersBuilder {
   private List parameters;
 
   // function descriptors of direct call entry point for Java workers
-  private static JavaFunctionDescriptor javaReaderAsyncFuncDesc = new JavaFunctionDescriptor(
-      JobWorker.class.getName(),
-      "onReaderMessage", "([B)V");
-  private static JavaFunctionDescriptor javaReaderSyncFuncDesc = new JavaFunctionDescriptor(
-      JobWorker.class.getName(),
-      "onReaderMessageSync", "([B)[B");
-  private static JavaFunctionDescriptor javaWriterAsyncFuncDesc = new JavaFunctionDescriptor(
-      JobWorker.class.getName(),
-      "onWriterMessage", "([B)V");
-  private static JavaFunctionDescriptor javaWriterSyncFuncDesc = new JavaFunctionDescriptor(
-      JobWorker.class.getName(),
-      "onWriterMessageSync", "([B)[B");
+  private static JavaFunctionDescriptor javaReaderAsyncFuncDesc =
+      new JavaFunctionDescriptor(JobWorker.class.getName(), "onReaderMessage", "([B)V");
+  private static JavaFunctionDescriptor javaReaderSyncFuncDesc =
+      new JavaFunctionDescriptor(JobWorker.class.getName(), "onReaderMessageSync", "([B)[B");
+  private static JavaFunctionDescriptor javaWriterAsyncFuncDesc =
+      new JavaFunctionDescriptor(JobWorker.class.getName(), "onWriterMessage", "([B)V");
+  private static JavaFunctionDescriptor javaWriterSyncFuncDesc =
+      new JavaFunctionDescriptor(JobWorker.class.getName(), "onWriterMessageSync", "([B)[B");
   // function descriptors of direct call entry point for Python workers
-  private static PyFunctionDescriptor pyReaderAsyncFunctionDesc = new PyFunctionDescriptor(
-      "ray.streaming.runtime.worker",
-      "JobWorker", "on_reader_message");
-  private static PyFunctionDescriptor pyReaderSyncFunctionDesc = new PyFunctionDescriptor(
-      "ray.streaming.runtime.worker",
-      "JobWorker", "on_reader_message_sync");
-  private static PyFunctionDescriptor pyWriterAsyncFunctionDesc = new PyFunctionDescriptor(
-      "ray.streaming.runtime.worker",
-      "JobWorker", "on_writer_message");
-  private static PyFunctionDescriptor pyWriterSyncFunctionDesc = new PyFunctionDescriptor(
-      "ray.streaming.runtime.worker",
-      "JobWorker", "on_writer_message_sync");
+  private static PyFunctionDescriptor pyReaderAsyncFunctionDesc =
+      new PyFunctionDescriptor("ray.streaming.runtime.worker", "JobWorker", "on_reader_message");
+  private static PyFunctionDescriptor pyReaderSyncFunctionDesc =
+      new PyFunctionDescriptor(
+          "ray.streaming.runtime.worker", "JobWorker", "on_reader_message_sync");
+  private static PyFunctionDescriptor pyWriterAsyncFunctionDesc =
+      new PyFunctionDescriptor("ray.streaming.runtime.worker", "JobWorker", "on_writer_message");
+  private static PyFunctionDescriptor pyWriterSyncFunctionDesc =
+      new PyFunctionDescriptor(
+          "ray.streaming.runtime.worker", "JobWorker", "on_writer_message_sync");
 
-  public ChannelCreationParametersBuilder() {
-  }
+  public ChannelCreationParametersBuilder() {}
 
   public static void setJavaReaderFunctionDesc(
-      JavaFunctionDescriptor asyncFunc,
-      JavaFunctionDescriptor syncFunc) {
+      JavaFunctionDescriptor asyncFunc, JavaFunctionDescriptor syncFunc) {
     javaReaderAsyncFuncDesc = asyncFunc;
     javaReaderSyncFuncDesc = syncFunc;
   }
 
   public static void setJavaWriterFunctionDesc(
-      JavaFunctionDescriptor asyncFunc,
-      JavaFunctionDescriptor syncFunc) {
+      JavaFunctionDescriptor asyncFunc, JavaFunctionDescriptor syncFunc) {
     javaWriterAsyncFuncDesc = asyncFunc;
     javaWriterSyncFuncDesc = syncFunc;
   }
 
   public ChannelCreationParametersBuilder buildInputQueueParameters(
-      List queues,
-      List actors) {
-    return buildParameters(queues, actors, javaWriterAsyncFuncDesc, javaWriterSyncFuncDesc,
-        pyWriterAsyncFunctionDesc, pyWriterSyncFunctionDesc);
+      List queues, List actors) {
+    return buildParameters(
+        queues,
+        actors,
+        javaWriterAsyncFuncDesc,
+        javaWriterSyncFuncDesc,
+        pyWriterAsyncFunctionDesc,
+        pyWriterSyncFunctionDesc);
   }
 
   public ChannelCreationParametersBuilder buildOutputQueueParameters(
-      List queues,
-      List actors) {
-    return buildParameters(queues, actors, javaReaderAsyncFuncDesc, javaReaderSyncFuncDesc,
-        pyReaderAsyncFunctionDesc, pyReaderSyncFunctionDesc);
+      List queues, List actors) {
+    return buildParameters(
+        queues,
+        actors,
+        javaReaderAsyncFuncDesc,
+        javaReaderSyncFuncDesc,
+        pyReaderAsyncFunctionDesc,
+        pyReaderSyncFunctionDesc);
   }
 
   private ChannelCreationParametersBuilder buildParameters(
@@ -127,8 +126,7 @@ public class ChannelCreationParametersBuilder {
       JavaFunctionDescriptor javaAsyncFunctionDesc,
       JavaFunctionDescriptor javaSyncFunctionDesc,
       PyFunctionDescriptor pyAsyncFunctionDesc,
-      PyFunctionDescriptor pySyncFunctionDesc
-  ) {
+      PyFunctionDescriptor pySyncFunctionDesc) {
     parameters = new ArrayList<>(queues.size());
 
     for (int i = 0; i < queues.size(); ++i) {
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/DataReader.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/DataReader.java
index f10571796..9e8f6318c 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/DataReader.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/DataReader.java
@@ -62,8 +62,8 @@ public class DataReader {
     Preconditions.checkArgument(inputChannels.size() == fromActors.size());
     ChannelCreationParametersBuilder initialParameters =
         new ChannelCreationParametersBuilder().buildInputQueueParameters(inputChannels, fromActors);
-    byte[][] inputChannelsBytes = inputChannels.stream()
-        .map(ChannelId::idStrToBytes).toArray(byte[][]::new);
+    byte[][] inputChannelsBytes =
+        inputChannels.stream().map(ChannelId::idStrToBytes).toArray(byte[][]::new);
 
     // get sequence ID and message ID from OffsetInfo
     long[] msgIds = new long[inputChannels.size()];
@@ -84,21 +84,23 @@ public class DataReader {
 
     // create native reader
     List creationStatus = new ArrayList<>();
-    this.nativeReaderPtr = createDataReaderNative(
-        initialParameters,
-        inputChannelsBytes,
-        msgIds,
-        timerInterval,
-        creationStatus,
-        ChannelUtils.toNativeConf(workerConfig),
-        isMock
-    );
+    this.nativeReaderPtr =
+        createDataReaderNative(
+            initialParameters,
+            inputChannelsBytes,
+            msgIds,
+            timerInterval,
+            creationStatus,
+            ChannelUtils.toNativeConf(workerConfig),
+            isMock);
     for (int i = 0; i < inputChannels.size(); ++i) {
-      queueCreationStatusMap
-          .put(inputChannels.get(i), ChannelCreationStatus.fromInt(creationStatus.get(i)));
+      queueCreationStatusMap.put(
+          inputChannels.get(i), ChannelCreationStatus.fromInt(creationStatus.get(i)));
     }
-    LOG.info("Create DataReader succeed for worker: {}, creation status={}.",
-        workerConfig.workerInternalConfig.workerName(), queueCreationStatusMap);
+    LOG.info(
+        "Create DataReader succeed for worker: {}, creation status={}.",
+        workerConfig.workerInternalConfig.workerName(),
+        queueCreationStatusMap);
   }
 
   private static native long createDataReaderNative(
@@ -113,8 +115,7 @@ public class DataReader {
   /**
    * Read message from input channels, if timeout, return null.
    *
-   * @param timeoutMillis timeout
-   * @return message or null
+   * @param timeoutMillis timeout Returns message or null
    */
   public ChannelMessage read(long timeoutMillis) {
     if (buf.isEmpty()) {
@@ -183,8 +184,11 @@ public class DataReader {
   }
 
   private void getBundle(long timeoutMillis) {
-    getBundleNative(nativeReaderPtr, timeoutMillis,
-        Platform.getAddress(getBundleParams), Platform.getAddress(bundleMeta));
+    getBundleNative(
+        nativeReaderPtr,
+        timeoutMillis,
+        Platform.getAddress(getBundleParams),
+        Platform.getAddress(bundleMeta));
     bundleMeta.rewind();
     long bundleAddress = getBundleParams.getLong(0);
     int bundleSize = getBundleParams.getInt(8);
@@ -192,16 +196,12 @@ public class DataReader {
     Platform.wrapDirectBuffer(bundleData, bundleAddress, bundleSize);
   }
 
-  /**
-   * Stop reader
-   */
+  /** Stop reader */
   public void stop() {
     stopReaderNative(nativeReaderPtr);
   }
 
-  /**
-   * Close reader to release resource
-   */
+  /** Close reader to release resource */
   public void close() {
     if (nativeReaderPtr == 0) {
       return;
@@ -213,10 +213,7 @@ public class DataReader {
   }
 
   private native void getBundleNative(
-      long nativeReaderPtr,
-      long timeoutMillis,
-      long params,
-      long metaAddress);
+      long nativeReaderPtr, long timeoutMillis, long params, long metaAddress);
 
   private native byte[] getOffsetsInfoNative(long nativeQueueConsumerPtr);
 
@@ -378,5 +375,4 @@ public class DataReader {
       return barrierOffsetInfo;
     }
   }
-
 }
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/DataWriter.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/DataWriter.java
index 55729c7fb..039b22b77 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/DataWriter.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/DataWriter.java
@@ -17,9 +17,7 @@ import java.util.Set;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-/**
- * DataWriter is a wrapper of streaming c++ DataWriter, which sends data to downstream workers
- */
+/** DataWriter is a wrapper of streaming c++ DataWriter, which sends data to downstream workers */
 public class DataWriter {
 
   private static final Logger LOG = LoggerFactory.getLogger(DataWriter.class);
@@ -51,8 +49,8 @@ public class DataWriter {
     ChannelCreationParametersBuilder initialParameters =
         new ChannelCreationParametersBuilder().buildOutputQueueParameters(outputChannels, toActors);
 
-    byte[][] outputChannelsBytes = outputChannels.stream()
-        .map(ChannelId::idStrToBytes).toArray(byte[][]::new);
+    byte[][] outputChannelsBytes =
+        outputChannels.stream().map(ChannelId::idStrToBytes).toArray(byte[][]::new);
     long channelSize = workerConfig.transferConfig.channelSize();
 
     // load message id from checkpoints
@@ -70,15 +68,16 @@ public class DataWriter {
     if (TransferChannelType.MEMORY_CHANNEL == channelType) {
       isMock = true;
     }
-    this.nativeWriterPtr = createWriterNative(
-        initialParameters,
-        outputChannelsBytes,
-        msgIds,
-        channelSize,
-        ChannelUtils.toNativeConf(workerConfig),
-        isMock
-    );
-    LOG.info("Create DataWriter succeed for worker: {}.",
+    this.nativeWriterPtr =
+        createWriterNative(
+            initialParameters,
+            outputChannelsBytes,
+            msgIds,
+            channelSize,
+            ChannelUtils.toNativeConf(workerConfig),
+            isMock);
+    LOG.info(
+        "Create DataWriter succeed for worker: {}.",
         workerConfig.workerInternalConfig.workerName());
   }
 
@@ -108,8 +107,8 @@ public class DataWriter {
    * Write msg into the specified channels
    *
    * @param ids channel ids
-   * @param item message item data section is specified by [position, limit).
-   *     item doesn't have to be a direct buffer.
+   * @param item message item data section is specified by [position, limit). item doesn't have to
+   *     be a direct buffer.
    */
   public void write(Set ids, ByteBuffer item) {
     int size = item.remaining();
@@ -150,16 +149,12 @@ public class DataWriter {
     clearCheckpointNative(nativeWriterPtr, checkpointId);
   }
 
-  /**
-   * stop writer
-   */
+  /** stop writer */
   public void stop() {
     stopWriterNative(nativeWriterPtr);
   }
 
-  /**
-   * close writer to release resources
-   */
+  /** close writer to release resources */
   public void close() {
     if (nativeWriterPtr == 0) {
       return;
@@ -180,12 +175,7 @@ public class DataWriter {
   private native long[] getOutputMsgIdNative(long nativeQueueProducerPtr);
 
   private native void broadcastBarrierNative(
-      long nativeQueueProducerPtr, long checkpointId,
-      byte[] data);
-
-  private native void clearCheckpointNative(
-      long nativeQueueProducerPtr,
-      long checkpointId
-  );
+      long nativeQueueProducerPtr, long checkpointId, byte[] data);
 
+  private native void clearCheckpointNative(long nativeQueueProducerPtr, long checkpointId);
 }
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/TransferHandler.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/TransferHandler.java
index aa7541468..816273d83 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/TransferHandler.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/TransferHandler.java
@@ -42,7 +42,6 @@ public class TransferHandler {
 
   private native long createReaderClientNative();
 
-
   private native void handleWriterMessageNative(long handler, byte[] buffer);
 
   private native byte[] handleWriterMessageSyncNative(long handler, byte[] buffer);
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/channel/ChannelId.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/channel/ChannelId.java
index dbd8fef73..3c4f8caf0 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/channel/ChannelId.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/channel/ChannelId.java
@@ -12,8 +12,7 @@ import java.util.Set;
 import sun.nio.ch.DirectBuffer;
 
 /**
- * ChannelID is used to identify a transfer channel between a upstream worker and downstream
- * worker.
+ * ChannelID is used to identify a transfer channel between a upstream worker and downstream worker.
  */
 public class ChannelId {
 
@@ -45,16 +44,12 @@ public class ChannelId {
 
   private static native void destroyNativeId(long nativeIdPtr);
 
-  /**
-   * @param id hex string representation of channel id
-   */
+  /** @param id hex string representation of channel id */
   public static ChannelId from(String id) {
     return from(id, ChannelId.idStrToBytes(id));
   }
 
-  /**
-   * @param idBytes bytes representation of channel id
-   */
+  /** @param idBytes bytes representation of channel id */
   public static ChannelId from(byte[] idBytes) {
     return from(idBytesToStr(idBytes), idBytes);
   }
@@ -76,9 +71,7 @@ public class ChannelId {
     return id;
   }
 
-  /**
-   * @return a random channel id string
-   */
+  /** Returns a random channel id string */
   public static String genRandomIdStr() {
     StringBuilder sb = new StringBuilder();
     Random random = new Random();
@@ -92,18 +85,20 @@ public class ChannelId {
    * Generate channel name, which will be 20 character
    *
    * @param fromTaskId upstream task id
-   * @param toTaskId downstream task id
-   * @return channel name
+   * @param toTaskId downstream task id Returns channel name
    */
   public static String genIdStr(int fromTaskId, int toTaskId, long ts) {
     /*
       |    Head    | Timestamp | Empty | From  |  To    |
       | 8 bytes    |  4bytes   | 4bytes| 2bytes| 2bytes |
     */
-    Preconditions.checkArgument(fromTaskId < Short.MAX_VALUE,
-        "fromTaskId %s is larger than %s", fromTaskId, Short.MAX_VALUE);
-    Preconditions.checkArgument(toTaskId < Short.MAX_VALUE,
-        "toTaskId %s is larger than %s", fromTaskId, Short.MAX_VALUE);
+    Preconditions.checkArgument(
+        fromTaskId < Short.MAX_VALUE,
+        "fromTaskId %s is larger than %s",
+        fromTaskId,
+        Short.MAX_VALUE);
+    Preconditions.checkArgument(
+        toTaskId < Short.MAX_VALUE, "toTaskId %s is larger than %s", fromTaskId, Short.MAX_VALUE);
     byte[] channelName = new byte[20];
 
     for (int i = 11; i >= 8; i--) {
@@ -120,8 +115,7 @@ public class ChannelId {
   }
 
   /**
-   * @param id hex string representation of channel id
-   * @return bytes representation of channel id
+   * @param id hex string representation of channel id Returns bytes representation of channel id
    */
   public static byte[] idStrToBytes(String id) {
     byte[] idBytes = BaseEncoding.base16().decode(id.toUpperCase());
@@ -130,8 +124,7 @@ public class ChannelId {
   }
 
   /**
-   * @param id bytes representation of channel id
-   * @return hex string representation of channel id
+   * @param id bytes representation of channel id Returns hex string representation of channel id
    */
   public static String idBytesToStr(byte[] id) {
     assert id.length == ChannelId.ID_LENGTH;
@@ -178,6 +171,4 @@ public class ChannelId {
   public int hashCode() {
     return strId.hashCode();
   }
-
 }
-
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/channel/ChannelRecoverInfo.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/channel/ChannelRecoverInfo.java
index 584f411ee..fd785cdbb 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/channel/ChannelRecoverInfo.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/channel/ChannelRecoverInfo.java
@@ -8,32 +8,29 @@ import java.util.Set;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-
 public class ChannelRecoverInfo implements Serializable {
 
   private static final Logger LOG = LoggerFactory.getLogger(ChannelRecoverInfo.class);
   public Map queueCreationStatusMap;
 
-
   public ChannelRecoverInfo(Map queueCreationStatusMap) {
     this.queueCreationStatusMap = queueCreationStatusMap;
   }
 
   public Set getDataLostQueues() {
     Set dataLostQueues = new HashSet<>();
-    queueCreationStatusMap.forEach((q, status) -> {
-      if (status.equals(ChannelCreationStatus.DataLost)) {
-        dataLostQueues.add(q);
-      }
-    });
+    queueCreationStatusMap.forEach(
+        (q, status) -> {
+          if (status.equals(ChannelCreationStatus.DataLost)) {
+            dataLostQueues.add(q);
+          }
+        });
     return dataLostQueues;
   }
 
   @Override
   public String toString() {
-    return MoreObjects.toStringHelper(this)
-        .add("dataLostQueues", getDataLostQueues())
-        .toString();
+    return MoreObjects.toStringHelper(this).add("dataLostQueues", getDataLostQueues()).toString();
   }
 
   public enum ChannelCreationStatus {
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/channel/ChannelUtils.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/channel/ChannelUtils.java
index 74e813134..d29543528 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/channel/ChannelUtils.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/channel/ChannelUtils.java
@@ -43,7 +43,7 @@ public class ChannelUtils {
       builder.setEmptyMessageInterval(emptyMsgInterval);
     }
 
-    //flow control type
+    // flow control type
     int flowControlType = workerConfig.transferConfig.flowControlType();
     if (flowControlType != -1) {
       builder.setFlowControlType(Streaming.FlowControlType.forNumber(flowControlType));
@@ -55,7 +55,7 @@ public class ChannelUtils {
       builder.setWriterConsumedStep(writerConsumedStep);
     }
 
-    //reader consumed step
+    // reader consumed step
     int readerConsumedStep = workerConfig.transferConfig.readerConsumedStep();
     if (readerConsumedStep != -1) {
       builder.setReaderConsumedStep(readerConsumedStep);
@@ -65,5 +65,4 @@ public class ChannelUtils {
     LOGGER.info("Streaming native conf {}", streamingConf.toString());
     return streamingConf.toByteArray();
   }
-
 }
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/channel/OffsetInfo.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/channel/OffsetInfo.java
index 5c3ea02a7..3cdfbf819 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/channel/OffsetInfo.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/channel/OffsetInfo.java
@@ -3,9 +3,7 @@ package io.ray.streaming.runtime.transfer.channel;
 import com.google.common.base.MoreObjects;
 import java.io.Serializable;
 
-/**
- * This data structure contains offset used by streaming queue.
- */
+/** This data structure contains offset used by streaming queue. */
 public class OffsetInfo implements Serializable {
 
   private long streamingMsgId;
@@ -24,8 +22,6 @@ public class OffsetInfo implements Serializable {
 
   @Override
   public String toString() {
-    return MoreObjects.toStringHelper(this)
-        .add("streamingMsgId", streamingMsgId)
-        .toString();
+    return MoreObjects.toStringHelper(this).add("streamingMsgId", streamingMsgId).toString();
   }
 }
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/message/BarrierMessage.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/message/BarrierMessage.java
index ffc694c53..7ea8d60f6 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/message/BarrierMessage.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/message/BarrierMessage.java
@@ -4,7 +4,6 @@ import io.ray.streaming.runtime.transfer.channel.OffsetInfo;
 import java.nio.ByteBuffer;
 import java.util.Map;
 
-
 public class BarrierMessage extends ChannelMessage {
 
   private final ByteBuffer data;
@@ -12,8 +11,12 @@ public class BarrierMessage extends ChannelMessage {
   private final Map inputOffsets;
 
   public BarrierMessage(
-      long msgId, long timestamp, String channelId,
-      ByteBuffer data, long checkpointId, Map inputOffsets) {
+      long msgId,
+      long timestamp,
+      String channelId,
+      ByteBuffer data,
+      long checkpointId,
+      Map inputOffsets) {
     super(msgId, timestamp, channelId);
     this.data = data;
     this.checkpointId = checkpointId;
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/message/DataMessage.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/message/DataMessage.java
index b3cf779bf..de873ae5b 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/message/DataMessage.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/transfer/message/DataMessage.java
@@ -2,10 +2,7 @@ package io.ray.streaming.runtime.transfer.message;
 
 import java.nio.ByteBuffer;
 
-
-/**
- * DataMessage represents data between upstream and downstream operators.
- */
+/** DataMessage represents data between upstream and downstream operators. */
 public class DataMessage extends ChannelMessage {
 
   private final ByteBuffer body;
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/CheckpointStateUtil.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/CheckpointStateUtil.java
index c32d2ef4f..b1da3f9c3 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/CheckpointStateUtil.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/CheckpointStateUtil.java
@@ -4,9 +4,7 @@ import io.ray.streaming.runtime.context.ContextBackend;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-/**
- * Handle exception for checkpoint state
- */
+/** Handle exception for checkpoint state */
 public class CheckpointStateUtil {
 
   private static final Logger LOG = LoggerFactory.getLogger(CheckpointStateUtil.class);
@@ -45,8 +43,7 @@ public class CheckpointStateUtil {
 
   public static class CheckpointStateRuntimeException extends RuntimeException {
 
-    public CheckpointStateRuntimeException() {
-    }
+    public CheckpointStateRuntimeException() {}
 
     public CheckpointStateRuntimeException(String message) {
       super(message);
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/CommonUtils.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/CommonUtils.java
index 37940df5d..3ce8c1fe3 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/CommonUtils.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/CommonUtils.java
@@ -2,9 +2,7 @@ package io.ray.streaming.runtime.util;
 
 import java.util.Map;
 
-/**
- * Common tools.
- */
+/** Common tools. */
 public class CommonUtils {
 
   public static Map strMapToObjectMap(Map srcMap) {
diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/EnvUtil.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/EnvUtil.java
index 6e14d45af..07fda18a6 100644
--- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/EnvUtil.java
+++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/EnvUtil.java
@@ -36,13 +36,14 @@ public class EnvUtil {
   /**
    * Execute an external command.
    *
-   * @return Whether the command succeeded.
+   * 

Returns Whether the command succeeded. */ public static boolean executeCommand(List command, int waitTimeoutSeconds) { try { - ProcessBuilder processBuilder = new ProcessBuilder(command) - .redirectOutput(ProcessBuilder.Redirect.INHERIT) - .redirectError(ProcessBuilder.Redirect.INHERIT); + ProcessBuilder processBuilder = + new ProcessBuilder(command) + .redirectOutput(ProcessBuilder.Redirect.INHERIT) + .redirectError(ProcessBuilder.Redirect.INHERIT); Process process = processBuilder.start(); boolean exit = process.waitFor(waitTimeoutSeconds, TimeUnit.SECONDS); if (!exit) { @@ -53,5 +54,4 @@ public class EnvUtil { throw new RuntimeException("Error executing command " + String.join(" ", command), e); } } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/Platform.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/Platform.java index 739b0424d..effafcc54 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/Platform.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/Platform.java @@ -9,9 +9,7 @@ import java.nio.ByteBuffer; import sun.misc.Unsafe; import sun.nio.ch.DirectBuffer; -/** - * Based on org.apache.spark.unsafe.Platform - */ +/** Based on org.apache.spark.unsafe.Platform */ public final class Platform { public static final Unsafe UNSAFE; @@ -51,18 +49,19 @@ public final class Platform { } private static final ThreadLocal localEmptyBuffer = - ThreadLocal.withInitial(() -> { - try { - return (ByteBuffer) DBB_CONSTRUCTOR.newInstance(0, 0); - } catch (InstantiationException | IllegalAccessException | InvocationTargetException e) { - UNSAFE.throwException(e); - } - throw new IllegalStateException("unreachable"); - }); + ThreadLocal.withInitial( + () -> { + try { + return (ByteBuffer) DBB_CONSTRUCTOR.newInstance(0, 0); + } catch (InstantiationException + | IllegalAccessException + | InvocationTargetException e) { + UNSAFE.throwException(e); + } + throw new IllegalStateException("unreachable"); + }); - /** - * Wrap a buffer [address, address + size) as a DirectByteBuffer. - */ + /** Wrap a buffer [address, address + size) as a DirectByteBuffer. */ public static ByteBuffer wrapDirectBuffer(long address, int size) { ByteBuffer buffer = localEmptyBuffer.get().duplicate(); UNSAFE.putLong(buffer, BUFFER_ADDRESS_FIELD_OFFSET, address); @@ -71,21 +70,15 @@ public final class Platform { return buffer; } - /** - * Wrap a buffer [address, address + size) into provided buffer. - */ + /** Wrap a buffer [address, address + size) into provided buffer. */ public static void wrapDirectBuffer(ByteBuffer buffer, long address, int size) { UNSAFE.putLong(buffer, BUFFER_ADDRESS_FIELD_OFFSET, address); UNSAFE.putInt(buffer, BUFFER_CAPACITY_FIELD_OFFSET, size); buffer.clear(); } - /** - * @param buffer a DirectBuffer backed by off-heap memory - * @return address of off-heap memory - */ + /** @param buffer a DirectBuffer backed by off-heap memory Returns address of off-heap memory */ public static long getAddress(ByteBuffer buffer) { return ((DirectBuffer) buffer).address(); } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/RayUtils.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/RayUtils.java index e0bd950fa..a97a2f5ba 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/RayUtils.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/RayUtils.java @@ -9,15 +9,13 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -/** - * RayUtils is the utility class to access ray runtime api. - */ +/** RayUtils is the utility class to access ray runtime api. */ public class RayUtils { /** * Get all node info from GCS * - * @return node info list + *

Returns node info list */ public static List getAllNodeInfo() { if (Ray.getRuntimeContext().isSingleProcess()) { @@ -30,7 +28,7 @@ public class RayUtils { /** * Get all alive node info map * - * @return node info map, key is unique node id , value is node info + *

Returns node info map, key is unique node id , value is node info */ public static Map getAliveNodeInfoMap() { return getAllNodeInfo().stream() @@ -50,13 +48,18 @@ public class RayUtils { for (int byteIndex = 0; byteIndex < UniqueId.LENGTH; ++byteIndex) { nodeIdBytes[byteIndex] = String.valueOf(i).getBytes()[0]; } - NodeInfo nodeInfo = new NodeInfo(new UniqueId(nodeIdBytes), - "localhost" + i, "localhost" + i, -1, - "", "", - true, resources); + NodeInfo nodeInfo = + new NodeInfo( + new UniqueId(nodeIdBytes), + "localhost" + i, + "localhost" + i, + -1, + "", + "", + true, + resources); nodeInfos.add(nodeInfo); } return nodeInfos; } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/ReflectionUtils.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/ReflectionUtils.java index 5852220af..bc04a1ded 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/ReflectionUtils.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/ReflectionUtils.java @@ -20,7 +20,7 @@ public class ReflectionUtils { /** * For covariant return type, return the most specific method. * - * @return all methods named by {@code methodName}, + *

Returns all methods named by {@code methodName}, */ public static List findMethods(Class cls, String methodName) { List> classes = new ArrayList<>(); @@ -55,10 +55,10 @@ public class ReflectionUtils { } /** - *

Gets a List of all interfaces implemented by the given - * class and its superclasses.

- *

The order is determined by looking through each interface in turn as - * declared in the source file and following its hierarchy up.

+ * Gets a List of all interfaces implemented by the given class and its superclasses. + * + *

The order is determined by looking through each interface in turn as declared in the source + * file and following its hierarchy up. */ public static List> getAllInterfaces(Class cls) { if (cls == null) { @@ -83,5 +83,4 @@ public class ReflectionUtils { cls = cls.getSuperclass(); } } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/ResourceUtil.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/ResourceUtil.java index 9b8ca22ae..b8336cd14 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/ResourceUtil.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/ResourceUtil.java @@ -16,35 +16,35 @@ import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - -/** - * Resource Utility collects current OS and JVM resource usage information - */ +/** Resource Utility collects current OS and JVM resource usage information */ public class ResourceUtil { public static final Logger LOG = LoggerFactory.getLogger(ResourceUtil.class); /** - * Refer to: https://docs.oracle.com/javase/8/docs/jre/api/management/extension/com/sun/management/OperatingSystemMXBean.html + * Refer to: + * https://docs.oracle.com/javase/8/docs/jre/api/management/extension/com/sun/management/OperatingSystemMXBean.html */ private static OperatingSystemMXBean osmxb = (OperatingSystemMXBean) ManagementFactory.getOperatingSystemMXBean(); - /** - * Log current jvm process's memory detail - */ + /** Log current jvm process's memory detail */ public static void logProcessMemoryDetail() { int mb = 1024 * 1024; - //Getting the runtime reference from system + // Getting the runtime reference from system Runtime runtime = Runtime.getRuntime(); StringBuilder sb = new StringBuilder(32); - sb.append("used memory: ").append((runtime.totalMemory() - runtime.freeMemory()) / mb) - .append(", free memory: ").append(runtime.freeMemory() / mb) - .append(", total memory: ").append(runtime.totalMemory() / mb) - .append(", max memory: ").append(runtime.maxMemory() / mb); + sb.append("used memory: ") + .append((runtime.totalMemory() - runtime.freeMemory()) / mb) + .append(", free memory: ") + .append(runtime.freeMemory() / mb) + .append(", total memory: ") + .append(runtime.totalMemory() / mb) + .append(", max memory: ") + .append(runtime.maxMemory() / mb); if (LOG.isInfoEnabled()) { LOG.info(sb.toString()); @@ -52,8 +52,8 @@ public class ResourceUtil { } /** - * @return jvm heap usage ratio. note that one of the survivor space is not include in total - * memory while calculating this ratio. + * Returns jvm heap usage ratio. note that one of the survivor space is not include in total + * memory while calculating this ratio. */ public static double getJvmHeapUsageRatio() { Runtime runtime = Runtime.getRuntime(); @@ -61,33 +61,27 @@ public class ResourceUtil { } /** - * @return jvm heap usage(in bytes). - * note that this value doesn't include one of the survivor space. + * Returns jvm heap usage(in bytes). note that this value doesn't include one of the survivor + * space. */ public static long getJvmHeapUsageInBytes() { Runtime runtime = Runtime.getRuntime(); return runtime.totalMemory() - runtime.freeMemory(); } - /** - * @return the total amount of physical memory in bytes. - */ + /** Returns the total amount of physical memory in bytes. */ public static long getSystemTotalMemory() { return osmxb.getTotalPhysicalMemorySize(); } - /** - * @return the used system physical memory in bytes - */ + /** Returns the used system physical memory in bytes */ public static long getSystemMemoryUsage() { long totalMemory = osmxb.getTotalPhysicalMemorySize(); long freeMemory = osmxb.getFreePhysicalMemorySize(); return totalMemory - freeMemory; } - /** - * @return the ratio of used system physical memory. This value is a double in the [0.0,1.0] - */ + /** Returns the ratio of used system physical memory. This value is a double in the [0.0,1.0] */ public static double getSystemMemoryUsageRatio() { double totalMemory = osmxb.getTotalPhysicalMemorySize(); double freeMemory = osmxb.getFreePhysicalMemorySize(); @@ -95,18 +89,14 @@ public class ResourceUtil { return 1 - ratio; } - /** - * @return the cpu load for current jvm process. This value is a double in the [0.0,1.0] - */ + /** Returns the cpu load for current jvm process. This value is a double in the [0.0,1.0] */ public static double getProcessCpuUsage() { return osmxb.getProcessCpuLoad(); } /** - * @return the system cpu usage. - * This value is a double in the [0.0,1.0] - * We will try to use `vsar` to get cpu usage by default, - * and use MXBean if any exception raised. + * Returns the system cpu usage. This value is a double in the [0.0,1.0] We will try to use `vsar` + * to get cpu usage by default, and use MXBean if any exception raised. */ public static double getSystemCpuUsage() { double cpuUsage = 0.0; @@ -128,9 +118,7 @@ public class ResourceUtil { return osmxb.getSystemCpuLoad(); } - /** - * Get system cpu util by vsar - */ + /** Get system cpu util by vsar */ public static double getSystemCpuUtilByVsar() throws Exception { double cpuUsageFromVsar = 0.0; String[] vsarCpuCommand = {"/bin/sh", "-c", "vsar --check --cpu -s util"}; @@ -156,16 +144,12 @@ public class ResourceUtil { return cpuUsageFromVsar; } - /** - * @returns the system load average for the last minute - */ + /** Returnss the system load average for the last minute */ public static double getSystemLoadAverage() { return osmxb.getSystemLoadAverage(); } - /** - * @return system cpu cores num - */ + /** Returns system cpu cores num */ public static int getCpuCores() { return osmxb.getAvailableProcessors(); } @@ -174,44 +158,40 @@ public class ResourceUtil { * Get containers by hostname of address * * @param containers container list - * @param containerHosts container hostname or address set - * @return matched containers + * @param containerHosts container hostname or address set Returns matched containers */ public static List getContainersByHostname( - List containers, - Collection containerHosts) { + List containers, Collection containerHosts) { return containers.stream() - .filter(container -> - containerHosts.contains(container.getHostname()) || - containerHosts.contains(container.getAddress())) + .filter( + container -> + containerHosts.contains(container.getHostname()) + || containerHosts.contains(container.getAddress())) .collect(Collectors.toList()); } /** * Get container by hostname * - * @param hostName container hostname - * @return container + * @param hostName container hostname Returns container */ public static Optional getContainerByHostname( - List containers, - String hostName) { + List containers, String hostName) { return containers.stream() - .filter(container -> container.getHostname().equals(hostName) || - container.getAddress().equals(hostName)) + .filter( + container -> + container.getHostname().equals(hostName) || container.getAddress().equals(hostName)) .findFirst(); } /** * Get container by id * - * @param containerID container id - * @return container + * @param containerID container id Returns container */ public static Optional getContainerById( - List containers, - ContainerId containerID) { + List containers, ContainerId containerID) { return containers.stream() .filter(container -> container.getId().equals(containerID)) .findFirst(); diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/Serializer.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/Serializer.java index 420215df1..df435e5a9 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/Serializer.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/util/Serializer.java @@ -11,5 +11,4 @@ public class Serializer { public static T decode(byte[] bytes) { return FstSerializer.decode(bytes); } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/JobWorker.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/JobWorker.java index 7aac6b0c6..5a6554802 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/JobWorker.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/JobWorker.java @@ -31,8 +31,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * The streaming worker implementation class, it is ray actor. JobWorker is created by - * {@link JobMaster} through ray api, and JobMaster communicates with JobWorker through Ray.call(). + * The streaming worker implementation class, it is ray actor. JobWorker is created by {@link + * JobMaster} through ray api, and JobMaster communicates with JobWorker through Ray.call(). * *

The JobWorker is responsible for creating tasks and defines the methods of communication * between workers. @@ -49,27 +49,23 @@ public class JobWorker implements Serializable { } public final Object initialStateChangeLock = new Object(); - /** - * isRecreate=true means this worker is initialized more than once after actor created. - */ + /** isRecreate=true means this worker is initialized more than once after actor created. */ public AtomicBoolean isRecreate = new AtomicBoolean(false); + public ContextBackend contextBackend; private JobWorkerContext workerContext; private ExecutionVertex executionVertex; private StreamingWorkerConfig workerConfig; - /** - * The while-loop thread to read message, process message, and write results - */ + /** The while-loop thread to read message, process message, and write results */ private StreamTask task; - /** - * transferHandler handles messages by ray direct call - */ + /** transferHandler handles messages by ray direct call */ private TransferHandler transferHandler; /** - * A flag to avoid duplicated rollback. Becomes true after requesting - * rollback, set to false when finish rollback. + * A flag to avoid duplicated rollback. Becomes true after requesting rollback, set to false when + * finish rollback. */ private boolean isNeedRollback = false; + private int rollbackCount = 0; public JobWorker(ExecutionVertex executionVertex) { @@ -80,7 +76,8 @@ public class JobWorker implements Serializable { this.workerConfig = new StreamingWorkerConfig(executionVertex.getWorkerConfig()); this.contextBackend = ContextBackendFactory.getContextBackend(this.workerConfig); - LOG.info("Ray.getRuntimeContext().wasCurrentActorRestarted()={}", + LOG.info( + "Ray.getRuntimeContext().wasCurrentActorRestarted()={}", Ray.getRuntimeContext().wasCurrentActorRestarted()); if (!Ray.getRuntimeContext().wasCurrentActorRestarted()) { saveContext(); @@ -93,14 +90,14 @@ public class JobWorker implements Serializable { byte[] bytes = CheckpointStateUtil.get(contextBackend, getJobWorkerContextKey()); if (bytes != null) { JobWorkerContext context = Serializer.decode(bytes); - LOG.info("Worker recover from checkpoint state, byte len={}, context={}.", bytes.length, - context); + LOG.info( + "Worker recover from checkpoint state, byte len={}, context={}.", bytes.length, context); init(context); requestRollback("LoadCheckpoint request rollback in new actor."); } else { LOG.error( - "Worker is reconstructed, but can't load checkpoint. " + - "Check whether you checkpoint state is reliable. Current checkpoint state is {}.", + "Worker is reconstructed, but can't load checkpoint. " + + "Check whether you checkpoint state is reliable. Current checkpoint state is {}.", contextBackend.getClass().getName()); } } @@ -108,19 +105,23 @@ public class JobWorker implements Serializable { public synchronized void saveContext() { byte[] contextBytes = Serializer.encode(workerContext); String key = getJobWorkerContextKey(); - LOG.info("Saving context, worker context={}, serialized byte length={}, key={}.", workerContext, - contextBytes.length, key); + LOG.info( + "Saving context, worker context={}, serialized byte length={}, key={}.", + workerContext, + contextBytes.length, + key); CheckpointStateUtil.put(contextBackend, key, contextBytes); } - /** - * Initialize JobWorker and data communication pipeline. - */ + /** Initialize JobWorker and data communication pipeline. */ public Boolean init(JobWorkerContext workerContext) { // IMPORTANT: some test cases depends on this log to find workers' pid, // be careful when changing this log. - LOG.info("Initiating job worker: {}. Worker context is: {}, pid={}.", - workerContext.getWorkerName(), workerContext, EnvUtil.getJvmPid()); + LOG.info( + "Initiating job worker: {}. Worker context is: {}, pid={}.", + workerContext.getWorkerName(), + workerContext, + EnvUtil.getJvmPid()); this.workerContext = workerContext; this.executionVertex = workerContext.getExecutionVertex(); @@ -136,20 +137,25 @@ public class JobWorker implements Serializable { /** * Start worker's stream tasks with specific checkpoint ID. * - * @return a {@link CallResult} with {@link ChannelRecoverInfo}, - * contains {@link ChannelCreationStatus} of each input queue. + *

Returns a {@link CallResult} with {@link ChannelRecoverInfo}, contains {@link + * ChannelCreationStatus} of each input queue. */ public CallResult rollback(Long checkpointId, Long startRollbackTs) { synchronized (initialStateChangeLock) { - if (task != null && task.isAlive() && checkpointId == task.lastCheckpointId && - task.isInitialState) { + if (task != null + && task.isAlive() + && checkpointId == task.lastCheckpointId + && task.isInitialState) { return CallResult.skipped("Task is already in initial state, skip this rollback."); } } long remoteCallCost = System.currentTimeMillis() - startRollbackTs; - LOG.info("Start rollback[{}], checkpoint is {}, remote call cost {}ms.", - executionVertex.getExecutionJobVertexName(), checkpointId, remoteCallCost); + LOG.info( + "Start rollback[{}], checkpoint is {}, remote call cost {}ms.", + executionVertex.getExecutionJobVertexName(), + checkpointId, + remoteCallCost); rollbackCount++; if (rollbackCount > 1) { @@ -157,7 +163,7 @@ public class JobWorker implements Serializable { } try { - //Init transfer + // Init transfer TransferChannelType channelType = workerConfig.transferConfig.channelType(); if (TransferChannelType.NATIVE_CHANNEL == channelType) { transferHandler = new TransferHandler(); @@ -174,8 +180,10 @@ public class JobWorker implements Serializable { ChannelRecoverInfo channelRecoverInfo = task.recover(isRecreate.get()); isNeedRollback = false; - LOG.info("Rollback job worker success, checkpoint is {}, channelRecoverInfo is {}.", - checkpointId, channelRecoverInfo); + LOG.info( + "Rollback job worker success, checkpoint is {}, channelRecoverInfo is {}.", + checkpointId, + channelRecoverInfo); return CallResult.success(channelRecoverInfo); } catch (Exception e) { @@ -184,13 +192,11 @@ public class JobWorker implements Serializable { } } - /** - * Create tasks based on the processor corresponding of the operator. - */ + /** Create tasks based on the processor corresponding of the operator. */ private StreamTask createStreamTask(long checkpointId) { StreamTask task; - StreamProcessor streamProcessor = ProcessBuilder - .buildProcessor(executionVertex.getStreamOperator()); + StreamProcessor streamProcessor = + ProcessBuilder.buildProcessor(executionVertex.getStreamOperator()); LOG.debug("Stream processor created: {}.", streamProcessor); if (streamProcessor instanceof SourceProcessor) { @@ -208,9 +214,7 @@ public class JobWorker implements Serializable { // Checkpoint // ---------------------------------------------------------------------- - /** - * Trigger source job worker checkpoint - */ + /** Trigger source job worker checkpoint */ public Boolean triggerCheckpoint(Long barrierId) { LOG.info("Receive trigger, barrierId is {}.", barrierId); if (task != null) { @@ -228,9 +232,11 @@ public class JobWorker implements Serializable { } public Boolean clearExpiredCheckpoint(Long expiredStateCpId, Long expiredQueueCpId) { - LOG.info("Clear expired checkpoint state, checkpoint id is {}; " + - "Clear expired queue msg, checkpoint id is {}", - expiredStateCpId, expiredQueueCpId); + LOG.info( + "Clear expired checkpoint state, checkpoint id is {}; " + + "Clear expired queue msg, checkpoint id is {}", + expiredStateCpId, + expiredQueueCpId); if (task != null) { if (expiredStateCpId > 0) { task.clearExpiredCpState(expiredStateCpId); @@ -247,13 +253,14 @@ public class JobWorker implements Serializable { LOG.info("Request rollback."); isNeedRollback = true; isRecreate.set(true); - boolean requestRet = RemoteCallMaster.requestJobWorkerRollback( - workerContext.getMaster(), new WorkerRollbackRequest( - workerContext.getWorkerActorId(), - exceptionMsg, - EnvUtil.getHostName(), - EnvUtil.getJvmPid() - )); + boolean requestRet = + RemoteCallMaster.requestJobWorkerRollback( + workerContext.getMaster(), + new WorkerRollbackRequest( + workerContext.getWorkerActorId(), + exceptionMsg, + EnvUtil.getHostName(), + EnvUtil.getJvmPid())); if (!requestRet) { LOG.warn("Job worker request rollback failed! exceptionMsg={}.", exceptionMsg); } @@ -262,8 +269,10 @@ public class JobWorker implements Serializable { public Boolean checkIfNeedRollback(Long startCallTs) { // No save checkpoint in this query. long remoteCallCost = System.currentTimeMillis() - startCallTs; - LOG.info("Finished checking if need to rollback with result: {}, rpc delay={}ms.", - isNeedRollback, remoteCallCost); + LOG.info( + "Finished checking if need to rollback with result: {}, rpc delay={}ms.", + isNeedRollback, + remoteCallCost); return isNeedRollback; } @@ -286,12 +295,11 @@ public class JobWorker implements Serializable { private String getJobWorkerContextKey() { return workerConfig.checkpointConfig.jobWorkerContextCpPrefixKey() + workerConfig.commonConfig.jobName() - + "_" + executionVertex.getExecutionVertexId(); + + "_" + + executionVertex.getExecutionVertexId(); } - /** - * Used by upstream streaming queue to send data to this actor - */ + /** Used by upstream streaming queue to send data to this actor */ public void onReaderMessage(byte[] buffer) { if (transferHandler != null) { transferHandler.onReaderMessage(buffer); @@ -308,9 +316,7 @@ public class JobWorker implements Serializable { return transferHandler.onReaderMessageSync(buffer); } - /** - * Used by downstream streaming queue to send data to this actor - */ + /** Used by downstream streaming queue to send data to this actor */ public void onWriterMessage(byte[] buffer) { if (transferHandler != null) { transferHandler.onWriterMessage(buffer); @@ -327,5 +333,4 @@ public class JobWorker implements Serializable { } return transferHandler.onWriterMessageSync(buffer); } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/context/JobWorkerContext.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/context/JobWorkerContext.java index e4fd3b992..d92e95bb2 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/context/JobWorkerContext.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/context/JobWorkerContext.java @@ -13,24 +13,16 @@ import io.ray.streaming.runtime.python.GraphPbBuilder; import java.io.Serializable; import java.util.Map; -/** - * Job worker context of java type. - */ +/** Job worker context of java type. */ public class JobWorkerContext implements Serializable { - /** - * JobMaster actor. - */ + /** JobMaster actor. */ private ActorHandle master; - /** - * Worker's vertex info. - */ + /** Worker's vertex info. */ private ExecutionVertex executionVertex; - public JobWorkerContext( - ActorHandle master, - ExecutionVertex executionVertex) { + public JobWorkerContext(ActorHandle master, ExecutionVertex executionVertex) { this.master = master; this.executionVertex = executionVertex; } @@ -81,14 +73,13 @@ public class JobWorkerContext implements Serializable { RemoteCall.ExecutionVertexContext executionVertexContext = new GraphPbBuilder().buildExecutionVertexContext(executionVertex); - byte[] contextBytes = RemoteCall.PythonJobWorkerContext.newBuilder() - .setMasterActor( - ByteString.copyFrom((((NativeActorHandle) (master)).toBytes()))) - .setExecutionVertexContext(executionVertexContext) - .build() - .toByteArray(); + byte[] contextBytes = + RemoteCall.PythonJobWorkerContext.newBuilder() + .setMasterActor(ByteString.copyFrom((((NativeActorHandle) (master)).toBytes()))) + .setExecutionVertexContext(executionVertexContext) + .build() + .toByteArray(); return contextBytes; } - } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/context/StreamingRuntimeContext.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/context/StreamingRuntimeContext.java index 6fe22a4ae..73fe4df03 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/context/StreamingRuntimeContext.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/context/StreamingRuntimeContext.java @@ -15,19 +15,14 @@ import io.ray.streaming.state.keystate.state.MapState; import io.ray.streaming.state.keystate.state.ValueState; import java.util.Map; -/** - * Use Ray to implement RuntimeContext. - */ +/** Use Ray to implement RuntimeContext. */ public class StreamingRuntimeContext implements RuntimeContext { - /** - * Backend for keyed state. This might be empty if we're not on a keyed stream. - */ + /** Backend for keyed state. This might be empty if we're not on a keyed stream. */ protected transient KeyStateBackend keyStateBackend; - /** - * Backend for operator state. This might be empty - */ + /** Backend for operator state. This might be empty */ protected transient OperatorStateBackend operatorStateBackend; + private int taskId; private int taskIndex; private int parallelism; @@ -35,8 +30,7 @@ public class StreamingRuntimeContext implements RuntimeContext { private Map config; public StreamingRuntimeContext( - ExecutionVertex executionVertex, Map config, - int parallelism) { + ExecutionVertex executionVertex, Map config, int parallelism) { this.taskId = executionVertex.getExecutionVertexId(); this.config = config; this.taskIndex = executionVertex.getExecutionVertexIndex(); @@ -118,8 +112,7 @@ public class StreamingRuntimeContext implements RuntimeContext { } protected void stateSanityCheck( - AbstractStateDescriptor stateDescriptor, - AbstractKeyStateBackend backend) { + AbstractStateDescriptor stateDescriptor, AbstractKeyStateBackend backend) { Preconditions.checkNotNull(stateDescriptor, "The state properties must not be null"); Preconditions.checkNotNull(backend, "backend must not be null"); } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/InputStreamTask.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/InputStreamTask.java index eeddf13e5..9ac14ad0b 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/InputStreamTask.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/InputStreamTask.java @@ -33,8 +33,7 @@ public abstract class InputStreamTask extends StreamTask { } @Override - protected void init() { - } + protected void init() {} @Override public void run() { @@ -71,7 +70,9 @@ public abstract class InputStreamTask extends StreamTask { queueBarrier.getData().get(barrierData); RemoteCall.Barrier barrierPb = RemoteCall.Barrier.parseFrom(barrierData); final long checkpointId = barrierPb.getId(); - LOG.info("Start to do checkpoint {}, worker name is {}.", checkpointId, + LOG.info( + "Start to do checkpoint {}, worker name is {}.", + checkpointId, jobWorker.getWorkerContext().getWorkerName()); final Map inputPoints = queueBarrier.getInputOffsets(); @@ -80,8 +81,8 @@ public abstract class InputStreamTask extends StreamTask { } } } catch (Throwable throwable) { - if (throwable instanceof ChannelInterruptException || - ExceptionUtils.getRootCause(throwable) instanceof ChannelInterruptException) { + if (throwable instanceof ChannelInterruptException + || ExceptionUtils.getRootCause(throwable) instanceof ChannelInterruptException) { LOG.info("queue has stopped."); } else { // error occurred, need to rollback @@ -95,8 +96,6 @@ public abstract class InputStreamTask extends StreamTask { @Override public String toString() { - return MoreObjects.toStringHelper(this) - .add("processor", processor) - .toString(); + return MoreObjects.toStringHelper(this).add("processor", processor).toString(); } } diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/OneInputStreamTask.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/OneInputStreamTask.java index 8eaf2ef66..483f88ae1 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/OneInputStreamTask.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/OneInputStreamTask.java @@ -3,9 +3,7 @@ package io.ray.streaming.runtime.worker.tasks; import io.ray.streaming.runtime.core.processor.Processor; import io.ray.streaming.runtime.worker.JobWorker; -/** - * Input stream task with 1 input. Such as: map operator. - */ +/** Input stream task with 1 input. Such as: map operator. */ public class OneInputStreamTask extends InputStreamTask { public OneInputStreamTask(Processor inputProcessor, JobWorker jobWorker, long lastCheckpointId) { diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/SourceStreamTask.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/SourceStreamTask.java index 9fc94c06d..d5041a97b 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/SourceStreamTask.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/SourceStreamTask.java @@ -16,11 +16,9 @@ public class SourceStreamTask extends StreamTask { private final SourceProcessor sourceProcessor; - - /** - * The pending barrier ID to be triggered. - */ + /** The pending barrier ID to be triggered. */ private final AtomicReference pendingBarrier = new AtomicReference<>(); + private long lastCheckpointId = 0; /** @@ -33,8 +31,7 @@ public class SourceStreamTask extends StreamTask { } @Override - protected void init() { - } + protected void init() {} @Override public void run() { @@ -50,15 +47,19 @@ public class SourceStreamTask extends StreamTask { // Important: because cp maybe timeout, master will use the old checkpoint id again if (pendingBarrier.compareAndSet(barrierId, null)) { // source fetcher only have outputPoints - LOG.info("Start to do checkpoint {}, worker name is {}.", - barrierId, jobWorker.getWorkerContext().getWorkerName()); + LOG.info( + "Start to do checkpoint {}, worker name is {}.", + barrierId, + jobWorker.getWorkerContext().getWorkerName()); doCheckpoint(barrierId, null); LOG.info("Finish to do checkpoint {}.", barrierId); } else { // pendingCheckpointId has modify, should not happen - LOG.warn("Pending checkpointId modify unexpected, expect={}, now={}.", barrierId, + LOG.warn( + "Pending checkpointId modify unexpected, expect={}, now={}.", + barrierId, pendingBarrier.get()); } } @@ -66,8 +67,8 @@ public class SourceStreamTask extends StreamTask { sourceProcessor.fetch(); } } catch (Throwable e) { - if (e instanceof ChannelInterruptException || - ExceptionUtils.getRootCause(e) instanceof ChannelInterruptException) { + if (e instanceof ChannelInterruptException + || ExceptionUtils.getRootCause(e) instanceof ChannelInterruptException) { LOG.info("queue has stopped."); } else { // occur error, need to rollback diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/StreamTask.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/StreamTask.java index 78ef0dbd4..6acf016e1 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/StreamTask.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/StreamTask.java @@ -63,8 +63,9 @@ public abstract class StreamTask implements Runnable { this.checkpointState = jobWorker.contextBackend; this.lastCheckpointId = lastCheckpointId; - this.thread = new Thread(Ray.wrapRunnable(this), - this.getClass().getName() + "-" + System.currentTimeMillis()); + this.thread = + new Thread( + Ray.wrapRunnable(this), this.getClass().getName() + "-" + System.currentTimeMillis()); this.thread.setDaemon(true); } @@ -98,18 +99,24 @@ public abstract class StreamTask implements Runnable { } /** - * Load checkpoint and build upstream and downstream data transmission - * channels according to {@link ExecutionVertex}. + * Load checkpoint and build upstream and downstream data transmission channels according to + * {@link ExecutionVertex}. */ private void prepareTask(boolean isRecreate) { LOG.info("Preparing stream task, isRecreate={}.", isRecreate); ExecutionVertex executionVertex = jobWorker.getExecutionVertex(); // set vertex info into config for native using - jobWorker.getWorkerConfig().workerInternalConfig.setProperty( - WorkerInternalConfig.WORKER_NAME_INTERNAL, executionVertex.getExecutionVertexName()); - jobWorker.getWorkerConfig().workerInternalConfig.setProperty( - WorkerInternalConfig.OP_NAME_INTERNAL, executionVertex.getExecutionJobVertexName()); + jobWorker + .getWorkerConfig() + .workerInternalConfig + .setProperty( + WorkerInternalConfig.WORKER_NAME_INTERNAL, executionVertex.getExecutionVertexName()); + jobWorker + .getWorkerConfig() + .workerInternalConfig + .setProperty( + WorkerInternalConfig.OP_NAME_INTERNAL, executionVertex.getExecutionJobVertexName()); OperatorCheckpointInfo operatorCheckpointInfo = new OperatorCheckpointInfo(); byte[] bytes = null; @@ -118,7 +125,9 @@ public abstract class StreamTask implements Runnable { // in rescaling or something like that. if (isRecreate) { String cpKey = genOpCheckpointKey(lastCheckpointId); - LOG.info("Getting task checkpoints from state, cpKey={}, checkpointId={}.", cpKey, + LOG.info( + "Getting task checkpoints from state, cpKey={}, checkpointId={}.", + cpKey, lastCheckpointId); bytes = CheckpointStateUtil.get(checkpointState, cpKey); if (bytes == null) { @@ -133,31 +142,36 @@ public abstract class StreamTask implements Runnable { processor.loadCheckpoint(operatorCheckpointInfo.processorCheckpoint); LOG.info( "Stream task recover from checkpoint state, checkpoint bytes len={}, checkpointInfo={}.", - bytes.length, operatorCheckpointInfo); + bytes.length, + operatorCheckpointInfo); } // writer if (!executionVertex.getOutputEdges().isEmpty()) { - LOG.info("Register queue writer, channels={}, outputCheckpoints={}.", - executionVertex.getOutputChannelIdList(), operatorCheckpointInfo.outputPoints); - writer = new DataWriter( + LOG.info( + "Register queue writer, channels={}, outputCheckpoints={}.", executionVertex.getOutputChannelIdList(), - executionVertex.getOutputActorList(), - operatorCheckpointInfo.outputPoints, - jobWorker.getWorkerConfig() - ); + operatorCheckpointInfo.outputPoints); + writer = + new DataWriter( + executionVertex.getOutputChannelIdList(), + executionVertex.getOutputActorList(), + operatorCheckpointInfo.outputPoints, + jobWorker.getWorkerConfig()); } // reader if (!executionVertex.getInputEdges().isEmpty()) { - LOG.info("Register queue reader, channels={}, inputCheckpoints={}.", - executionVertex.getInputChannelIdList(), operatorCheckpointInfo.inputPoints); - reader = new DataReader( + LOG.info( + "Register queue reader, channels={}, inputCheckpoints={}.", executionVertex.getInputChannelIdList(), - executionVertex.getInputActorList(), - operatorCheckpointInfo.inputPoints, - jobWorker.getWorkerConfig() - ); + operatorCheckpointInfo.inputPoints); + reader = + new DataReader( + executionVertex.getInputChannelIdList(), + executionVertex.getInputActorList(), + operatorCheckpointInfo.inputPoints, + jobWorker.getWorkerConfig()); } openProcessor(); @@ -186,27 +200,31 @@ public abstract class StreamTask implements Runnable { opGroupedActor.get(opName).add(executionVertex.getOutputActorList().get(i)); opPartitionMap.put(opName, edge.getPartition()); } - opPartitionMap.keySet().forEach(opName -> { - collectors.add(new OutputCollector( - writer, opGroupedChannelId.get(opName), - opGroupedActor.get(opName), opPartitionMap.get(opName) - )); - }); + opPartitionMap + .keySet() + .forEach( + opName -> { + collectors.add( + new OutputCollector( + writer, + opGroupedChannelId.get(opName), + opGroupedActor.get(opName), + opPartitionMap.get(opName))); + }); - RuntimeContext runtimeContext = new StreamingRuntimeContext(executionVertex, - jobWorker.getWorkerConfig().configMap, executionVertex.getParallelism()); + RuntimeContext runtimeContext = + new StreamingRuntimeContext( + executionVertex, + jobWorker.getWorkerConfig().configMap, + executionVertex.getParallelism()); processor.open(collectors, runtimeContext); } - /** - * Task initialization related work. - */ + /** Task initialization related work. */ protected abstract void init() throws Exception; - /** - * Close running tasks. - */ + /** Close running tasks. */ public void close() { this.running = false; if (thread.isAlive() && !Ray.getRuntimeContext().isSingleProcess()) { @@ -230,23 +248,24 @@ public abstract class StreamTask implements Runnable { Map outputPoints = null; if (writer != null) { outputPoints = writer.getOutputCheckpoints(); - RemoteCall.Barrier barrierPb = - RemoteCall.Barrier.newBuilder().setId(checkpointId).build(); + RemoteCall.Barrier barrierPb = RemoteCall.Barrier.newBuilder().setId(checkpointId).build(); ByteBuffer byteBuffer = ByteBuffer.wrap(barrierPb.toByteArray()); byteBuffer.order(ByteOrder.nativeOrder()); writer.broadcastBarrier(checkpointId, byteBuffer); } - LOG.info("Start do checkpoint, cp id={}, inputPoints={}, outputPoints={}.", checkpointId, - inputPoints, outputPoints); + LOG.info( + "Start do checkpoint, cp id={}, inputPoints={}, outputPoints={}.", + checkpointId, + inputPoints, + outputPoints); this.lastCheckpointId = checkpointId; Serializable processorCheckpoint = processor.saveCheckpoint(); try { OperatorCheckpointInfo opCpInfo = - new OperatorCheckpointInfo(inputPoints, outputPoints, processorCheckpoint, - checkpointId); + new OperatorCheckpointInfo(inputPoints, outputPoints, processorCheckpoint, checkpointId); saveCpStateAndReport(opCpInfo, checkpointId); } catch (Exception e) { // there will be exceptions when flush state to backend. @@ -258,8 +277,7 @@ public abstract class StreamTask implements Runnable { } private void saveCpStateAndReport( - OperatorCheckpointInfo operatorCheckpointInfo, - long checkpointId) { + OperatorCheckpointInfo operatorCheckpointInfo, long checkpointId) { saveCp(operatorCheckpointInfo, checkpointId); reportCommit(checkpointId); @@ -269,8 +287,11 @@ public abstract class StreamTask implements Runnable { private void saveCp(OperatorCheckpointInfo operatorCheckpointInfo, long checkpointId) { byte[] bytes = Serializer.encode(operatorCheckpointInfo); String cpKey = genOpCheckpointKey(checkpointId); - LOG.info("Saving task checkpoint, cpKey={}, byte len={}, checkpointInfo={}.", cpKey, - bytes.length, operatorCheckpointInfo); + LOG.info( + "Saving task checkpoint, cpKey={}, byte len={}, checkpointInfo={}.", + cpKey, + bytes.length, + operatorCheckpointInfo); synchronized (checkpointState) { if (outdatedCheckpoints.contains(checkpointId)) { LOG.info("Outdated checkpoint, skip save checkpoint."); @@ -284,8 +305,8 @@ public abstract class StreamTask implements Runnable { private void reportCommit(long checkpointId) { final JobWorkerContext context = jobWorker.getWorkerContext(); LOG.info("Report commit async, checkpoint id {}.", checkpointId); - RemoteCallMaster.reportJobWorkerCommitAsync(context.getMaster(), - new WorkerCommitReport(context.getWorkerActorId(), checkpointId)); + RemoteCallMaster.reportJobWorkerCommitAsync( + context.getMaster(), new WorkerCommitReport(context.getWorkerActorId(), checkpointId)); } public void notifyCheckpointTimeout(long checkpointId) { @@ -335,7 +356,11 @@ public abstract class StreamTask implements Runnable { // TODO: need to support job restart and actorId changed final JobWorkerContext context = jobWorker.getWorkerContext(); return jobWorker.getWorkerConfig().checkpointConfig.jobWorkerOpCpPrefixKey() - + context.getJobName() + "_" + context.getWorkerName() + "_" + checkpointId; + + context.getJobName() + + "_" + + context.getWorkerName() + + "_" + + checkpointId; } // ---------------------------------------------------------------------- diff --git a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/TwoInputStreamTask.java b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/TwoInputStreamTask.java index 40870f51a..3ae3c6fe4 100644 --- a/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/TwoInputStreamTask.java +++ b/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/worker/tasks/TwoInputStreamTask.java @@ -4,9 +4,7 @@ import io.ray.streaming.runtime.core.processor.Processor; import io.ray.streaming.runtime.core.processor.TwoInputProcessor; import io.ray.streaming.runtime.worker.JobWorker; -/** - * Input stream task with 2 inputs. Such as: join operator. - */ +/** Input stream task with 2 inputs. Such as: join operator. */ public class TwoInputStreamTask extends InputStreamTask { public TwoInputStreamTask( @@ -19,5 +17,4 @@ public class TwoInputStreamTask extends InputStreamTask { ((TwoInputProcessor) (super.processor)).setLeftStream(leftStream); ((TwoInputProcessor) (super.processor)).setRightStream(rightStream); } - } diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/BaseUnitTest.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/BaseUnitTest.java index 569888335..9a343f06f 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/BaseUnitTest.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/BaseUnitTest.java @@ -24,13 +24,17 @@ public abstract class BaseUnitTest { @BeforeMethod public void testBegin(Method method) { - LOG.info(">>>>>>>>>>>>>>>>>>>> Test case: {}.{} began >>>>>>>>>>>>>>>>>>>>", - method.getDeclaringClass(), method.getName()); + LOG.info( + ">>>>>>>>>>>>>>>>>>>> Test case: {}.{} began >>>>>>>>>>>>>>>>>>>>", + method.getDeclaringClass(), + method.getName()); } @AfterMethod public void testEnd(Method method) { - LOG.info(">>>>>>>>>>>>>>>>>>>> Test case: {}.{} end >>>>>>>>>>>>>>>>>>>>", - method.getDeclaringClass(), method.getName()); + LOG.info( + ">>>>>>>>>>>>>>>>>>>> Test case: {}.{} end >>>>>>>>>>>>>>>>>>>>", + method.getDeclaringClass(), + method.getName()); } } diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/TestHelper.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/TestHelper.java index dd200d318..af0aec2db 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/TestHelper.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/TestHelper.java @@ -15,4 +15,4 @@ public class TestHelper { public static boolean isUT() { return UT_FLAG; } -} \ No newline at end of file +} diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/core/graph/ExecutionGraphTest.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/core/graph/ExecutionGraphTest.java index 9af1899ac..c6bb79679 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/core/graph/ExecutionGraphTest.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/core/graph/ExecutionGraphTest.java @@ -43,20 +43,25 @@ public class ExecutionGraphTest extends BaseUnitTest { Assert.assertEquals(executionJobVertices.size(), jobGraph.getJobVertices().size()); - int totalVertexNum = jobGraph.getJobVertices().stream() - .mapToInt(JobVertex::getParallelism).sum(); + int totalVertexNum = + jobGraph.getJobVertices().stream().mapToInt(JobVertex::getParallelism).sum(); Assert.assertEquals(executionGraph.getAllExecutionVertices().size(), totalVertexNum); - Assert.assertEquals(executionGraph.getAllExecutionVertices().size(), + Assert.assertEquals( + executionGraph.getAllExecutionVertices().size(), executionGraph.getExecutionVertexIdGenerator().get()); - executionGraph.getAllExecutionVertices().forEach(vertex -> { - Assert.assertNotNull(vertex.getStreamOperator()); - Assert.assertNotNull(vertex.getExecutionJobVertexName()); - Assert.assertNotNull(vertex.getVertexType()); - Assert.assertNotNull(vertex.getLanguage()); - Assert.assertEquals(vertex.getExecutionVertexName(), - vertex.getExecutionJobVertexName() + "-" + vertex.getExecutionVertexIndex()); - }); + executionGraph + .getAllExecutionVertices() + .forEach( + vertex -> { + Assert.assertNotNull(vertex.getStreamOperator()); + Assert.assertNotNull(vertex.getExecutionJobVertexName()); + Assert.assertNotNull(vertex.getVertexType()); + Assert.assertNotNull(vertex.getLanguage()); + Assert.assertEquals( + vertex.getExecutionVertexName(), + vertex.getExecutionJobVertexName() + "-" + vertex.getExecutionVertexIndex()); + }); int startIndex = 0; ExecutionJobVertex upStream = executionJobVertices.get(startIndex); @@ -65,13 +70,17 @@ public class ExecutionGraphTest extends BaseUnitTest { List upStreamVertices = upStream.getExecutionVertices(); List downStreamVertices = downStream.getExecutionVertices(); - upStreamVertices.forEach(vertex -> { - Assert.assertEquals((double) vertex.getResource().get(ResourceType.CPU.name()), 2.0); - vertex.getOutputEdges().forEach(upStreamOutPutEdge -> { - Assert - .assertTrue(downStreamVertices.contains(upStreamOutPutEdge.getTargetExecutionVertex())); - }); - }); + upStreamVertices.forEach( + vertex -> { + Assert.assertEquals((double) vertex.getResource().get(ResourceType.CPU.name()), 2.0); + vertex + .getOutputEdges() + .forEach( + upStreamOutPutEdge -> { + Assert.assertTrue( + downStreamVertices.contains(upStreamOutPutEdge.getTargetExecutionVertex())); + }); + }); } public static ExecutionGraph buildExecutionGraph(GraphManager graphManager) { @@ -84,8 +93,8 @@ public class ExecutionGraphTest extends BaseUnitTest { public static JobGraph buildJobGraph() { StreamingContext streamingContext = StreamingContext.buildContext(); - DataStream dataStream = DataStreamSource.fromCollection(streamingContext, - Lists.newArrayList("a", "b", "c")); + DataStream dataStream = + DataStreamSource.fromCollection(streamingContext, Lists.newArrayList("a", "b", "c")); StreamSink streamSink = dataStream.sink(x -> LOG.info(x)); Map jobConfig = new HashMap<>(); @@ -94,10 +103,9 @@ public class ExecutionGraphTest extends BaseUnitTest { jobConfig.put(ResourceConfig.TASK_RESOURCE_CPU, "2.0"); jobConfig.put(ResourceConfig.TASK_RESOURCE_MEM, "2.0"); - JobGraphBuilder jobGraphBuilder = new JobGraphBuilder( - Lists.newArrayList(streamSink), "test", jobConfig); + JobGraphBuilder jobGraphBuilder = + new JobGraphBuilder(Lists.newArrayList(streamSink), "test", jobConfig); return jobGraphBuilder.build(); } - } diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/HybridStreamTest.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/HybridStreamTest.java index 4db1a1f0f..35eeeddc2 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/HybridStreamTest.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/HybridStreamTest.java @@ -45,8 +45,7 @@ public class HybridStreamTest { @Test(timeOut = 60000) public void testHybridDataStream() throws Exception { Ray.shutdown(); - Preconditions.checkArgument( - EnvUtil.executeCommand(ImmutableList.of("ray", "stop"), 5)); + Preconditions.checkArgument(EnvUtil.executeCommand(ImmutableList.of("ray", "stop"), 5)); String sinkFileName = "/tmp/testHybridDataStream.txt"; Files.deleteIfExists(Paths.get(sinkFileName)); @@ -59,18 +58,22 @@ public class HybridStreamTest { .map("ray.streaming.tests.test_hybrid_stream", "map_func1") .filter("ray.streaming.tests.test_hybrid_stream", "filter_func1") .asJavaStream() - .sink((SinkFunction) value -> { - LOG.info("HybridStreamTest: {}", value); - try { - if (!Files.exists(Paths.get(sinkFileName))) { - Files.createFile(Paths.get(sinkFileName)); - } - Files.write(Paths.get(sinkFileName), value.toString().getBytes(), - StandardOpenOption.APPEND); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + .sink( + (SinkFunction) + value -> { + LOG.info("HybridStreamTest: {}", value); + try { + if (!Files.exists(Paths.get(sinkFileName))) { + Files.createFile(Paths.get(sinkFileName)); + } + Files.write( + Paths.get(sinkFileName), + value.toString().getBytes(), + StandardOpenOption.APPEND); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); context.execute("HybridStreamTestJob"); int sleptTime = 0; TimeUnit.SECONDS.sleep(3); @@ -94,5 +97,4 @@ public class HybridStreamTest { context.stop(); LOG.info("HybridStreamTest succeed"); } - } diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/UnionStreamTest.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/UnionStreamTest.java index b7e2aef61..26f679fc2 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/UnionStreamTest.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/UnionStreamTest.java @@ -35,18 +35,22 @@ public class UnionStreamTest { DataStreamSource.fromCollection(context, Arrays.asList(1, 1)); streamSource1 .union(streamSource2, streamSource3) - .sink((SinkFunction) value -> { - LOG.info("UnionStreamTest, sink: {}", value); - try { - if (!Files.exists(Paths.get(sinkFileName))) { - Files.createFile(Paths.get(sinkFileName)); - } - Files.write(Paths.get(sinkFileName), value.toString().getBytes(), - StandardOpenOption.APPEND); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); + .sink( + (SinkFunction) + value -> { + LOG.info("UnionStreamTest, sink: {}", value); + try { + if (!Files.exists(Paths.get(sinkFileName))) { + Files.createFile(Paths.get(sinkFileName)); + } + Files.write( + Paths.get(sinkFileName), + value.toString().getBytes(), + StandardOpenOption.APPEND); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); context.execute("UnionStreamTest"); int sleptTime = 0; TimeUnit.SECONDS.sleep(3); @@ -68,5 +72,4 @@ public class UnionStreamTest { context.stop(); LOG.info("HybridStreamTest succeed"); } - } diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/WordCountTest.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/WordCountTest.java index 11234b7d9..dc75b2b07 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/WordCountTest.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/demo/WordCountTest.java @@ -37,19 +37,21 @@ public class WordCountTest extends BaseUnitTest implements Serializable { text.add("hello world eagle eagle eagle"); DataStreamSource streamSource = DataStreamSource.fromCollection(streamingContext, text); streamSource - .flatMap((FlatMapFunction) (value, collector) -> { - String[] records = value.split(" "); - for (String record : records) { - collector.collect(new WordAndCount(record, 1)); - } - }) + .flatMap( + (FlatMapFunction) + (value, collector) -> { + String[] records = value.split(" "); + for (String record : records) { + collector.collect(new WordAndCount(record, 1)); + } + }) .filter(pair -> !pair.word.contains("world")) .keyBy(pair -> pair.word) - .reduce((ReduceFunction) (oldValue, newValue) -> - new WordAndCount(oldValue.word, - oldValue.count + newValue.count)) - .sink((SinkFunction) - result -> wordCount.put(result.word, result.count)); + .reduce( + (ReduceFunction) + (oldValue, newValue) -> + new WordAndCount(oldValue.word, oldValue.count + newValue.count)) + .sink((SinkFunction) result -> wordCount.put(result.word, result.count)); streamingContext.execute("testWordCount"); @@ -74,5 +76,4 @@ public class WordCountTest extends BaseUnitTest implements Serializable { this.count = count; } } - } diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/master/JobMasterTest.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/master/JobMasterTest.java index 3951911bb..def36a2c0 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/master/JobMasterTest.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/master/JobMasterTest.java @@ -31,5 +31,4 @@ public class JobMasterTest extends BaseUnitTest { Assert.assertNull(jobMaster.getJobMasterActor()); Assert.assertFalse(jobMaster.init(false)); } - } diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/master/jobscheduler/JobClientTest.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/master/jobscheduler/JobClientTest.java index 0d5b2766b..19c3b8d34 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/master/jobscheduler/JobClientTest.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/master/jobscheduler/JobClientTest.java @@ -8,5 +8,4 @@ public class JobClientTest { public void testSchedule() { // TODO (tianyi): need JobWorker Part to do this. } - } diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/master/resourcemanager/strategy/PipelineFirstStrategyTest.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/master/resourcemanager/strategy/PipelineFirstStrategyTest.java index 2e42e606b..ad25cb684 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/master/resourcemanager/strategy/PipelineFirstStrategyTest.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/master/resourcemanager/strategy/PipelineFirstStrategyTest.java @@ -70,6 +70,4 @@ public class PipelineFirstStrategyTest extends BaseUnitTest { ResourceAssignmentView assignmentView = strategy.assignResource(containers, executionGraph); Assert.assertNotNull(assignmentView); } - - } diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/python/PythonGatewayTest.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/python/PythonGatewayTest.java index 831e35b92..dfd412520 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/python/PythonGatewayTest.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/python/PythonGatewayTest.java @@ -2,7 +2,6 @@ package io.ray.streaming.runtime.python; import static org.testng.Assert.assertEquals; - import io.ray.streaming.api.stream.StreamSink; import io.ray.streaming.jobgraph.JobGraph; import io.ray.streaming.jobgraph.JobGraphBuilder; @@ -24,27 +23,25 @@ public class PythonGatewayTest { config.put("k1", "v1"); gateway.withConfig(serializer.serialize(config)); byte[] mockPySource = new byte[0]; - Object source = serializer.deserialize( - gateway.createPythonStreamSource(mockPySource)); + Object source = serializer.deserialize(gateway.createPythonStreamSource(mockPySource)); byte[] mockPyFunc = new byte[0]; Object mapPyFunc = serializer.deserialize(gateway.createPyFunc(mockPyFunc)); - Object mapStream = serializer.deserialize( - gateway.callMethod( - serializer.serialize(Arrays.asList(source, "map", mapPyFunc)))); + Object mapStream = + serializer.deserialize( + gateway.callMethod(serializer.serialize(Arrays.asList(source, "map", mapPyFunc)))); byte[] mockPyPartition = new byte[0]; - Object partition = serializer.deserialize( - gateway.createPyPartition(mockPyPartition)); - Object partitionedStream = serializer.deserialize( - gateway.callMethod( - serializer.serialize(Arrays.asList(mapStream, "partitionBy", partition)))); + Object partition = serializer.deserialize(gateway.createPyPartition(mockPyPartition)); + Object partitionedStream = + serializer.deserialize( + gateway.callMethod( + serializer.serialize(Arrays.asList(mapStream, "partitionBy", partition)))); byte[] mockSinkFunc = new byte[0]; Object sinkPyFunc = serializer.deserialize(gateway.createPyFunc(mockSinkFunc)); - gateway.callMethod( - serializer.serialize(Arrays.asList(partitionedStream, "sink", sinkPyFunc))); + gateway.callMethod(serializer.serialize(Arrays.asList(partitionedStream, "sink", sinkPyFunc))); List streamSinks = gateway.getStreamingContext().getStreamSinks(); assertEquals(streamSinks.size(), 1); JobGraphBuilder jobGraphBuilder = new JobGraphBuilder(streamSinks, "py_job"); JobGraph jobGraph = jobGraphBuilder.build(); jobGraph.printJobGraph(); } -} \ No newline at end of file +} diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/serialization/CrossLangSerializerTest.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/serialization/CrossLangSerializerTest.java index 2c37a0d5f..a3584d2e7 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/serialization/CrossLangSerializerTest.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/serialization/CrossLangSerializerTest.java @@ -3,7 +3,6 @@ package io.ray.streaming.runtime.serialization; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; - import io.ray.streaming.message.KeyRecord; import io.ray.streaming.message.Record; import org.apache.commons.lang3.builder.EqualsBuilder; @@ -17,11 +16,11 @@ public class CrossLangSerializerTest { CrossLangSerializer serializer = new CrossLangSerializer(); Record record = new Record("value"); record.setStream("stream1"); - assertTrue(EqualsBuilder.reflectionEquals(record, - serializer.deserialize(serializer.serialize(record)))); + assertTrue( + EqualsBuilder.reflectionEquals( + record, serializer.deserialize(serializer.serialize(record)))); KeyRecord keyRecord = new KeyRecord("key", "value"); keyRecord.setStream("stream2"); - assertEquals(keyRecord, - serializer.deserialize(serializer.serialize(keyRecord))); + assertEquals(keyRecord, serializer.deserialize(serializer.serialize(keyRecord))); } -} \ No newline at end of file +} diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/serialization/MsgPackSerializerTest.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/serialization/MsgPackSerializerTest.java index 9c140c9cc..4a574e4dd 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/serialization/MsgPackSerializerTest.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/serialization/MsgPackSerializerTest.java @@ -3,7 +3,6 @@ package io.ray.streaming.runtime.serialization; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; - import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -18,20 +17,17 @@ public class MsgPackSerializerTest { public void testSerializeByte() { MsgPackSerializer serializer = new MsgPackSerializer(); - assertEquals(serializer.deserialize( - serializer.serialize((byte) 1)), (byte) 1); + assertEquals(serializer.deserialize(serializer.serialize((byte) 1)), (byte) 1); } @Test public void testSerialize() { MsgPackSerializer serializer = new MsgPackSerializer(); - assertEquals(serializer.deserialize - (serializer.serialize(Short.MAX_VALUE)), Short.MAX_VALUE); - assertEquals(serializer.deserialize( - serializer.serialize(Integer.MAX_VALUE)), Integer.MAX_VALUE); - assertEquals(serializer.deserialize( - serializer.serialize(Long.MAX_VALUE)), Long.MAX_VALUE); + assertEquals(serializer.deserialize(serializer.serialize(Short.MAX_VALUE)), Short.MAX_VALUE); + assertEquals( + serializer.deserialize(serializer.serialize(Integer.MAX_VALUE)), Integer.MAX_VALUE); + assertEquals(serializer.deserialize(serializer.serialize(Long.MAX_VALUE)), Long.MAX_VALUE); Map map = new HashMap(); List list = new ArrayList<>(); @@ -47,8 +43,7 @@ public class MsgPackSerializerTest { assertEquals(o, map); byte[] binary = {1, 2, 3, 4}; - assertTrue(Arrays.equals( - binary, (byte[]) (serializer.deserialize(serializer.serialize(binary))))); + assertTrue( + Arrays.equals(binary, (byte[]) (serializer.deserialize(serializer.serialize(binary))))); } - -} \ No newline at end of file +} diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/StreamingQueueTest.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/StreamingQueueTest.java index c93d19911..3ff425d25 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/StreamingQueueTest.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/StreamingQueueTest.java @@ -3,7 +3,6 @@ package io.ray.streaming.runtime.streamingqueue; import com.google.common.collect.ImmutableMap; import io.ray.api.ActorHandle; import io.ray.api.Ray; -import io.ray.runtime.config.RayConfig; import io.ray.streaming.api.context.StreamingContext; import io.ray.streaming.api.function.impl.FlatMapFunction; import io.ray.streaming.api.function.impl.ReduceFunction; @@ -76,8 +75,8 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable { @Test(timeOut = 300000) public void testReaderWriter() { - LOGGER.info("StreamingQueueTest.testReaderWriter run-mode: {}", - System.getProperty("ray.run-mode")); + LOGGER.info( + "StreamingQueueTest.testReaderWriter run-mode: {}", System.getProperty("ray.run-mode")); Ray.shutdown(); System.setProperty("ray.head-args.0", "--num-cpus=4"); System.setProperty("ray.head-args.1", "--resources={\"RES-A\":4}"); @@ -90,10 +89,10 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable { ActorHandle writerActor = Ray.actor(WriterWorker::new, "writer").remote(); ActorHandle readerActor = Ray.actor(ReaderWorker::new, "reader").remote(); - LOGGER.info("call getName on writerActor: {}", - writerActor.task(WriterWorker::getName).remote().get()); - LOGGER.info("call getName on readerActor: {}", - readerActor.task(ReaderWorker::getName).remote().get()); + LOGGER.info( + "call getName on writerActor: {}", writerActor.task(WriterWorker::getName).remote().get()); + LOGGER.info( + "call getName on readerActor: {}", readerActor.task(ReaderWorker::getName).remote().get()); // LOGGER.info(writerActor.task(WriterWorker::testCallReader, readerActor).remote().get()); List outputQueueList = new ArrayList<>(); @@ -117,8 +116,8 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable { writerActor.task(WriterWorker::init, outputQueueList, readerActor, msgCount).remote(); long time = 0; - while (time < 20000 && - readerActor.task(ReaderWorker::getTotalMsg).remote().get() < msgCount * queueNum) { + while (time < 20000 + && readerActor.task(ReaderWorker::getTotalMsg).remote().get() < msgCount * queueNum) { try { Thread.sleep(1000); time += 1000; @@ -128,8 +127,7 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable { } Assert.assertEquals( - readerActor.task(ReaderWorker::getTotalMsg).remote().get().intValue(), - msgCount * queueNum); + readerActor.task(ReaderWorker::getTotalMsg).remote().get().intValue(), msgCount * queueNum); } @Test(timeOut = 60000) @@ -143,8 +141,8 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable { // ray init Ray.init(); LOGGER.info("testWordCount"); - LOGGER.info("StreamingQueueTest.testWordCount run-mode: {}", - System.getProperty("ray.run-mode")); + LOGGER.info( + "StreamingQueueTest.testWordCount run-mode: {}", System.getProperty("ray.run-mode")); String resultFile = "/tmp/io.ray.streaming.runtime.streamingqueue.testWordCount.txt"; deleteResultFile(resultFile); @@ -158,22 +156,27 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable { text.add("hello world eagle eagle eagle"); DataStreamSource streamSource = DataStreamSource.fromCollection(streamingContext, text); streamSource - .flatMap((FlatMapFunction) (value, collector) -> { - String[] records = value.split(" "); - for (String record : records) { - collector.collect(new WordAndCount(record, 1)); - } - }) + .flatMap( + (FlatMapFunction) + (value, collector) -> { + String[] records = value.split(" "); + for (String record : records) { + collector.collect(new WordAndCount(record, 1)); + } + }) .keyBy(pair -> pair.word) - .reduce((ReduceFunction) (oldValue, newValue) -> { - LOGGER.info("reduce: {} {}", oldValue, newValue); - return new WordAndCount(oldValue.word, oldValue.count + newValue.count); - }) - .sink(s -> { - LOGGER.info("sink {} {}", s.word, s.count); - wordCount.put(s.word, s.count); - serializeResultToFile(resultFile, wordCount); - }); + .reduce( + (ReduceFunction) + (oldValue, newValue) -> { + LOGGER.info("reduce: {} {}", oldValue, newValue); + return new WordAndCount(oldValue.word, oldValue.count + newValue.count); + }) + .sink( + s -> { + LOGGER.info("sink {} {}", s.word, s.count); + wordCount.put(s.word, s.count); + serializeResultToFile(resultFile, wordCount); + }); streamingContext.execute("testSQWordCount"); @@ -190,8 +193,7 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable { checkWordCount = (Map) deserializeResultFromFile(resultFile); } LOGGER.info("check"); - Assert.assertEquals(checkWordCount, - ImmutableMap.of("eagle", 3, "hello", 1, "world", 1)); + Assert.assertEquals(checkWordCount, ImmutableMap.of("eagle", 3, "hello", 1, "world", 1)); } private void serializeResultToFile(String fileName, Object obj) { @@ -208,8 +210,7 @@ public class StreamingQueueTest extends BaseUnitTest implements Serializable { try { ObjectInputStream in = new ObjectInputStream(new FileInputStream(fileName)); checkWordCount = (Map) in.readObject(); - Assert.assertEquals(checkWordCount, - ImmutableMap.of("eagle", 3, "hello", 1, "world", 1)); + Assert.assertEquals(checkWordCount, ImmutableMap.of("eagle", 3, "hello", 1, "world", 1)); } catch (Exception e) { LOGGER.error(String.valueOf(e)); } diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/Worker.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/Worker.java index 1305267e2..59201e453 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/Worker.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/streamingqueue/Worker.java @@ -107,12 +107,15 @@ class ReaderWorker extends Worker { dataReader = new DataReader(inputQueueList, inputActors, new HashMap<>(), workerConfig); // Should not GetBundle in RayCall thread - Thread readThread = new Thread(Ray.wrapRunnable(new Runnable() { - @Override - public void run() { - consume(); - } - })); + Thread readThread = + new Thread( + Ray.wrapRunnable( + new Runnable() { + @Override + public void run() { + consume(); + } + })); readThread.start(); LOGGER.info("ReaderWorker init done"); @@ -136,8 +139,11 @@ class ReaderWorker extends Worker { int dataSize = dataMessage.body().getInt(); // check size - LOGGER.info("capacity {} bufferSize {} dataSize {}", - dataMessage.body().capacity(), bufferSize, dataSize); + LOGGER.info( + "capacity {} bufferSize {} dataSize {}", + dataMessage.body().capacity(), + bufferSize, + dataSize); Assert.assertEquals(bufferSize, dataSize); if (dataMessage instanceof DataMessage) { if (LOGGER.isInfoEnabled()) { @@ -158,9 +164,7 @@ class ReaderWorker extends Worker { LOGGER.info("ReaderWorker consume data done."); } - void onQueueTransfer(long handler, byte[] buffer) { - } - + void onQueueTransfer(long handler, byte[] buffer) {} public boolean done() { return totalMsg == msgCount; @@ -233,12 +237,15 @@ class WriterWorker extends Worker { new JavaFunctionDescriptor(Worker.class.getName(), "onReaderMessageSync", "([B)[B")); StreamingWorkerConfig workerConfig = new StreamingWorkerConfig(conf); dataWriter = new DataWriter(outputQueueList, outputActors, new HashMap<>(), workerConfig); - Thread writerThread = new Thread(Ray.wrapRunnable(new Runnable() { - @Override - public void run() { - produce(); - } - })); + Thread writerThread = + new Thread( + Ray.wrapRunnable( + new Runnable() { + @Override + public void run() { + produce(); + } + })); writerThread.start(); LOGGER.info("WriterWorker init done"); diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/transfer/ChannelIdTest.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/transfer/ChannelIdTest.java index 46270837e..0a91d4539 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/transfer/ChannelIdTest.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/transfer/ChannelIdTest.java @@ -2,7 +2,6 @@ package io.ray.streaming.runtime.transfer; import static org.testng.Assert.assertEquals; - import io.ray.streaming.runtime.BaseUnitTest; import io.ray.streaming.runtime.transfer.channel.ChannelId; import io.ray.streaming.runtime.util.EnvUtil; @@ -20,5 +19,4 @@ public class ChannelIdTest extends BaseUnitTest { assertEquals(idStr.length(), ChannelId.ID_LENGTH * 2); assertEquals(ChannelId.idStrToBytes(idStr).length, ChannelId.ID_LENGTH); } - -} \ No newline at end of file +} diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/util/Mockitools.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/util/Mockitools.java index 6ba018e27..5fe774e20 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/util/Mockitools.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/util/Mockitools.java @@ -10,23 +10,17 @@ import java.util.Map; import java.util.stream.Collectors; import org.powermock.api.mockito.PowerMockito; -/** - * Mockitools is a tool based on powermock and mokito to mock external service api - */ +/** Mockitools is a tool based on powermock and mokito to mock external service api */ public class Mockitools { - /** - * Mock GCS get node info api - */ + /** Mock GCS get node info api */ public static void mockGscApi() { PowerMockito.mockStatic(RayUtils.class); PowerMockito.when(RayUtils.getAliveNodeInfoMap()) .thenReturn(mockGetNodeInfoMap(mockGetAllNodeInfo())); } - /** - * Mock get all node info from GCS - */ + /** Mock get all node info from GCS */ public static List mockGetAllNodeInfo() { List nodeInfos = new LinkedList<>(); @@ -55,25 +49,18 @@ public class Mockitools { /** * Mock get node info map * - * @param nodeInfos all node infos fetched from GCS - * @return node info map, key is node unique id, value is node info + * @param nodeInfos all node infos fetched from GCS Returns node info map, key is node unique id, + * value is node info */ public static Map mockGetNodeInfoMap(List nodeInfos) { - return nodeInfos.stream().filter(nodeInfo -> nodeInfo.isAlive).collect( - Collectors.toMap(nodeInfo -> nodeInfo.nodeId, nodeInfo -> nodeInfo)); + return nodeInfos.stream() + .filter(nodeInfo -> nodeInfo.isAlive) + .collect(Collectors.toMap(nodeInfo -> nodeInfo.nodeId, nodeInfo -> nodeInfo)); } private static NodeInfo mockNodeInfo(int i, Map resources) { return new NodeInfo( - createNodeId(i), - "localhost" + i, - "localhost" + i, - -1, - "", - "", - - true, - resources); + createNodeId(i), "localhost" + i, "localhost" + i, -1, "", "", true, resources); } private static UniqueId createNodeId(int id) { diff --git a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/util/ReflectionUtilsTest.java b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/util/ReflectionUtilsTest.java index d263ead06..570f067d7 100644 --- a/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/util/ReflectionUtilsTest.java +++ b/streaming/java/streaming-runtime/src/test/java/io/ray/streaming/runtime/util/ReflectionUtilsTest.java @@ -2,7 +2,6 @@ package io.ray.streaming.runtime.util; import static org.testng.Assert.assertEquals; - import java.io.Serializable; import java.util.Collections; import org.testng.annotations.Test; @@ -11,20 +10,16 @@ public class ReflectionUtilsTest { static class Foo implements Serializable { - public void f1() { - } + public void f1() {} - public void f2() { - } + public void f2() {} - public void f2(boolean a1) { - } + public void f2(boolean a1) {} } @Test public void testFindMethod() throws NoSuchMethodException { - assertEquals(Foo.class.getDeclaredMethod("f1"), - ReflectionUtils.findMethod(Foo.class, "f1")); + assertEquals(Foo.class.getDeclaredMethod("f1"), ReflectionUtils.findMethod(Foo.class, "f1")); } @Test @@ -34,7 +29,7 @@ public class ReflectionUtilsTest { @Test public void testGetAllInterfaces() { - assertEquals(ReflectionUtils.getAllInterfaces(Foo.class), - Collections.singletonList(Serializable.class)); + assertEquals( + ReflectionUtils.getAllInterfaces(Foo.class), Collections.singletonList(Serializable.class)); } -} \ No newline at end of file +} diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/KeyValueState.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/KeyValueState.java index b62571e59..69a7931a5 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/KeyValueState.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/KeyValueState.java @@ -18,18 +18,12 @@ package io.ray.streaming.state; -/** - * Key Value State interface. - */ +/** Key Value State interface. */ public interface KeyValueState { - /** - * get value from state - */ + /** get value from state */ V get(K key); - /** - * put key and value into state - */ + /** put key and value into state */ void put(K k, V v); } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/PartitionRecord.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/PartitionRecord.java index 4f58ba7c1..50fd3fe43 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/PartitionRecord.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/PartitionRecord.java @@ -20,19 +20,15 @@ package io.ray.streaming.state; import java.io.Serializable; -/** - * value record for partition. - */ +/** value record for partition. */ public class PartitionRecord implements Serializable { - /** - * The partition number of the partitioned value. - */ + /** The partition number of the partitioned value. */ private int partitionID; + private T value; - public PartitionRecord() { - } + public PartitionRecord() {} public PartitionRecord(int partitionID, T value) { this.partitionID = partitionID; diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/StateException.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/StateException.java index af608eac5..f3eb5b2cc 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/StateException.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/StateException.java @@ -18,9 +18,7 @@ package io.ray.streaming.state; -/** - * RuntimeException wrapper, indicating the exceptions occurs in states. - */ +/** RuntimeException wrapper, indicating the exceptions occurs in states. */ public class StateException extends RuntimeException { public StateException(Throwable t) { diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/StateStoreManager.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/StateStoreManager.java index d5d568a7c..b1f3c547d 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/StateStoreManager.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/StateStoreManager.java @@ -20,8 +20,8 @@ package io.ray.streaming.state; /** * TransactionState interface. - *

- * Streaming State should implement transaction in case of failure, which in our case is four + * + *

Streaming State should implement transaction in case of failure, which in our case is four * default method, finish, commit, ackCommit, rollback. */ public interface StateStoreManager { @@ -44,8 +44,6 @@ public interface StateStoreManager { */ void ackCommit(long checkpointId, long timeStamp); - /** - * The rollback method is used for recovering the checkpoint. - */ + /** The rollback method is used for recovering the checkpoint. */ void rollBack(long checkpointId); } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/StorageRecord.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/StorageRecord.java index 03421e679..dcce1d017 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/StorageRecord.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/StorageRecord.java @@ -20,16 +20,13 @@ package io.ray.streaming.state; import java.io.Serializable; -/** - * This Class contains a record with some checkpointId. - */ +/** This Class contains a record with some checkpointId. */ public class StorageRecord implements Serializable { private long checkpointId; private T value; - public StorageRecord() { - } + public StorageRecord() {} public StorageRecord(long checkpointId, T value) { this.checkpointId = checkpointId; diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/AbstractKeyStateBackend.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/AbstractKeyStateBackend.java index 6edc312c1..43c4c7813 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/AbstractKeyStateBackend.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/AbstractKeyStateBackend.java @@ -36,8 +36,8 @@ import org.slf4j.LoggerFactory; /** * Transaction support primitive operations like finish, commit, ackcommit and rollback. - *

- * State value modification is not thread safe! By default, every processing thread has its own + * + *

State value modification is not thread safe! By default, every processing thread has its own * space to handle state. */ public abstract class AbstractKeyStateBackend implements StateStoreManager { @@ -52,10 +52,9 @@ public abstract class AbstractKeyStateBackend implements StateStoreManager { protected Map mapManagerProxyHashMap = new HashMap<>(); protected Set descNamespace; - /** - * tablename, KeyValueStore key, checkpointId, content - */ + /** tablename, KeyValueStore key, checkpointId, content */ protected Map>> backStorageCache; + private AbstractStateBackend backend; public AbstractKeyStateBackend(AbstractStateBackend backend) { diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/AbstractStateBackend.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/AbstractStateBackend.java index 3392f32df..1f0408282 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/AbstractStateBackend.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/AbstractStateBackend.java @@ -20,7 +20,6 @@ package io.ray.streaming.state.backend; import static io.ray.streaming.state.config.ConfigKey.DELIMITER; - import io.ray.streaming.state.config.ConfigKey; import io.ray.streaming.state.keystate.desc.AbstractStateDescriptor; import io.ray.streaming.state.serialization.KeyMapStoreSerializer; @@ -29,9 +28,7 @@ import io.ray.streaming.state.store.KeyValueStore; import java.io.Serializable; import java.util.Map; -/** - * This class is the abstract class for different kinds of StateBackend. - */ +/** This class is the abstract class for different kinds of StateBackend. */ public abstract class AbstractStateBackend implements Serializable { protected final Map config; diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/BackendType.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/BackendType.java index cc616647e..7c63d3e65 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/BackendType.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/BackendType.java @@ -18,18 +18,12 @@ package io.ray.streaming.state.backend; -/** - * Backend Types. - */ +/** Backend Types. */ public enum BackendType { - /** - * Saving the state values in memory - */ + /** Saving the state values in memory */ MEMORY; - /** - * get the enum from input string value, ignoring the case - */ + /** get the enum from input string value, ignoring the case */ public static BackendType getEnum(String value) { for (BackendType v : values()) { if (v.name().equalsIgnoreCase(value)) { diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/KeyStateBackend.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/KeyStateBackend.java index 4dea4511c..bbf7619e8 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/KeyStateBackend.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/KeyStateBackend.java @@ -40,16 +40,13 @@ public class KeyStateBackend extends AbstractKeyStateBackend { protected final KeyGroup keyGroup; public KeyStateBackend( - int numberOfKeyGroups, KeyGroup keyGroup, - AbstractStateBackend abstractStateBackend) { + int numberOfKeyGroups, KeyGroup keyGroup, AbstractStateBackend abstractStateBackend) { super(abstractStateBackend); this.numberOfKeyGroups = numberOfKeyGroups; this.keyGroup = keyGroup; } - /** - * get new value state proxy - */ + /** get new value state proxy */ protected ValueStateStoreManagerProxy newValueStateProxy( ValueStateDescriptor stateDescriptor) { return new ValueStateStoreManagerProxy<>(this, stateDescriptor); @@ -66,9 +63,7 @@ public class KeyStateBackend extends AbstractKeyStateBackend { } } - /** - * get new list state proxy - */ + /** get new list state proxy */ protected ListStateStoreManagerProxy newListStateProxy( ListStateDescriptor stateDescriptor) { return new ListStateStoreManagerProxy<>(this, stateDescriptor); @@ -86,9 +81,7 @@ public class KeyStateBackend extends AbstractKeyStateBackend { } } - /** - * get map state proxy - */ + /** get map state proxy */ protected MapStateStoreManagerProxy newMapStateProxy( MapStateDescriptor stateDescriptor) { return new MapStateStoreManagerProxy<>(this, stateDescriptor); @@ -108,8 +101,8 @@ public class KeyStateBackend extends AbstractKeyStateBackend { @Override public void setCurrentKey(Object currentKey) { - super.keyGroupIndex = KeyGroupAssignment - .assignKeyGroupIndexForKey(currentKey, numberOfKeyGroups); + super.keyGroupIndex = + KeyGroupAssignment.assignKeyGroupIndexForKey(currentKey, numberOfKeyGroups); super.currentKey = currentKey; } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/OperatorStateBackend.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/OperatorStateBackend.java index 68bdac581..c41ad7887 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/OperatorStateBackend.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/OperatorStateBackend.java @@ -23,9 +23,7 @@ import io.ray.streaming.state.keystate.state.ListState; import io.ray.streaming.state.keystate.state.impl.OperatorStateImpl; import io.ray.streaming.state.keystate.state.proxy.ListStateStoreManagerProxy; -/** - * OperatorState manager for getting split or union list state. - */ +/** OperatorState manager for getting split or union list state. */ public class OperatorStateBackend extends AbstractKeyStateBackend { public OperatorStateBackend(AbstractStateBackend backend) { @@ -42,9 +40,7 @@ public class OperatorStateBackend extends AbstractKeyStateBackend { return new ListStateStoreManagerProxy<>(this, stateDescriptor); } - /** - * get spitted List for different operator instance. - */ + /** get spitted List for different operator instance. */ public ListState getSplitListState(ListStateDescriptor stateDescriptor) { String desc = stateDescriptor.getIdentify(); if (listManagerProxyHashMap.containsKey(desc)) { @@ -58,9 +54,7 @@ public class OperatorStateBackend extends AbstractKeyStateBackend { } } - /** - * get a union List for different operator instance. - */ + /** get a union List for different operator instance. */ public ListState getUnionListState(ListStateDescriptor stateDescriptor) { String desc = stateDescriptor.getIdentify(); if (listManagerProxyHashMap.containsKey(desc)) { diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/StateBackendBuilder.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/StateBackendBuilder.java index 0921c5a26..91c3e2bea 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/StateBackendBuilder.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/StateBackendBuilder.java @@ -22,14 +22,11 @@ import io.ray.streaming.state.backend.impl.MemoryStateBackend; import io.ray.streaming.state.config.ConfigKey; import java.util.Map; -/** - * State Backend Builder. - */ +/** State Backend Builder. */ public class StateBackendBuilder { private static AbstractStateBackend getStateBackend( - Map config, - BackendType type) { + Map config, BackendType type) { switch (type) { case MEMORY: return new MemoryStateBackend(config); diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/StateStrategy.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/StateStrategy.java index 40d231cf8..cf81bac36 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/StateStrategy.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/StateStrategy.java @@ -18,21 +18,14 @@ package io.ray.streaming.state.backend; -/** - * This class describe State Saving Model. - */ +/** This class describe State Saving Model. */ public enum StateStrategy { - /** - * save two version together in case of rollback. - */ + /** save two version together in case of rollback. */ DUAL_VERSION, - /** - * for storage supporting mvcc, we save only current version. - */ + /** for storage supporting mvcc, we save only current version. */ SINGLE_VERSION; - public static StateStrategy getEnum(String value) { for (StateStrategy v : values()) { if (v.name().equalsIgnoreCase(value)) { diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/impl/MemoryStateBackend.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/impl/MemoryStateBackend.java index 67eeebf33..241f00dc2 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/impl/MemoryStateBackend.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/backend/impl/MemoryStateBackend.java @@ -26,9 +26,7 @@ import io.ray.streaming.state.store.impl.MemoryKeyMapStore; import io.ray.streaming.state.store.impl.MemoryKeyValueStore; import java.util.Map; -/** - * MemoryStateBackend. Supporting memory store. - */ +/** MemoryStateBackend. Supporting memory store. */ public class MemoryStateBackend extends AbstractStateBackend { public MemoryStateBackend(Map config) { @@ -50,5 +48,4 @@ public class MemoryStateBackend extends AbstractStateBackend { String tableName, KeyMapStoreSerializer keyMapStoreSerializer) { return new MemoryKeyMapStore<>(); } - } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/config/ConfigHelper.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/config/ConfigHelper.java index cc5ad02cc..92a80da57 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/config/ConfigHelper.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/config/ConfigHelper.java @@ -20,9 +20,7 @@ package io.ray.streaming.state.config; import java.util.Map; -/** - * Config Helper figure out the config info. Todo replace this to config module. - */ +/** Config Helper figure out the config info. Todo replace this to config module. */ public class ConfigHelper { public static int getIntegerOrDefault(Map config, String configKey, int defaultValue) { @@ -40,5 +38,4 @@ public class ConfigHelper { return defaultValue; } } - } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/config/ConfigKey.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/config/ConfigKey.java index 4ce263856..52704f02f 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/config/ConfigKey.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/config/ConfigKey.java @@ -20,15 +20,12 @@ package io.ray.streaming.state.config; import java.util.Map; -/** - * state config keys. Todo replace this to config module. - */ +/** state config keys. Todo replace this to config module. */ public final class ConfigKey { - /** - * backend - */ + /** backend */ public static final String STATE_BACKEND_TYPE = "state.backend.type"; + public static final String STATE_TABLE_NAME = "state.table.name"; public static final String STATE_STRATEGY_MODE = "state.strategy.mode"; public static final String NUMBER_PER_CHECKPOINT = "number.per.checkpoint"; diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/KeyGroup.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/KeyGroup.java index c3d6ad6ef..d5132365c 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/KeyGroup.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/KeyGroup.java @@ -43,9 +43,7 @@ public class KeyGroup implements Serializable { Preconditions.checkArgument(size() >= 0, "overflow detected."); } - /** - * @return The number of key-group in the range - */ + /** Returns The number of key-group in the range */ public int size() { return 1 + endIndex - startIndex; } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/KeyGroupAssignment.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/KeyGroupAssignment.java index 5e49259e2..10f99c0b6 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/KeyGroupAssignment.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/KeyGroupAssignment.java @@ -24,9 +24,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -/** - * This class defines key-group assignment algorithm。 - */ +/** This class defines key-group assignment algorithm。 */ public final class KeyGroupAssignment { /** @@ -37,8 +35,10 @@ public final class KeyGroupAssignment { * @param index index of the operator instance. */ public static KeyGroup getKeyGroup(int maxParallelism, int parallelism, int index) { - Preconditions.checkArgument(maxParallelism >= parallelism, - "Maximum parallelism (%s) must not be smaller than parallelism(%s)", maxParallelism, + Preconditions.checkArgument( + maxParallelism >= parallelism, + "Maximum parallelism (%s) must not be smaller than parallelism(%s)", + maxParallelism, parallelism); int start = index == 0 ? 0 : ((index * maxParallelism - 1) / parallelism) + 1; @@ -50,25 +50,24 @@ public final class KeyGroupAssignment { * Assigning the key to a key-group index. * * @param key the key to assign. - * @param maxParallelism the maximum parallelism. - * @return the key-group index to which the given key is assigned. + * @param maxParallelism the maximum parallelism. Returns the key-group index to which the given + * key is assigned. */ public static int assignKeyGroupIndexForKey(Object key, int maxParallelism) { return Math.abs(key.hashCode() % maxParallelism); } public static Map> computeKeyGroupToTask( - int maxParallelism, - List targetTasks) { + int maxParallelism, List targetTasks) { Map> keyGroupToTask = new ConcurrentHashMap<>(); for (int index = 0; index < targetTasks.size(); index++) { KeyGroup taskKeyGroup = getKeyGroup(maxParallelism, targetTasks.size(), index); - for (int groupId = taskKeyGroup.getStartIndex(); groupId <= taskKeyGroup.getEndIndex(); + for (int groupId = taskKeyGroup.getStartIndex(); + groupId <= taskKeyGroup.getEndIndex(); groupId++) { keyGroupToTask.put(groupId, ImmutableList.of(targetTasks.get(index))); } } return keyGroupToTask; } - } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/desc/AbstractStateDescriptor.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/desc/AbstractStateDescriptor.java index c85decf95..950f922c0 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/desc/AbstractStateDescriptor.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/desc/AbstractStateDescriptor.java @@ -21,9 +21,7 @@ package io.ray.streaming.state.keystate.desc; import com.google.common.base.Preconditions; import io.ray.streaming.state.keystate.state.State; -/** - * This class defines basic data structures of StateDescriptor. - */ +/** This class defines basic data structures of StateDescriptor. */ public abstract class AbstractStateDescriptor { private final String name; @@ -65,24 +63,26 @@ public abstract class AbstractStateDescriptor { @Override public String toString() { - return "AbstractStateDescriptor{" + "tableName='" + tableName + '\'' + ", name='" + name + '\'' - + ", type=" + type + '}'; + return "AbstractStateDescriptor{" + + "tableName='" + + tableName + + '\'' + + ", name='" + + name + + '\'' + + ", type=" + + type + + '}'; } public enum StateType { - /** - * value state - */ + /** value state */ VALUE, - /** - * list state - */ + /** list state */ LIST, - /** - * map state - */ + /** map state */ MAP } } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/desc/ListStateDescriptor.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/desc/ListStateDescriptor.java index aa2100350..62c7a249d 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/desc/ListStateDescriptor.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/desc/ListStateDescriptor.java @@ -18,15 +18,11 @@ package io.ray.streaming.state.keystate.desc; - import static io.ray.streaming.state.config.ConfigKey.DELIMITER; - import io.ray.streaming.state.keystate.state.ListState; -/** - * ListStateDescriptor. - */ +/** ListStateDescriptor. */ public class ListStateDescriptor extends AbstractStateDescriptor, T> { private final boolean isOperatorList; @@ -43,8 +39,7 @@ public class ListStateDescriptor extends AbstractStateDescriptor } public static ListStateDescriptor build( - String name, Class type, - boolean isOperatorList) { + String name, Class type, boolean isOperatorList) { return new ListStateDescriptor<>(name, type, isOperatorList); } @@ -76,9 +71,8 @@ public class ListStateDescriptor extends AbstractStateDescriptor @Override public String getIdentify() { if (isOperatorList) { - return String - .format("%s%s%d%s%d", super.getIdentify(), DELIMITER, partitionNum, DELIMITER, - index); + return String.format( + "%s%s%d%s%d", super.getIdentify(), DELIMITER, partitionNum, DELIMITER, index); } else { return super.getIdentify(); } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/desc/MapStateDescriptor.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/desc/MapStateDescriptor.java index f1803b03e..b6c3c3829 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/desc/MapStateDescriptor.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/desc/MapStateDescriptor.java @@ -21,9 +21,7 @@ package io.ray.streaming.state.keystate.desc; import io.ray.streaming.state.keystate.state.MapState; import java.util.Map; -/** - * MapStateDescriptor. - */ +/** MapStateDescriptor. */ public class MapStateDescriptor extends AbstractStateDescriptor, Map> { public MapStateDescriptor(String name, Class keyType, Class valueType) { @@ -32,8 +30,7 @@ public class MapStateDescriptor extends AbstractStateDescriptor MapStateDescriptor build( - String name, Class keyType, - Class valueType) { + String name, Class keyType, Class valueType) { return new MapStateDescriptor<>(name, keyType, valueType); } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/desc/ValueStateDescriptor.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/desc/ValueStateDescriptor.java index e067ba71d..79c7df474 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/desc/ValueStateDescriptor.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/desc/ValueStateDescriptor.java @@ -20,9 +20,7 @@ package io.ray.streaming.state.keystate.desc; import io.ray.streaming.state.keystate.state.ValueState; -/** - * ValueStateDescriptor. - */ +/** ValueStateDescriptor. */ public class ValueStateDescriptor extends AbstractStateDescriptor, T> { private final T defaultValue; diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/ListState.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/ListState.java index d5efa6670..966ee7ab2 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/ListState.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/ListState.java @@ -20,9 +20,7 @@ package io.ray.streaming.state.keystate.state; import java.util.List; -/** - * ListState interface. - */ +/** ListState interface. */ public interface ListState extends UnaryState> { /** diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/MapState.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/MapState.java index 663da8f71..933081af5 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/MapState.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/MapState.java @@ -22,16 +22,13 @@ import java.util.Iterator; import java.util.Map; import java.util.Map.Entry; -/** - * MapState interface. - */ +/** MapState interface. */ public interface MapState extends UnaryState> { /** * Returns the current value associated with the given key. * - * @param key The key of the mapping - * @return The value of the mapping with the given key + * @param key The key of the mapping Returns The value of the mapping with the given key */ V get(K key); @@ -67,8 +64,8 @@ public interface MapState extends UnaryState> { /** * Returns whether there exists the given mapping. * - * @param key The key of the mapping - * @return True if there exists a mapping whose key equals to the given key + * @param key The key of the mapping Returns True if there exists a mapping whose key equals to + * the given key */ default boolean contains(K key) { return get().containsKey(key); @@ -77,7 +74,7 @@ public interface MapState extends UnaryState> { /** * Returns all the mappings in the state * - * @return An iterable view of all the key-value pairs in the state. + *

Returns An iterable view of all the key-value pairs in the state. */ default Iterable> entries() { return get().entrySet(); @@ -86,7 +83,7 @@ public interface MapState extends UnaryState> { /** * Returns all the keys in the state * - * @return An iterable view of all the keys in the state. + *

Returns An iterable view of all the keys in the state. */ default Iterable keys() { return get().keySet(); @@ -95,7 +92,7 @@ public interface MapState extends UnaryState> { /** * Returns all the values in the state. * - * @return An iterable view of all the values in the state. + *

Returns An iterable view of all the values in the state. */ default Iterable values() { return get().values(); @@ -104,7 +101,7 @@ public interface MapState extends UnaryState> { /** * Iterates over all the mappings in the state. * - * @return An iterator over all the mappings in the state + *

Returns An iterator over all the mappings in the state */ default Iterator> iterator() { return get().entrySet().iterator(); diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/State.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/State.java index 6b18c62d8..3e8704693 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/State.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/State.java @@ -18,13 +18,9 @@ package io.ray.streaming.state.keystate.state; -/** - * State interface. - */ +/** State interface. */ public interface State { - /** - * set current key of the state - */ + /** set current key of the state */ void setCurrentKey(Object currentKey); } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/UnaryState.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/UnaryState.java index a67196d20..5c250b594 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/UnaryState.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/UnaryState.java @@ -18,15 +18,13 @@ package io.ray.streaming.state.keystate.state; -/** - * one value per state. - */ +/** one value per state. */ public interface UnaryState extends State { /** * get the value in state * - * @return the value in state + *

Returns the value in state */ O get(); -} \ No newline at end of file +} diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/ValueState.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/ValueState.java index e26b23b8a..3c1b06377 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/ValueState.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/ValueState.java @@ -18,9 +18,7 @@ package io.ray.streaming.state.keystate.state; -/** - * ValueState interface. - */ +/** ValueState interface. */ public interface ValueState extends UnaryState { /** diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/ListStateImpl.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/ListStateImpl.java index a56da03ec..7dce9642d 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/ListStateImpl.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/ListStateImpl.java @@ -24,9 +24,7 @@ import io.ray.streaming.state.keystate.state.ListState; import java.util.ArrayList; import java.util.List; -/** - * ListState implementation. - */ +/** ListState implementation. */ public class ListStateImpl implements ListState { private final StateHelper> helper; diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/MapStateImpl.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/MapStateImpl.java index 0d0729133..d31977dcb 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/MapStateImpl.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/MapStateImpl.java @@ -24,9 +24,7 @@ import io.ray.streaming.state.keystate.state.MapState; import java.util.HashMap; import java.util.Map; -/** - * MapState implementation. - */ +/** MapState implementation. */ public class MapStateImpl implements MapState { private final StateHelper> helper; @@ -82,9 +80,7 @@ public class MapStateImpl implements MapState { helper.put(map); } - /** - * set current key of the state - */ + /** set current key of the state */ @Override public void setCurrentKey(Object currentKey) { helper.setCurrentKey(currentKey); diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/OperatorStateImpl.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/OperatorStateImpl.java index b5a50415b..ee298705a 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/OperatorStateImpl.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/OperatorStateImpl.java @@ -20,7 +20,6 @@ package io.ray.streaming.state.keystate.state.impl; import static io.ray.streaming.state.config.ConfigKey.DELIMITER; - import com.google.common.base.Preconditions; import io.ray.streaming.state.PartitionRecord; import io.ray.streaming.state.backend.AbstractKeyStateBackend; @@ -73,8 +72,8 @@ public class OperatorStateImpl implements ListState { int partitionNum = -1; int index = 0; while (true) { - List> list = helper.getBackend() - .get(descriptor, getKey(descriptor.getIdentify(), index)); + List> list = + helper.getBackend().get(descriptor, getKey(descriptor.getIdentify(), index)); if (list != null && !list.isEmpty()) { partitionNum = list.get(0).getPartitionID(); allList.addAll(list); diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/StateHelper.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/StateHelper.java index 23511081a..9a6fe0f04 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/StateHelper.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/StateHelper.java @@ -22,9 +22,7 @@ import com.google.common.base.Preconditions; import io.ray.streaming.state.backend.AbstractKeyStateBackend; import io.ray.streaming.state.keystate.desc.AbstractStateDescriptor; -/** - * State Helper Class. - */ +/** State Helper Class. */ public class StateHelper { private final AbstractKeyStateBackend backend; diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/ValueStateImpl.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/ValueStateImpl.java index 50a72ec65..8343b2140 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/ValueStateImpl.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/impl/ValueStateImpl.java @@ -22,9 +22,7 @@ import io.ray.streaming.state.backend.KeyStateBackend; import io.ray.streaming.state.keystate.desc.ValueStateDescriptor; import io.ray.streaming.state.keystate.state.ValueState; -/** - * ValueState implementation. - */ +/** ValueState implementation. */ public class ValueStateImpl implements ValueState { private final StateHelper helper; @@ -48,9 +46,7 @@ public class ValueStateImpl implements ValueState { } } - /** - * set current key of the state - */ + /** set current key of the state */ @Override public void setCurrentKey(Object currentKey) { helper.setCurrentKey(currentKey); diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/proxy/ListStateStoreManagerProxy.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/proxy/ListStateStoreManagerProxy.java index 82136da27..ac10f9d11 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/proxy/ListStateStoreManagerProxy.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/proxy/ListStateStoreManagerProxy.java @@ -27,17 +27,14 @@ import io.ray.streaming.state.keystate.state.impl.OperatorStateImpl; import io.ray.streaming.state.strategy.StateStoreManagerProxy; import java.util.List; -/** - * This class defines ListState Wrapper, connecting state and backend. - */ -public class ListStateStoreManagerProxy extends StateStoreManagerProxy> implements - KeyValueState> { +/** This class defines ListState Wrapper, connecting state and backend. */ +public class ListStateStoreManagerProxy extends StateStoreManagerProxy> + implements KeyValueState> { private final ListState listState; public ListStateStoreManagerProxy( - AbstractKeyStateBackend keyStateBackend, - ListStateDescriptor stateDescriptor) { + AbstractKeyStateBackend keyStateBackend, ListStateDescriptor stateDescriptor) { super(keyStateBackend, stateDescriptor); if (stateDescriptor.isOperatorList()) { this.listState = new OperatorStateImpl<>(stateDescriptor, keyStateBackend); diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/proxy/MapStateStoreManagerProxy.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/proxy/MapStateStoreManagerProxy.java index 7542a5bf3..ab8727392 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/proxy/MapStateStoreManagerProxy.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/proxy/MapStateStoreManagerProxy.java @@ -26,17 +26,14 @@ import io.ray.streaming.state.keystate.state.impl.MapStateImpl; import io.ray.streaming.state.strategy.StateStoreManagerProxy; import java.util.Map; -/** - * This class defines MapState Wrapper, connecting state and backend. - */ -public class MapStateStoreManagerProxy extends StateStoreManagerProxy> implements - KeyValueState> { +/** This class defines MapState Wrapper, connecting state and backend. */ +public class MapStateStoreManagerProxy extends StateStoreManagerProxy> + implements KeyValueState> { private final MapStateImpl mapState; public MapStateStoreManagerProxy( - KeyStateBackend keyStateBackend, - MapStateDescriptor stateDescriptor) { + KeyStateBackend keyStateBackend, MapStateDescriptor stateDescriptor) { super(keyStateBackend, stateDescriptor); this.mapState = new MapStateImpl<>(stateDescriptor, keyStateBackend); } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/proxy/ValueStateStoreManagerProxy.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/proxy/ValueStateStoreManagerProxy.java index 13e68c8dc..ed68bcdc5 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/proxy/ValueStateStoreManagerProxy.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/keystate/state/proxy/ValueStateStoreManagerProxy.java @@ -25,17 +25,14 @@ import io.ray.streaming.state.keystate.state.ValueState; import io.ray.streaming.state.keystate.state.impl.ValueStateImpl; import io.ray.streaming.state.strategy.StateStoreManagerProxy; -/** - * This class defines ValueState Wrapper, connecting state and backend. - */ -public class ValueStateStoreManagerProxy extends StateStoreManagerProxy implements - KeyValueState { +/** This class defines ValueState Wrapper, connecting state and backend. */ +public class ValueStateStoreManagerProxy extends StateStoreManagerProxy + implements KeyValueState { private final ValueStateImpl valueState; public ValueStateStoreManagerProxy( - KeyStateBackend keyStateBackend, - ValueStateDescriptor stateDescriptor) { + KeyStateBackend keyStateBackend, ValueStateDescriptor stateDescriptor) { super(keyStateBackend, stateDescriptor); this.valueState = new ValueStateImpl<>(stateDescriptor, keyStateBackend); } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/KeyMapStoreSerializer.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/KeyMapStoreSerializer.java index 620913a4c..314289afb 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/KeyMapStoreSerializer.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/KeyMapStoreSerializer.java @@ -18,9 +18,7 @@ package io.ray.streaming.state.serialization; -/** - * Key Map Serialization and Deserialization. - */ +/** Key Map Serialization and Deserialization. */ public interface KeyMapStoreSerializer { byte[] serializeKey(K key); @@ -32,5 +30,4 @@ public interface KeyMapStoreSerializer { byte[] serializeUValue(T uv); T deserializeUValue(byte[] uvArray); - } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/KeyValueStoreSerialization.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/KeyValueStoreSerialization.java index dc53ef7ba..b08613562 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/KeyValueStoreSerialization.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/KeyValueStoreSerialization.java @@ -18,9 +18,7 @@ package io.ray.streaming.state.serialization; -/** - * Key Value Serialization and Deserialization. - */ +/** Key Value Serialization and Deserialization. */ public interface KeyValueStoreSerialization { byte[] serializeKey(K key); diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/Serializer.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/Serializer.java index ec66ede98..eaae2a0cc 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/Serializer.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/Serializer.java @@ -20,9 +20,7 @@ package io.ray.streaming.state.serialization; import org.nustaq.serialization.FSTConfiguration; -/** - * fst wrapper. - */ +/** fst wrapper. */ public class Serializer { private static final ThreadLocal conf = diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/impl/AbstractSerialization.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/impl/AbstractSerialization.java index cc8daee6e..c2d2ec05b 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/impl/AbstractSerialization.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/impl/AbstractSerialization.java @@ -24,9 +24,7 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * AbstractSerialization. Generate row key. - */ +/** AbstractSerialization. Generate row key. */ public abstract class AbstractSerialization { private static final Logger LOG = LoggerFactory.getLogger(AbstractSerialization.class); diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/impl/DefaultKeyMapStoreSerializer.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/impl/DefaultKeyMapStoreSerializer.java index 99f51764b..fa6dfb6b1 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/impl/DefaultKeyMapStoreSerializer.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/impl/DefaultKeyMapStoreSerializer.java @@ -21,11 +21,9 @@ package io.ray.streaming.state.serialization.impl; import io.ray.streaming.state.serialization.KeyMapStoreSerializer; import io.ray.streaming.state.serialization.Serializer; -/** - * Default Key Map Serialization and Deserialization. - */ -public class DefaultKeyMapStoreSerializer extends AbstractSerialization implements - KeyMapStoreSerializer { +/** Default Key Map Serialization and Deserialization. */ +public class DefaultKeyMapStoreSerializer extends AbstractSerialization + implements KeyMapStoreSerializer { @Override public byte[] serializeKey(K key) { diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/impl/DefaultKeyValueStoreSerialization.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/impl/DefaultKeyValueStoreSerialization.java index ff0a843f6..9a9c7f637 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/impl/DefaultKeyValueStoreSerialization.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/serialization/impl/DefaultKeyValueStoreSerialization.java @@ -21,11 +21,9 @@ package io.ray.streaming.state.serialization.impl; import io.ray.streaming.state.serialization.KeyValueStoreSerialization; import io.ray.streaming.state.serialization.Serializer; -/** - * KV Store Serialization and Deserialization. - */ -public class DefaultKeyValueStoreSerialization extends AbstractSerialization implements - KeyValueStoreSerialization { +/** KV Store Serialization and Deserialization. */ +public class DefaultKeyValueStoreSerialization extends AbstractSerialization + implements KeyValueStoreSerialization { @Override public byte[] serializeKey(K key) { diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/store/KeyMapStore.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/store/KeyMapStore.java index 0393ee294..21981f815 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/store/KeyMapStore.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/store/KeyMapStore.java @@ -21,18 +21,12 @@ package io.ray.streaming.state.store; import java.io.IOException; import java.util.Map; -/** - * Key Map Store interface. - */ +/** Key Map Store interface. */ public interface KeyMapStore extends KeyValueStore> { - /** - * put sub key value into the store incrementally. - */ + /** put sub key value into the store incrementally. */ void put(K key, S subKey, T value) throws IOException; - /** - * get subValue from store. - */ + /** get subValue from store. */ T get(K key, S subKey) throws IOException; } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/store/KeyValueStore.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/store/KeyValueStore.java index 506baef34..2e04f9d0d 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/store/KeyValueStore.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/store/KeyValueStore.java @@ -21,38 +21,24 @@ package io.ray.streaming.state.store; import java.io.IOException; import java.io.Serializable; -/** - * Key Value Store interface. - */ +/** Key Value Store interface. */ public interface KeyValueStore extends Serializable { - /** - * put key value into store. - */ + /** put key value into store. */ void put(K key, V value) throws IOException; - /** - * get value from store. - */ + /** get value from store. */ V get(K key) throws IOException; - /** - * remove key in the store. - */ + /** remove key in the store. */ void remove(K key) throws IOException; - /** - * flush to disk. - */ + /** flush to disk. */ void flush() throws IOException; - /** - * clear all cache. - */ + /** clear all cache. */ void clearCache(); - /** - * close the store. - */ + /** close the store. */ void close() throws IOException; } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/store/impl/MemoryKeyMapStore.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/store/impl/MemoryKeyMapStore.java index aad605c5a..5182aa22b 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/store/impl/MemoryKeyMapStore.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/store/impl/MemoryKeyMapStore.java @@ -24,9 +24,7 @@ import java.io.IOException; import java.util.HashMap; import java.util.Map; -/** - * Memory Key Map Store. - */ +/** Memory Key Map Store. */ public class MemoryKeyMapStore implements KeyMapStore { private Map> memoryStore; @@ -70,14 +68,10 @@ public class MemoryKeyMapStore implements KeyMapStore { } @Override - public void flush() throws IOException { - - } + public void flush() throws IOException {} @Override - public void clearCache() { - - } + public void clearCache() {} @Override public void close() throws IOException { @@ -85,5 +79,4 @@ public class MemoryKeyMapStore implements KeyMapStore { memoryStore.clear(); } } - } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/store/impl/MemoryKeyValueStore.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/store/impl/MemoryKeyValueStore.java index d5882832c..4e4113ff1 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/store/impl/MemoryKeyValueStore.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/store/impl/MemoryKeyValueStore.java @@ -23,9 +23,7 @@ import io.ray.streaming.state.store.KeyValueStore; import java.io.IOException; import java.util.Map; -/** - * Memory Key Value Store. - */ +/** Memory Key Value Store. */ public class MemoryKeyValueStore implements KeyValueStore { private Map memoryStore; @@ -50,14 +48,10 @@ public class MemoryKeyValueStore implements KeyValueStore { } @Override - public void flush() throws IOException { - - } + public void flush() throws IOException {} @Override - public void clearCache() { - - } + public void clearCache() {} @Override public void close() throws IOException { diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/strategy/AbstractStateStoreManager.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/strategy/AbstractStateStoreManager.java index 41873966b..3ee70cfd4 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/strategy/AbstractStateStoreManager.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/strategy/AbstractStateStoreManager.java @@ -33,20 +33,15 @@ import java.util.concurrent.ConcurrentHashMap; */ public abstract class AbstractStateStoreManager implements StateStoreManager { - /** - * read-write - */ + /** read-write */ protected Map> frontStore = new ConcurrentHashMap<>(); - /** - * remote-storage - */ + /** remote-storage */ protected KeyValueStore> kvStore; - /** - * read-only - */ + /** read-only */ protected Map> middleStore = new ConcurrentHashMap<>(); + protected int keyGroupIndex = -1; public AbstractStateStoreManager(KeyValueStore> backStore) { diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/strategy/DualStateStoreManager.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/strategy/DualStateStoreManager.java index f3f31b1b2..b096781d2 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/strategy/DualStateStoreManager.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/strategy/DualStateStoreManager.java @@ -31,9 +31,7 @@ import java.util.Map.Entry; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** - * This class define the checkpoint store strategy, which saves two-version data once. - */ +/** This class define the checkpoint store strategy, which saves two-version data once. */ public class DualStateStoreManager extends AbstractStateStoreManager { private static final Logger LOG = LoggerFactory.getLogger(DualStateStoreManager.class); @@ -68,14 +66,12 @@ public class DualStateStoreManager extends AbstractStateStoreManager { byte[] value = entry.getValue(); /** - * 2 is specific key in kv store and indicates that new value - * should be stored with this key after overwriting old value in key 1. i.e. + * 2 is specific key in kv store and indicates that new value should be stored with this key + * after overwriting old value in key 1. i.e. * - * -2 -1 1 2 - * k1 6 5 a b - * k2 9 7 d e + *

-2 -1 1 2 k1 6 5 a b k2 9 7 d e * - * k1's value for checkpoint 5 is a, and b for checkpoint 6. + *

k1's value for checkpoint 5 is a, and b for checkpoint 6. */ Map remoteData = super.kvStore.get(key); if (remoteData == null || remoteData.size() == 0) { @@ -85,12 +81,12 @@ public class DualStateStoreManager extends AbstractStateStoreManager { } else { long oldBatchId = Longs.fromByteArray(remoteData.get(-2L)); if (oldBatchId < checkpointId) { - //move the old data + // move the old data remoteData.put(1L, remoteData.get(2L)); remoteData.put(-1L, remoteData.get(-2L)); } - //put the new data here + // put the new data here remoteData.put(2L, value); remoteData.put(-2L, Longs.toByteArray(checkpointId)); } @@ -119,7 +115,7 @@ public class DualStateStoreManager extends AbstractStateStoreManager { return storageRecord.getValue(); } - //get from not commit cp info + // get from not commit cp info List checkpointIds = new ArrayList<>(middleStore.keySet()); Collections.sort(checkpointIds); for (int i = checkpointIds.size() - 1; i >= 0; i--) { diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/strategy/MVStateStoreManager.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/strategy/MVStateStoreManager.java index 82208c234..cbb59a171 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/strategy/MVStateStoreManager.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/strategy/MVStateStoreManager.java @@ -28,9 +28,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -/** - * This class define the multi-version store strategy, which leverages external storage's mvcc. - */ +/** This class define the multi-version store strategy, which leverages external storage's mvcc. */ public class MVStateStoreManager extends AbstractStateStoreManager { public MVStateStoreManager(KeyValueStore> kvStore) { @@ -153,5 +151,4 @@ public class MVStateStoreManager extends AbstractStateStoreManager { this.middleStore.remove(commitBatchId); } } - } diff --git a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/strategy/StateStoreManagerProxy.java b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/strategy/StateStoreManagerProxy.java index 268af0b0d..eb4b2a3a6 100644 --- a/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/strategy/StateStoreManagerProxy.java +++ b/streaming/java/streaming-state/src/main/java/io/ray/streaming/state/strategy/StateStoreManagerProxy.java @@ -27,22 +27,20 @@ import java.util.Map; /** * This class support ITransactionState. - *

- * Based on the given StorageMode, different implementation instance of the AbstractStateStrategy + * + *

Based on the given StorageMode, different implementation instance of the AbstractStateStrategy * class will be created. All method calls will be delegated to the strategy instance. */ - public abstract class StateStoreManagerProxy implements StateStoreManager { protected final AbstractStateStoreManager stateStrategy; private final AbstractKeyStateBackend keyStateBackend; public StateStoreManagerProxy( - AbstractKeyStateBackend keyStateBackend, - AbstractStateDescriptor stateDescriptor) { + AbstractKeyStateBackend keyStateBackend, AbstractStateDescriptor stateDescriptor) { this.keyStateBackend = keyStateBackend; - KeyValueStore> backStorage = keyStateBackend - .getBackStorage(stateDescriptor); + KeyValueStore> backStorage = + keyStateBackend.getBackStorage(stateDescriptor); StateStrategy stateStrategy = keyStateBackend.getStateStrategy(); switch (stateStrategy) { case DUAL_VERSION: @@ -65,17 +63,13 @@ public abstract class StateStoreManagerProxy implements StateStoreManager { this.stateStrategy.finish(checkpointId); } - /** - * The commit can be used in another thread to reach async state commit. - */ + /** The commit can be used in another thread to reach async state commit. */ @Override public void commit(long checkpointId) { this.stateStrategy.commit(checkpointId); } - /** - * The ackCommit must be called after commit in the same thread. - */ + /** The ackCommit must be called after commit in the same thread. */ @Override public void ackCommit(long checkpointId, long timeStamp) { this.stateStrategy.ackCommit(checkpointId); diff --git a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/backend/KeyStateBackendTest.java b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/backend/KeyStateBackendTest.java index 514b120e7..c690aecf5 100644 --- a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/backend/KeyStateBackendTest.java +++ b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/backend/KeyStateBackendTest.java @@ -37,8 +37,8 @@ public class KeyStateBackendTest { public void testGetValueState() { keyStateBackend.setCheckpointId(1L); - ValueStateDescriptor valueStateDescriptor = ValueStateDescriptor - .build("value", String.class, null); + ValueStateDescriptor valueStateDescriptor = + ValueStateDescriptor.build("value", String.class, null); valueStateDescriptor.setTableName("kepler_hlg_ut"); ValueState valueState = keyStateBackend.getValueState(valueStateDescriptor); @@ -114,13 +114,12 @@ public class KeyStateBackendTest { Assert.assertEquals(valueState.get(), "jack"); valueState.setCurrentKey("9"); Assert.assertNull(valueState.get()); - } public void testGetListState() { keyStateBackend.setCheckpointId(1l); - ListStateDescriptor listStateDescriptor = ListStateDescriptor - .build("list", String.class); + ListStateDescriptor listStateDescriptor = + ListStateDescriptor.build("list", String.class); listStateDescriptor.setTableName("kepler_hlg_ut"); ListState listState = keyStateBackend.getListState(listStateDescriptor); @@ -207,9 +206,8 @@ public class KeyStateBackendTest { public void testGetMapState() { keyStateBackend.setCheckpointId(1l); - MapStateDescriptor mapStateDescriptor = MapStateDescriptor - .build("map", String.class, - String.class); + MapStateDescriptor mapStateDescriptor = + MapStateDescriptor.build("map", String.class, String.class); mapStateDescriptor.setTableName("kepler_hlg_ut"); MapState mapState = keyStateBackend.getMapState(mapStateDescriptor); @@ -223,7 +221,7 @@ public class KeyStateBackendTest { Assert.assertEquals(mapState.get("hello3"), null); mapState.setCurrentKey("2"); - //Assert.assertEquals(mapState.iterator(), (new HashMap())); + // Assert.assertEquals(mapState.iterator(), (new HashMap())); mapState.setCurrentKey("2"); mapState.put("eagle", "eagle-1"); @@ -303,7 +301,6 @@ public class KeyStateBackendTest { Assert.assertNull(mapState.get("6666")); } - @Test public void testMem() { stateBackend = StateBackendBuilder.buildStateBackend(new HashMap<>()); @@ -312,5 +309,4 @@ public class KeyStateBackendTest { testGetListState(); testGetMapState(); } - } diff --git a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/impl/DefaultKeyMapStoreSerializationTest.java b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/impl/DefaultKeyMapStoreSerializationTest.java index 94716a107..0f64ea126 100644 --- a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/impl/DefaultKeyMapStoreSerializationTest.java +++ b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/impl/DefaultKeyMapStoreSerializationTest.java @@ -56,5 +56,4 @@ public class DefaultKeyMapStoreSerializationTest { byte[] result = this.defaultKMapStoreSerDe.serializeUValue(value); Assert.assertEquals(value, this.defaultKMapStoreSerDe.deserializeUValue(result)); } - } diff --git a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/impl/MemoryKeyMapStoreTest.java b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/impl/MemoryKeyMapStoreTest.java index 4f9e5f20b..f16103e50 100644 --- a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/impl/MemoryKeyMapStoreTest.java +++ b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/impl/MemoryKeyMapStoreTest.java @@ -58,7 +58,6 @@ public class MemoryKeyMapStoreTest { Assert.assertNotEquals(IKeyMapStore.get("hello"), map); Assert.assertEquals(IKeyMapStore.get("hello"), map2); - } catch (IOException e) { e.printStackTrace(); } diff --git a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/KeyGroupAssignmentTest.java b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/KeyGroupAssignmentTest.java index 31aa81004..22d5d36d5 100644 --- a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/KeyGroupAssignmentTest.java +++ b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/KeyGroupAssignmentTest.java @@ -20,12 +20,10 @@ package io.ray.streaming.state.keystate; import static org.testng.Assert.assertEquals; - import org.testng.annotations.Test; public class KeyGroupAssignmentTest { - @Test public void testComputeKeyGroupRangeForOperatorIndex() throws Exception { KeyGroup keyGroup = KeyGroupAssignment.getKeyGroup(4096, 1, 0); @@ -53,5 +51,4 @@ public class KeyGroupAssignmentTest { assertEquals(keyGroup3.getEndIndex(), 4095); assertEquals(keyGroup3.size(), 1365); } - } diff --git a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/desc/ListStateDescriptorTest.java b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/desc/ListStateDescriptorTest.java index 7daed7d0b..4475109b6 100644 --- a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/desc/ListStateDescriptorTest.java +++ b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/desc/ListStateDescriptorTest.java @@ -26,8 +26,8 @@ public class ListStateDescriptorTest { @Test public void test() { - ListStateDescriptor descriptor = ListStateDescriptor - .build("lsdTest", Integer.class, true); + ListStateDescriptor descriptor = + ListStateDescriptor.build("lsdTest", Integer.class, true); descriptor.setTableName("table"); Assert.assertTrue(descriptor.isOperatorList()); @@ -37,7 +37,8 @@ public class ListStateDescriptorTest { Assert.assertEquals(descriptor.getPartitionNumber(), 3); Assert.assertEquals(descriptor.getIndex(), 0); - Assert.assertEquals(descriptor.getIdentify(), + Assert.assertEquals( + descriptor.getIdentify(), "lsdTest" + ConfigKey.DELIMITER + "3" + ConfigKey.DELIMITER + "0"); } } diff --git a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/desc/MapStateDescriptorTest.java b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/desc/MapStateDescriptorTest.java index 1c6de6a67..18a1844a7 100644 --- a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/desc/MapStateDescriptorTest.java +++ b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/desc/MapStateDescriptorTest.java @@ -25,13 +25,11 @@ public class MapStateDescriptorTest { @Test public void test() { - MapStateDescriptor descriptor = MapStateDescriptor - .build("msdTest", String.class, - Integer.class); + MapStateDescriptor descriptor = + MapStateDescriptor.build("msdTest", String.class, Integer.class); descriptor.setTableName("table"); Assert.assertEquals(descriptor.getTableName(), "table"); Assert.assertEquals(descriptor.getName(), "msdTest"); } - } diff --git a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/desc/ValueStateDescriptorTest.java b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/desc/ValueStateDescriptorTest.java index 9ee939f89..626e897f2 100644 --- a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/desc/ValueStateDescriptorTest.java +++ b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/desc/ValueStateDescriptorTest.java @@ -25,8 +25,8 @@ public class ValueStateDescriptorTest { @Test public void test() { - ValueStateDescriptor descriptor = ValueStateDescriptor - .build("vsdTest", Integer.class, 0); + ValueStateDescriptor descriptor = + ValueStateDescriptor.build("vsdTest", Integer.class, 0); Assert.assertEquals(descriptor.getDefaultValue().intValue(), 0); } } diff --git a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/impl/ListStateImplTest.java b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/impl/ListStateImplTest.java index 049499507..0323ca7bc 100644 --- a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/impl/ListStateImplTest.java +++ b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/impl/ListStateImplTest.java @@ -36,10 +36,10 @@ public class ListStateImplTest { @BeforeClass public void setUp() throws Exception { - keyStateBackend = new KeyStateBackend(1, new KeyGroup(1, 2), - new MemoryStateBackend(new HashMap<>())); - ListStateDescriptor descriptor = ListStateDescriptor - .build("ListStateImplTest", Integer.class); + keyStateBackend = + new KeyStateBackend(1, new KeyGroup(1, 2), new MemoryStateBackend(new HashMap<>())); + ListStateDescriptor descriptor = + ListStateDescriptor.build("ListStateImplTest", Integer.class); descriptor.setTableName("table"); listState = (ListStateImpl) keyStateBackend.getListState(descriptor); @@ -72,7 +72,6 @@ public class ListStateImplTest { Assert.assertEquals(listState.get(), Arrays.asList(5, 6)); } - @Test(dependsOnMethods = {"testAddGet"}) public void testUpdate() throws Exception { Assert.assertEquals(listState.get(), Arrays.asList(5, 6)); diff --git a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/impl/MapStateImplTest.java b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/impl/MapStateImplTest.java index 5b86ad4a5..77d6e00b0 100644 --- a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/impl/MapStateImplTest.java +++ b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/impl/MapStateImplTest.java @@ -36,11 +36,10 @@ public class MapStateImplTest { @BeforeClass public void setUp() throws Exception { - keyStateBackend = new KeyStateBackend(1, new KeyGroup(1, 2), - new MemoryStateBackend(new HashMap<>())); - MapStateDescriptor descriptor = MapStateDescriptor - .build("MapStateImplTest", Integer.class, - String.class); + keyStateBackend = + new KeyStateBackend(1, new KeyGroup(1, 2), new MemoryStateBackend(new HashMap<>())); + MapStateDescriptor descriptor = + MapStateDescriptor.build("MapStateImplTest", Integer.class, String.class); descriptor.setTableName("table"); mapState = (MapStateImpl) keyStateBackend.getMapState(descriptor); } @@ -88,7 +87,6 @@ public class MapStateImplTest { Assert.assertEquals(mapState.get().size(), 0); } - @Test public void testFailover() throws Exception { keyStateBackend.setContext(1L, 1); diff --git a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/impl/OperatorStateImplTest.java b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/impl/OperatorStateImplTest.java index d3d3297b4..4639abf0a 100644 --- a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/impl/OperatorStateImplTest.java +++ b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/impl/OperatorStateImplTest.java @@ -41,9 +41,9 @@ public class OperatorStateImplTest { operatorStateBackend = new OperatorStateBackend(new MemoryStateBackend(config)); - descriptor = ListStateDescriptor - .build("OperatorStateImplTest" + System.currentTimeMillis(), Integer.class, - true); + descriptor = + ListStateDescriptor.build( + "OperatorStateImplTest" + System.currentTimeMillis(), Integer.class, true); descriptor.setPartitionNumber(1); descriptor.setIndex(0); descriptor.setTableName(table_name); diff --git a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/impl/ValueStateImplTest.java b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/impl/ValueStateImplTest.java index 3a45dd014..efc74be0a 100644 --- a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/impl/ValueStateImplTest.java +++ b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/impl/ValueStateImplTest.java @@ -34,11 +34,10 @@ public class ValueStateImplTest { @BeforeClass public void setUp() throws Exception { - keyStateBackend = new KeyStateBackend(1, new KeyGroup(1, 2), - new MemoryStateBackend(new HashMap<>())); - ValueStateDescriptor descriptor = ValueStateDescriptor - .build("ValueStateImplTest", String.class, - "hello"); + keyStateBackend = + new KeyStateBackend(1, new KeyGroup(1, 2), new MemoryStateBackend(new HashMap<>())); + ValueStateDescriptor descriptor = + ValueStateDescriptor.build("ValueStateImplTest", String.class, "hello"); descriptor.setTableName("table"); valueState = (ValueStateImpl) keyStateBackend.getValueState(descriptor); diff --git a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/proxy/MapStateStoreManagerTest.java b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/proxy/MapStateStoreManagerTest.java index 8b8ea73d6..fd99a2481 100644 --- a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/proxy/MapStateStoreManagerTest.java +++ b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/proxy/MapStateStoreManagerTest.java @@ -31,8 +31,8 @@ public class MapStateStoreManagerTest extends StateStoreManagerTest { @BeforeClass public void setUp() { - MapStateDescriptor descriptor = MapStateDescriptor - .build("map", String.class, Integer.class); + MapStateDescriptor descriptor = + MapStateDescriptor.build("map", String.class, Integer.class); descriptor.setTableName("tableName"); keyStateBackend.setContext(1L, "key"); facade = new MapStateStoreManagerProxy<>(keyStateBackend, descriptor); @@ -52,5 +52,4 @@ public class MapStateStoreManagerTest extends StateStoreManagerTest { facade.put("key2", map); Assert.assertEquals(facade.get("key2"), map); } - } diff --git a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/proxy/StateStoreManagerTest.java b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/proxy/StateStoreManagerTest.java index e6a0971fb..b919b5a09 100644 --- a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/proxy/StateStoreManagerTest.java +++ b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/proxy/StateStoreManagerTest.java @@ -25,6 +25,7 @@ import java.util.HashMap; public class StateStoreManagerTest { - protected KeyStateBackend keyStateBackend = new KeyStateBackend(1, new KeyGroup(1, 1), - new MemoryStateBackend(new HashMap())); + protected KeyStateBackend keyStateBackend = + new KeyStateBackend( + 1, new KeyGroup(1, 1), new MemoryStateBackend(new HashMap())); } diff --git a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/proxy/ValueStateStoreManagerTest.java b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/proxy/ValueStateStoreManagerTest.java index f3a8584b8..7802b05f5 100644 --- a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/proxy/ValueStateStoreManagerTest.java +++ b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/keystate/state/proxy/ValueStateStoreManagerTest.java @@ -30,8 +30,8 @@ public class ValueStateStoreManagerTest extends StateStoreManagerTest { @BeforeClass public void setUp() { - ValueStateDescriptor descriptor = ValueStateDescriptor - .build("value", Integer.class, 0); + ValueStateDescriptor descriptor = + ValueStateDescriptor.build("value", Integer.class, 0); descriptor.setTableName("tableName"); keyStateBackend.setContext(1L, "key"); proxy = new ValueStateStoreManagerProxy<>(keyStateBackend, descriptor); diff --git a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/strategy/DualStateStrategyTest.java b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/strategy/DualStateStrategyTest.java index 2d0c8173f..446632874 100644 --- a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/strategy/DualStateStrategyTest.java +++ b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/strategy/DualStateStrategyTest.java @@ -54,9 +54,8 @@ public class DualStateStrategyTest { } public void caseKV() { - ValueStateDescriptor valueStateDescriptor = ValueStateDescriptor - .build("VALUE-" + currentTime, - String.class, defaultValue); + ValueStateDescriptor valueStateDescriptor = + ValueStateDescriptor.build("VALUE-" + currentTime, String.class, defaultValue); valueStateDescriptor.setTableName(table); ValueState state = this.keyStateBackend.getValueState(valueStateDescriptor); @@ -116,7 +115,7 @@ public class DualStateStrategyTest { this.keyStateBackend.commit(2); this.keyStateBackend.ackCommit(2, 2); - //do rollback, all memory data is deleted. + // do rollback, all memory data is deleted. this.keyStateBackend.rollBack(1); this.keyStateBackend.setCheckpointId(1); state.setCurrentKey(("1")); @@ -159,9 +158,8 @@ public class DualStateStrategyTest { } public void caseKVGap() { - ValueStateDescriptor valueStateDescriptor = ValueStateDescriptor - .build("value2-" + currentTime, - String.class, defaultValue); + ValueStateDescriptor valueStateDescriptor = + ValueStateDescriptor.build("value2-" + currentTime, String.class, defaultValue); valueStateDescriptor.setTableName(table); ValueState state = this.keyStateBackend.getValueState(valueStateDescriptor); @@ -205,13 +203,11 @@ public class DualStateStrategyTest { this.keyStateBackend.setCheckpointId(11); this.keyStateBackend.rollBack(11); Assert.assertEquals(state.get(), "info"); - } public void caseKList() { - ListStateDescriptor listStateDescriptor = ListStateDescriptor - .build("LIST-" + currentTime, - Integer.class); + ListStateDescriptor listStateDescriptor = + ListStateDescriptor.build("LIST-" + currentTime, Integer.class); listStateDescriptor.setTableName(table); ListState state = this.keyStateBackend.getListState(listStateDescriptor); @@ -272,7 +268,7 @@ public class DualStateStrategyTest { this.keyStateBackend.commit(2); this.keyStateBackend.ackCommit(2, 2); - //do rollback, all memory data is deleted. + // do rollback, all memory data is deleted. this.keyStateBackend.rollBack(1); this.keyStateBackend.setCheckpointId(1); state.setCurrentKey(("1")); @@ -312,9 +308,8 @@ public class DualStateStrategyTest { } public void caseKMap() { - MapStateDescriptor mapStateDescriptor = MapStateDescriptor - .build("MAP-" + currentTime, - Integer.class, Integer.class); + MapStateDescriptor mapStateDescriptor = + MapStateDescriptor.build("MAP-" + currentTime, Integer.class, Integer.class); mapStateDescriptor.setTableName(table); MapState state = this.keyStateBackend.getMapState(mapStateDescriptor); @@ -378,7 +373,7 @@ public class DualStateStrategyTest { this.keyStateBackend.commit(2); this.keyStateBackend.ackCommit(2, 2); - //do rollback, memory data is deleted. + // do rollback, memory data is deleted. this.keyStateBackend.rollBack(1); this.keyStateBackend.setCheckpointId(1); state.setCurrentKey(("1")); @@ -428,8 +423,8 @@ public class DualStateStrategyTest { @Test public void testMem() { config.put(ConfigKey.STATE_BACKEND_TYPE, BackendType.MEMORY.name()); - this.keyStateBackend = new KeyStateBackend(10, new KeyGroup(1, 3), - StateBackendBuilder.buildStateBackend(config)); + this.keyStateBackend = + new KeyStateBackend(10, new KeyGroup(1, 3), StateBackendBuilder.buildStateBackend(config)); caseKV(); caseKVGap(); caseKList(); diff --git a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/strategy/MVStateStrategyTest.java b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/strategy/MVStateStrategyTest.java index 6b7316595..79c156664 100644 --- a/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/strategy/MVStateStrategyTest.java +++ b/streaming/java/streaming-state/src/test/java/io/ray/streaming/state/strategy/MVStateStrategyTest.java @@ -52,9 +52,8 @@ public class MVStateStrategyTest { public void caseKV() { - ValueStateDescriptor valueStateDescriptor = ValueStateDescriptor - .build("mvint-" + currentTime, - String.class, ""); + ValueStateDescriptor valueStateDescriptor = + ValueStateDescriptor.build("mvint-" + currentTime, String.class, ""); valueStateDescriptor.setTableName(table); ValueState state = this.keyStateBackend.getValueState(valueStateDescriptor); @@ -170,9 +169,8 @@ public class MVStateStrategyTest { } public void caseKList() { - ListStateDescriptor listStateDescriptor = ListStateDescriptor - .build("mvlist-" + currentTime, - Integer.class); + ListStateDescriptor listStateDescriptor = + ListStateDescriptor.build("mvlist-" + currentTime, Integer.class); listStateDescriptor.setTableName(table); ListState state = this.keyStateBackend.getListState(listStateDescriptor); @@ -275,9 +273,8 @@ public class MVStateStrategyTest { } public void caseKMap() { - MapStateDescriptor mapStateDescriptor = MapStateDescriptor - .build("mvmap-" + currentTime, - Integer.class, Integer.class); + MapStateDescriptor mapStateDescriptor = + MapStateDescriptor.build("mvmap-" + currentTime, Integer.class, Integer.class); mapStateDescriptor.setTableName(table); MapState state = this.keyStateBackend.getMapState(mapStateDescriptor); @@ -394,15 +391,13 @@ public class MVStateStrategyTest { Assert.assertNull(state.get(7)); } - @Test public void testMemMV() { config.put(ConfigKey.STATE_BACKEND_TYPE, BackendType.MEMORY.name()); - this.keyStateBackend = new KeyStateBackend(10, new KeyGroup(1, 3), - StateBackendBuilder.buildStateBackend(config)); + this.keyStateBackend = + new KeyStateBackend(10, new KeyGroup(1, 3), StateBackendBuilder.buildStateBackend(config)); caseKV(); caseKList(); caseKMap(); } } - diff --git a/streaming/java/test.sh b/streaming/java/test.sh index f0db1cf49..166457be5 100755 --- a/streaming/java/test.sh +++ b/streaming/java/test.sh @@ -7,6 +7,14 @@ set -x ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +pushd "$ROOT_DIR" + echo "Check java code format." + # check google java style + mvn -T16 spotless:check + # check naming and others + mvn -T16 checkstyle:check +popd + echo "build ray streaming" bazel build //streaming/java:all @@ -18,9 +26,6 @@ if [ -n "${symbols_conflict}" ]; then exit 1 fi -echo "Linting Java code with checkstyle." -bazel test //streaming/java:all --test_tag_filters="checkstyle" --build_tests_only - echo "Running streaming tests." java -cp "$ROOT_DIR"/../../bazel-bin/streaming/java/all_streaming_tests_deploy.jar\ org.testng.TestNG -d /tmp/ray_streaming_java_test_output "$ROOT_DIR"/testng.xml || @@ -62,7 +67,7 @@ cd "$ROOT_DIR"/../../java echo "build ray maven deps" bazel build gen_maven_deps echo "maven install ray" -mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN clean install -DskipTests +mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN clean install -DskipTests -Dcheckstyle.skip cd "$ROOT_DIR" echo "maven install ray streaming" -mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN clean install -DskipTests +mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN clean install -DskipTests -Dcheckstyle.skip