mirror of
https://github.com/wassname/ray.git
synced 2026-06-27 23:39:37 +08:00
[dist] Fix travis deploy for java dist (#9768)
This commit is contained in:
chaokunyang
+44
-26
@@ -1,5 +1,6 @@
|
||||
load("//bazel:ray.bzl", "define_java_module")
|
||||
load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
|
||||
load("//bazel:ray.bzl", "native_java_binary")
|
||||
load("//bazel:ray.bzl", "native_java_library")
|
||||
load("@rules_proto_grpc//java:defs.bzl", "java_proto_compile")
|
||||
|
||||
exports_files([
|
||||
@@ -143,43 +144,38 @@ filegroup(
|
||||
],
|
||||
)
|
||||
|
||||
# plasma_store_server binary should be located in jar root path
|
||||
# and keep in sync with mvn resource
|
||||
copy_file(
|
||||
name = "cp_plasma_store_server",
|
||||
src = "//:plasma_store_server",
|
||||
out = "plasma_store_server",
|
||||
)
|
||||
native_java_binary("runtime", "raylet", "//:raylet")
|
||||
|
||||
# `//:redis-server`'s full path is `external/com_github_antirez_redis/redis-server`,
|
||||
# This rule removes the prefix, and only keeps `redis-server`.
|
||||
copy_file(
|
||||
name = "redis-server-without-prefix",
|
||||
src = "//:redis-server",
|
||||
out = "redis-server",
|
||||
)
|
||||
native_java_binary("runtime", "plasma_store_server", "//:plasma_store_server")
|
||||
|
||||
native_java_binary("runtime", "redis-server", "//:redis-server")
|
||||
|
||||
native_java_binary("runtime", "gcs_server", "//:gcs_server")
|
||||
|
||||
native_java_binary("runtime", "libray_redis_module.so", "//:libray_redis_module.so")
|
||||
|
||||
native_java_library("runtime", "core_worker_library_java", "//:libcore_worker_library_java.so")
|
||||
|
||||
filegroup(
|
||||
name = "java_native_deps",
|
||||
srcs = [
|
||||
":cp_plasma_store_server",
|
||||
":redis-server-without-prefix",
|
||||
"//:core_worker_library_java",
|
||||
"//:gcs_server",
|
||||
"//:libray_redis_module.so",
|
||||
"//:raylet",
|
||||
":core_worker_library_java",
|
||||
":gcs_server",
|
||||
":libray_redis_module.so",
|
||||
":plasma_store_server",
|
||||
":raylet",
|
||||
":redis-server",
|
||||
],
|
||||
)
|
||||
|
||||
# Generates the dependencies needed by maven.
|
||||
genrule(
|
||||
name = "gen_maven_deps",
|
||||
name = "cp_java_generated",
|
||||
srcs = [
|
||||
":all_java_proto",
|
||||
":java_native_deps",
|
||||
":copy_pom_file",
|
||||
],
|
||||
outs = ["gen_maven_deps.out"],
|
||||
outs = ["cp_java_generated.out"],
|
||||
cmd = """
|
||||
WORK_DIR="$$(pwd)"
|
||||
# Copy protobuf-generated files.
|
||||
@@ -187,8 +183,30 @@ genrule(
|
||||
for f in $(locations //java:all_java_proto); do
|
||||
unzip "$$f" -x META-INF/MANIFEST.MF -d "$$WORK_DIR/java/runtime/src/main/java"
|
||||
done
|
||||
# Copy native dependecies.
|
||||
NATIVE_DEPS_DIR="$$WORK_DIR/java/runtime/native_dependencies/"
|
||||
date > $@
|
||||
""",
|
||||
local = 1,
|
||||
tags = ["no-cache"],
|
||||
)
|
||||
|
||||
# Generates the dependencies needed by maven.
|
||||
genrule(
|
||||
name = "gen_maven_deps",
|
||||
srcs = [
|
||||
":cp_java_generated",
|
||||
":java_native_deps",
|
||||
],
|
||||
outs = ["gen_maven_deps.out"],
|
||||
cmd = """
|
||||
WORK_DIR="$${PWD}"
|
||||
# Copy native dependencies.
|
||||
OS_NAME=""
|
||||
case "$${OSTYPE}" in
|
||||
linux*) OS_NAME="linux";;
|
||||
darwin*) OS_NAME="darwin";;
|
||||
*) echo "$${OSTYPE} is not supported currently"; exit 1;;
|
||||
esac
|
||||
NATIVE_DEPS_DIR="$$WORK_DIR/java/runtime/native_dependencies/native/$$OS_NAME"
|
||||
rm -rf "$$NATIVE_DEPS_DIR"
|
||||
mkdir -p "$$NATIVE_DEPS_DIR"
|
||||
for f in $(locations //java:java_native_deps); do
|
||||
|
||||
Executable
+168
@@ -0,0 +1,168 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -x
|
||||
|
||||
# Cause the script to exit if a single command fails.
|
||||
set -e
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)"
|
||||
WORKSPACE_DIR="${ROOT_DIR}/.."
|
||||
JAVA_DIRS_PATH=('java' 'streaming/java')
|
||||
RAY_JAVA_MODULES=('api' 'runtime')
|
||||
RAY_STREAMING_JAVA_MODULES=('streaming-api' 'streaming-runtime' 'streaming-state')
|
||||
JAR_BASE_DIR="$WORKSPACE_DIR"/.jar
|
||||
mkdir -p "$JAR_BASE_DIR"
|
||||
cd "$WORKSPACE_DIR/java"
|
||||
# ray jar version, ex: 0.1-SNAPSHORT
|
||||
version=$(python -c "import xml.etree.ElementTree as ET; r = ET.parse('pom.xml').getroot(); print(r.find(r.tag.replace('project', 'version')).text);" | tail -n 1)
|
||||
cd -
|
||||
|
||||
build_jars() {
|
||||
local platform="$1"
|
||||
local bazel_build="${2:-true}"
|
||||
echo "bazel_build $bazel_build"
|
||||
echo "Start building jar for $platform"
|
||||
local JAR_DIR="$JAR_BASE_DIR/$platform"
|
||||
mkdir -p "$JAR_DIR"
|
||||
for p in "${JAVA_DIRS_PATH[@]}"; do
|
||||
cd "$WORKSPACE_DIR/$p"
|
||||
if [[ $bazel_build == "true" ]]; then
|
||||
echo "Starting building java native dependencies for $p"
|
||||
bazel build gen_maven_deps
|
||||
echo "Finished building java native dependencies for $p"
|
||||
fi
|
||||
echo "Start building jars for $p"
|
||||
mvn -T16 clean package install -Dmaven.test.skip=true -Dcheckstyle.skip
|
||||
mvn -T16 source:jar -Dmaven.test.skip=true -Dcheckstyle.skip
|
||||
echo "Finished building jars for $p"
|
||||
done
|
||||
copy_jars "$JAR_DIR"
|
||||
# ray runtime jar and streaming runtime jar are in a dir specifed by maven-jar-plugin
|
||||
cp -f "$WORKSPACE_DIR"/build/java/ray*.jar "$JAR_DIR"
|
||||
cp -f "$WORKSPACE_DIR"/streaming/build/java/streaming*.jar "$JAR_DIR"
|
||||
echo "Finished building jar for $platform"
|
||||
}
|
||||
|
||||
copy_jars() {
|
||||
local JAR_DIR="$1"
|
||||
echo "Copy to dir $JAR_DIR"
|
||||
for module in "${RAY_JAVA_MODULES[@]}"; do
|
||||
cp -f "$WORKSPACE_DIR"/java/"$module"/target/*jar "$JAR_DIR"
|
||||
done
|
||||
for module in "${RAY_STREAMING_JAVA_MODULES[@]}"; do
|
||||
cp -f "$WORKSPACE_DIR"/streaming/java/"$module"/target/*jar "$JAR_DIR"
|
||||
done
|
||||
# ray runtime jar and streaming runtime jar are in a dir specifed by maven-jar-plugin
|
||||
cp -f "$WORKSPACE_DIR"/build/java/ray*.jar "$JAR_DIR"
|
||||
cp -f "$WORKSPACE_DIR"/streaming/build/java/streaming*.jar "$JAR_DIR"
|
||||
}
|
||||
|
||||
# This function assuem all dependencies are installed already.
|
||||
build_jars_linux() {
|
||||
build_jars linux
|
||||
}
|
||||
|
||||
# This function assuem all dependencies are installed already.
|
||||
build_jars_darwin() {
|
||||
build_jars darwin
|
||||
}
|
||||
|
||||
build_jars_multiplatform() {
|
||||
if [[ "${TRAVIS_REPO_SLUG-}" != "ray-project/ray" || "${TRAVIS_PULL_REQUEST-}" != "false" ]]; then
|
||||
echo "Skip build multiplatform jars when this build is from a pull request or
|
||||
not a build for commit in ray-project/ray."
|
||||
return
|
||||
fi
|
||||
download_jars "ray-runtime-$version.jar" "streaming-runtime-$version.jar"
|
||||
prepare_native
|
||||
build_jars multiplatform false
|
||||
}
|
||||
|
||||
# Download darwin/windows ray-related jar from s3
|
||||
# This function assumes linux jars exist already.
|
||||
download_jars() {
|
||||
local wait_time=0
|
||||
local sleep_time_units=60
|
||||
|
||||
for f in "$@"; do
|
||||
for os in 'darwin' 'windows'; do
|
||||
if [[ "$os" == "windows" ]]; then
|
||||
break
|
||||
fi
|
||||
local url="https://ray-wheels.s3-us-west-2.amazonaws.com/jars/$TRAVIS_BRANCH/$TRAVIS_COMMIT/$os/$f"
|
||||
mkdir -p "$JAR_BASE_DIR/$os"
|
||||
local dest_file="$JAR_BASE_DIR/$os/$f"
|
||||
echo "Jar url: $url"
|
||||
echo "Jar dest_file: $dest_file"
|
||||
while true; do
|
||||
if ! wget -q "$url" -O "$dest_file">/dev/null; then
|
||||
echo "Waiting $url to be ready for $wait_time seconds..."
|
||||
sleep $sleep_time_units
|
||||
wait_time=$((wait_time + sleep_time_units))
|
||||
if [[ wait_time == $((60 * 120)) ]]; then
|
||||
echo "Download $url timeout"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo "Download $url to $dest_file succeed"
|
||||
break
|
||||
fi
|
||||
done
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
# prepare native binaries and libraries.
|
||||
prepare_native() {
|
||||
for os in 'darwin' 'linux'; do
|
||||
cd "$JAR_BASE_DIR/$os"
|
||||
jar xf "ray-runtime-$version.jar" "native/$os"
|
||||
local native_dir="$WORKSPACE_DIR/java/runtime/native_dependencies/native/$os"
|
||||
mkdir -p "$native_dir"
|
||||
rm -rf "$native_dir"
|
||||
mv "native/$os" "$native_dir"
|
||||
jar xf "streaming-runtime-$version.jar" "native/$os"
|
||||
local native_dir="$WORKSPACE_DIR/streaming/java/streaming-runtime/native_dependencies/native/$os"
|
||||
mkdir -p "$native_dir"
|
||||
rm -rf "$native_dir"
|
||||
mv "native/$os" "$native_dir"
|
||||
done
|
||||
}
|
||||
|
||||
# This function assume all multiplatform binaries are prepared already.
|
||||
deploy_jars() {
|
||||
if [ "${TRAVIS-}" = true ]; then
|
||||
if [[ "$TRAVIS_REPO_SLUG" != "ray-project/ray" ||
|
||||
"$TRAVIS_PULL_REQUEST" != "false" || "$TRAVIS_BRANCH" != "master" ]]; then
|
||||
echo "Skip deploying jars when this build is from a pull request or
|
||||
not a build for commit of master branch in ray-project/ray"
|
||||
return
|
||||
fi
|
||||
fi
|
||||
echo "Start deploying jars"
|
||||
cd "$WORKSPACE_DIR"/java
|
||||
mvn -T16 deploy -Dmaven.test.skip=true -Dcheckstyle.skip
|
||||
cd "$WORKSPACE_DIR"/streaming/java
|
||||
mvn -T16 deploy -Dmaven.test.skip=true -Dcheckstyle.skip
|
||||
echo "Finished deploying jars"
|
||||
}
|
||||
|
||||
case "$1" in
|
||||
linux) # build jars that only contains Linux binaries.
|
||||
build_jars_linux
|
||||
;;
|
||||
darwin) # build jars that only contains macos binaries.
|
||||
build_jars_darwin
|
||||
;;
|
||||
multiplatform) # downloading jars of multiple platforms and packaging them into one jar.
|
||||
build_jars_multiplatform
|
||||
;;
|
||||
deploy) # Deploy jars to maven repository.
|
||||
deploy_jars
|
||||
;;
|
||||
*)
|
||||
echo "Execute command $*"
|
||||
"$@"
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -16,6 +16,7 @@ import io.ray.runtime.runner.RunManager;
|
||||
import io.ray.runtime.task.NativeTaskExecutor;
|
||||
import io.ray.runtime.task.NativeTaskSubmitter;
|
||||
import io.ray.runtime.task.TaskExecutor;
|
||||
import io.ray.runtime.util.BinaryFileUtil;
|
||||
import io.ray.runtime.util.JniUtils;
|
||||
import java.io.File;
|
||||
import java.io.IOException;
|
||||
@@ -49,7 +50,7 @@ public final class RayNativeRuntime extends AbstractRayRuntime {
|
||||
rayConfig.setSessionDir(sessionDir);
|
||||
}
|
||||
|
||||
JniUtils.loadLibrary("core_worker_library_java", true);
|
||||
JniUtils.loadLibrary(BinaryFileUtil.CORE_WORKER_JAVA_LIBRARY, true);
|
||||
LOGGER.debug("Native libraries loaded.");
|
||||
// Reset library path at runtime.
|
||||
resetLibraryPath(rayConfig);
|
||||
|
||||
@@ -237,7 +237,7 @@ public class RunManager {
|
||||
}
|
||||
|
||||
// See `src/ray/gcs/gcs_server/gcs_server_main.cc` for the meaning of each parameter.
|
||||
final File gcsServerFile = BinaryFileUtil.getFile(
|
||||
final File gcsServerFile = BinaryFileUtil.getNativeFile(
|
||||
rayConfig.sessionDir, BinaryFileUtil.GCS_SERVER_BINARY_NAME);
|
||||
Preconditions.checkState(gcsServerFile.setExecutable(true));
|
||||
List<String> command = ImmutableList.of(
|
||||
@@ -254,7 +254,7 @@ public class RunManager {
|
||||
}
|
||||
|
||||
private String startRedisInstance(String ip, int port, String password, Integer shard) {
|
||||
final File redisServerFile = BinaryFileUtil.getFile(
|
||||
final File redisServerFile = BinaryFileUtil.getNativeFile(
|
||||
rayConfig.sessionDir, BinaryFileUtil.REDIS_SERVER_BINARY_NAME);
|
||||
Preconditions.checkState(redisServerFile.setExecutable(true));
|
||||
List<String> command = Lists.newArrayList(
|
||||
@@ -268,7 +268,7 @@ public class RunManager {
|
||||
"warning",
|
||||
"--loadmodule",
|
||||
// The redis module file.
|
||||
BinaryFileUtil.getFile(
|
||||
BinaryFileUtil.getNativeFile(
|
||||
rayConfig.sessionDir, BinaryFileUtil.REDIS_MODULE_LIBRARY_NAME).getAbsolutePath()
|
||||
);
|
||||
|
||||
@@ -305,7 +305,7 @@ public class RunManager {
|
||||
}
|
||||
|
||||
// See `src/ray/raylet/main.cc` for the meaning of each parameter.
|
||||
final File rayletFile = BinaryFileUtil.getFile(
|
||||
final File rayletFile = BinaryFileUtil.getNativeFile(
|
||||
rayConfig.sessionDir, BinaryFileUtil.RAYLET_BINARY_NAME);
|
||||
Preconditions.checkState(rayletFile.setExecutable(true));
|
||||
List<String> command = ImmutableList.of(
|
||||
@@ -373,7 +373,7 @@ public class RunManager {
|
||||
}
|
||||
|
||||
private void startObjectStore() {
|
||||
final File objectStoreFile = BinaryFileUtil.getFile(
|
||||
final File objectStoreFile = BinaryFileUtil.getNativeFile(
|
||||
rayConfig.sessionDir, BinaryFileUtil.PLASMA_STORE_SERVER_BINARY_NAME);
|
||||
Preconditions.checkState(objectStoreFile.setExecutable(true));
|
||||
List<String> command = ImmutableList.of(
|
||||
|
||||
@@ -9,6 +9,7 @@ import java.nio.channels.FileLock;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Paths;
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.apache.commons.lang3.SystemUtils;
|
||||
|
||||
public class BinaryFileUtil {
|
||||
public static final String REDIS_SERVER_BINARY_NAME = "redis-server";
|
||||
@@ -21,11 +22,10 @@ public class BinaryFileUtil {
|
||||
|
||||
public static final String REDIS_MODULE_LIBRARY_NAME = "libray_redis_module.so";
|
||||
|
||||
public static final String CORE_WORKER_JAVA_LIBRARY =
|
||||
System.mapLibraryName("core_worker_library_java");
|
||||
public static final String CORE_WORKER_JAVA_LIBRARY = "core_worker_library_java";
|
||||
|
||||
/**
|
||||
* Extract a resource file to <code>destDir</code>.
|
||||
* Extract a platform-native resource file to <code>destDir</code>.
|
||||
* Note that this a process-safe operation. If multi processes extract the file to same
|
||||
* directory concurrently, this operation will be protected by a file lock.
|
||||
*
|
||||
@@ -33,7 +33,7 @@ public class BinaryFileUtil {
|
||||
* @param fileName resource file name
|
||||
* @return extracted resource file
|
||||
*/
|
||||
public static File getFile(String destDir, String fileName) {
|
||||
public static File getNativeFile(String destDir, String fileName) {
|
||||
final File dir = new File(destDir);
|
||||
if (!dir.exists()) {
|
||||
try {
|
||||
@@ -45,17 +45,26 @@ public class BinaryFileUtil {
|
||||
String lockFilePath = destDir + File.separator + "file_lock";
|
||||
try (FileLock ignored = new RandomAccessFile(lockFilePath, "rw")
|
||||
.getChannel().lock()) {
|
||||
String resourceDir;
|
||||
if (SystemUtils.IS_OS_MAC) {
|
||||
resourceDir = "native/darwin/";
|
||||
} else if (SystemUtils.IS_OS_LINUX) {
|
||||
resourceDir = "native/linux/";
|
||||
} else {
|
||||
throw new UnsupportedOperationException("Unsupported os " + SystemUtils.OS_NAME);
|
||||
}
|
||||
String resourcePath = resourceDir + fileName;
|
||||
File file = new File(String.format("%s/%s", destDir, fileName));
|
||||
if (file.exists()) {
|
||||
return file;
|
||||
}
|
||||
|
||||
// File does not exist.
|
||||
try (InputStream is = BinaryFileUtil.class.getResourceAsStream("/" + fileName)) {
|
||||
Preconditions.checkNotNull(is, "{} doesn't exist.", fileName);
|
||||
try (InputStream is = BinaryFileUtil.class.getResourceAsStream("/" + resourcePath)) {
|
||||
Preconditions.checkNotNull(is, "{} doesn't exist.", resourcePath);
|
||||
Files.copy(is, Paths.get(file.getCanonicalPath()));
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException("Couldn't get temp file from resource " + fileName, e);
|
||||
throw new RuntimeException("Couldn't get temp file from resource " + resourcePath, e);
|
||||
}
|
||||
return file;
|
||||
} catch (IOException e) {
|
||||
|
||||
@@ -39,7 +39,7 @@ public class JniUtils {
|
||||
// Load native library.
|
||||
String fileName = System.mapLibraryName(libraryName);
|
||||
final String sessionDir = RayConfig.getInstance().sessionDir;
|
||||
final File file = BinaryFileUtil.getFile(sessionDir, fileName);
|
||||
final File file = BinaryFileUtil.getNativeFile(sessionDir, fileName);
|
||||
|
||||
if (exportSymbols) {
|
||||
// Expose library symbols using RTLD_GLOBAL which may be depended by other shared
|
||||
|
||||
Reference in New Issue
Block a user