From be2cbdf1306182007b904b5d976c0c318e6864e9 Mon Sep 17 00:00:00 2001 From: Hao Chen Date: Thu, 18 Apr 2019 16:53:25 +0800 Subject: [PATCH] [Bazel] Use rules_jvm_external to manage java dependencies (#4615) --- .gitignore | 5 --- WORKSPACE | 21 ++++++++++- bazel/BUILD.plasma | 4 +- build.sh | 1 - java/BUILD.bazel | 76 ++++++++++++++++--------------------- java/dependencies.bzl | 27 +++++++++++++ java/dependencies.yaml | 86 ------------------------------------------ java/repo.bzl | 18 --------- java/third_party/BUILD | 0 9 files changed, 81 insertions(+), 157 deletions(-) create mode 100644 java/dependencies.bzl delete mode 100644 java/dependencies.yaml delete mode 100644 java/repo.bzl delete mode 100644 java/third_party/BUILD diff --git a/.gitignore b/.gitignore index 079705a08..b26c1d6b4 100644 --- a/.gitignore +++ b/.gitignore @@ -146,11 +146,6 @@ java/**/.classpath java/**/.project java/runtime/native_dependencies/ -# Java third-party dependencies. -java/third_party/*/ -!java/third_party/workspace.bzl -!java/third_party/BUILD - # python virtual env venv diff --git a/WORKSPACE b/WORKSPACE index 86751bcfb..4ca8b3d13 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,9 +1,22 @@ workspace(name = "com_github_ray_project_ray") load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository", "new_git_repository") -load("//java:repo.bzl", "java_repositories") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -java_repositories() +RULES_JVM_EXTERNAL_TAG = "1.2" + +RULES_JVM_EXTERNAL_SHA = "e5c68b87f750309a79f59c2b69ead5c3221ffa54ff9496306937bfa1c9c8c86b" + +http_archive( + name = "rules_jvm_external", + sha256 = RULES_JVM_EXTERNAL_SHA, + strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, + url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, +) + +load("//java:dependencies.bzl", "gen_java_deps") + +gen_java_deps() git_repository( name = "com_github_checkstyle_java", @@ -11,6 +24,10 @@ git_repository( remote = "https://github.com/ruifangChen/checkstyle_java", ) +load("@com_github_checkstyle_java//:repo.bzl", "checkstyle_deps") + +checkstyle_deps() + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") git_repository( diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index 45dd170b2..a2b75b997 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -6,7 +6,7 @@ java_library( data = [":plasma_client_java"], visibility = ["//visibility:public"], deps = [ - "@org_slf4j_slf4j_api//jar", + "@maven//:org_slf4j_slf4j_api", ], ) @@ -17,7 +17,7 @@ java_binary( visibility = ["//visibility:public"], deps = [ ":org_apache_arrow_arrow_plasma", - "@junit_junit//jar", + "@maven//:junit_junit", ], ) diff --git a/build.sh b/build.sh index f7e027da8..05b4aa0f8 100755 --- a/build.sh +++ b/build.sh @@ -101,7 +101,6 @@ $PYTHON_EXECUTABLE -m pip install \ export PYTHON_BIN_PATH="$PYTHON_EXECUTABLE" if [ "$RAY_BUILD_JAVA" == "YES" ]; then - bazel run //java:bazel_deps -- generate -r $ROOT_DIR -s java/third_party/workspace.bzl -d java/dependencies.yaml bazel build //java:all --verbose_failures fi diff --git a/java/BUILD.bazel b/java/BUILD.bazel index d20847272..b2e78e552 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -7,12 +7,6 @@ exports_files([ "streaming/testng.xml", ]) -java_binary( - name = "bazel_deps", - main_class = "com.github.johnynek.bazel_deps.ParseProject", - runtime_deps = ["@bazel_deps//jar"], -) - java_import( name = "all_modules", jars = [ @@ -42,11 +36,11 @@ java_import( define_java_module( name = "api", deps = [ - "@org_slf4j_slf4j_log4j12//jar", - "@log4j_log4j//jar", - "@javax_xml_bind_jaxb_api//jar", - "@com_sun_xml_bind_jaxb_core//jar", - "@com_sun_xml_bind_jaxb_impl//jar", + "@maven//:org_slf4j_slf4j_log4j12", + "@maven//:log4j_log4j", + "@maven//:javax_xml_bind_jaxb_api", + "@maven//:com_sun_xml_bind_jaxb_core", + "@maven//:com_sun_xml_bind_jaxb_impl", ], ) @@ -62,28 +56,24 @@ define_java_module( test_deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", - "@org_apache_commons_commons_lang3//jar", - "@com_beust_jcommander//jar", - "@org_testng_testng//jar", - "@commons_io_commons_io//jar", + "@maven//:org_apache_commons_commons_lang3", + "@maven//:com_beust_jcommander", + "@maven//:org_testng_testng", + "@maven//:commons_io_commons_io", ], deps = [ ":org_ray_ray_api", "@plasma//:org_apache_arrow_arrow_plasma", - "@org_slf4j_slf4j_api//jar", - "@com_fasterxml_jackson_core_jackson_core//jar", - "@org_apache_commons_commons_pool2//jar", - "@org_javassist_javassist//jar", - "@org_objenesis_objenesis//jar", - "@com_typesafe_config//jar", - "@org_apache_commons_commons_lang3//jar", - "@de_ruedigermoeller_fst//jar", - "@com_github_davidmoten_flatbuffers_java//jar", - "@redis_clients_jedis//jar", - "@commons_io_commons_io//jar", - "@com_google_guava_guava//jar", - "@org_slf4j_slf4j_log4j12//jar", - "@org_ow2_asm_asm//jar", + "@maven//:com_github_davidmoten_flatbuffers_java", + "@maven//:com_google_guava_guava", + "@maven//:com_typesafe_config", + "@maven//:commons_io_commons_io", + "@maven//:de_ruedigermoeller_fst", + "@maven//:org_apache_commons_commons_lang3", + "@maven//:org_ow2_asm_asm", + "@maven//:org_slf4j_slf4j_api", + "@maven//:org_slf4j_slf4j_log4j12", + "@maven//:redis_clients_jedis", ], ) @@ -92,7 +82,7 @@ define_java_module( deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", - "@com_google_guava_guava//jar", + "@maven//:com_google_guava_guava", ], ) @@ -101,12 +91,12 @@ define_java_module( deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", - "@org_apache_commons_commons_lang3//jar", - "@org_slf4j_slf4j_api//jar", "@plasma//:org_apache_arrow_arrow_plasma", - "@org_testng_testng//jar", - "@com_google_guava_guava//jar", - "@commons_io_commons_io//jar", + "@maven//:org_apache_commons_commons_lang3", + "@maven//:org_slf4j_slf4j_api", + "@maven//:org_testng_testng", + "@maven//:com_google_guava_guava", + "@maven//:commons_io_commons_io", ], ) @@ -115,20 +105,20 @@ define_java_module( deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", - "@com_google_guava_guava//jar", - "@org_slf4j_slf4j_api//jar", - "@org_slf4j_slf4j_log4j12//jar", + "@maven//:com_google_guava_guava", + "@maven//:org_slf4j_slf4j_api", + "@maven//:org_slf4j_slf4j_log4j12", ], define_test_lib = True, test_deps = [ ":org_ray_ray_api", ":org_ray_ray_runtime", ":org_ray_ray_streaming", - "@com_beust_jcommander//jar", - "@com_google_guava_guava//jar", - "@org_slf4j_slf4j_api//jar", - "@org_slf4j_slf4j_log4j12//jar", - "@org_testng_testng//jar", + "@maven//:com_beust_jcommander", + "@maven//:com_google_guava_guava", + "@maven//:org_slf4j_slf4j_api", + "@maven//:org_slf4j_slf4j_log4j12", + "@maven//:org_testng_testng", ], ) diff --git a/java/dependencies.bzl b/java/dependencies.bzl new file mode 100644 index 000000000..d0178ba0f --- /dev/null +++ b/java/dependencies.bzl @@ -0,0 +1,27 @@ +load("@rules_jvm_external//:defs.bzl", "maven_install") + +def gen_java_deps(): + maven_install( + artifacts = [ + "com.beust:jcommander:1.72", + "com.github.davidmoten:flatbuffers-java:1.9.0.1", + "com.google.guava:guava:27.0.1-jre", + "com.puppycrawl.tools:checkstyle:8.15", + "com.sun.xml.bind:jaxb-core:2.3.0", + "com.sun.xml.bind:jaxb-impl:2.3.0", + "com.typesafe:config:1.3.2", + "commons-io:commons-io:2.5", + "de.ruedigermoeller:fst:2.47", + "javax.xml.bind:jaxb-api:2.3.0", + "org.apache.commons:commons-lang3:3.4", + "org.ow2.asm:asm:6.0", + "org.slf4j:slf4j-log4j12:1.7.25", + "org.testng:testng:6.9.9", + "redis.clients:jedis:2.8.0", + ], + repositories = [ + "https://repo1.maven.org/maven2", + ], + # Fetch srcjars. Defaults to False. + fetch_sources = True, + ) diff --git a/java/dependencies.yaml b/java/dependencies.yaml deleted file mode 100644 index d2f89f0f6..000000000 --- a/java/dependencies.yaml +++ /dev/null @@ -1,86 +0,0 @@ -options: - languages: [ "java" ] - licenses: [ "permissive" ] - namePrefix: "" - resolverType: "coursier" - resolvers: - - id: "mavencentral" - type: "default" - url: https://repo.maven.apache.org/maven2/ - thirdPartyDirectory: "java/third_party/" - transitivity: runtime_deps - versionConflictPolicy: highest - -dependencies: - com.puppycrawl.tools: - checkstyle: - lang: java - version: "8.15" - - de.ruedigermoeller: - fst: - lang: java - version: "2.47" - - org.ow2.asm: - asm: - lang: java - version: "6.0" - - com.github.davidmoten: - flatbuffers-java: - lang: java - version: "1.9.0.1" - - com.beust: - jcommander: - lang: java - version: "1.72" - - redis.clients: - jedis: - lang: java - version: "2.8.0" - - commons-io: - commons-io: - lang: java - version: "2.5" - - org.apache.commons: - commons-lang3: - lang: java - version: "3.4" - - com.google.guava: - guava: - lang: java - version: "27.0.1-jre" - - org.slf4j: - slf4j-log4j12: - lang: java - version: "1.7.25" - - com.typesafe: - config: - lang: java - version: "1.3.2" - - org.testng: - testng: - lang: java - version: "6.9.9" - - javax.xml.bind: - jaxb-api: - lang: java - version: "2.3.0" - - com.sun.xml.bind: - jaxb-core: - lang: java - version: "2.3.0" - jaxb-impl: - lang: java - version: "2.3.0" diff --git a/java/repo.bzl b/java/repo.bzl deleted file mode 100644 index a06c285f2..000000000 --- a/java/repo.bzl +++ /dev/null @@ -1,18 +0,0 @@ -load("//java/third_party:workspace.bzl", "maven_dependencies") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_jar") - -def create_if_needed(rule, name, **kwargs): - if name not in native.existing_rules(): - rule(name = name, **kwargs) - -def bazel_deps(): - create_if_needed( - http_jar, - name = "bazel_deps", - sha256 = "98b05c2826f2248f70e7356dc6c78bc52395904bb932fbb409a5abf5416e4292", - urls = ["https://github.com/oferb/startupos-binaries/releases/download/0.1.01/bazel_deps.jar"], - ) - -def java_repositories(): - maven_dependencies() - bazel_deps() diff --git a/java/third_party/BUILD b/java/third_party/BUILD deleted file mode 100644 index e69de29bb..000000000