Bazel move more shell to native rules (#9314)

Co-authored-by: Mehrdad <noreply@github.com>

BUILD.bazel

@@ -1,13 +1,14 @@
 # Bazel build
 # C/C++ documentation: https://docs.bazel.build/versions/master/be/c-cpp.html
 
+load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 load("@rules_proto//proto:defs.bzl", "proto_library")
 load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library", "cc_test")
 load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library")
 load("@com_github_grpc_grpc//bazel:cython_library.bzl", "pyx_library")
 load("@rules_proto_grpc//python:defs.bzl", "python_grpc_compile")
 load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library")
-load("//bazel:ray.bzl", "COPTS", "PYX_COPTS", "PYX_SRCS")
+load("//bazel:ray.bzl", "COPTS", "PYX_COPTS", "PYX_SRCS", "copy_to_workspace")
 
 config_setting(
     name = "windows_msvc-cl",
@@ -334,22 +335,20 @@ cc_binary(
     deps = [":plasma_client"],
 )
 
-genrule(
+copy_file(
     name = "copy_jni_h",
-    srcs = ["@bazel_tools//tools/jdk:jni_header"],
-    outs = ["jni.h"],
-    cmd = "cp -f $< $@",
+    src = "@bazel_tools//tools/jdk:jni_header",
+    out = "jni.h",
 )
 
-genrule(
+copy_file(
     name = "copy_jni_md_h",
-    srcs = select({
-        "@bazel_tools//src/conditions:windows": ["@bazel_tools//tools/jdk:jni_md_header-windows"],
-        "@bazel_tools//src/conditions:darwin": ["@bazel_tools//tools/jdk:jni_md_header-darwin"],
-        "//conditions:default": ["@bazel_tools//tools/jdk:jni_md_header-linux"],
+    src = select({
+        "@bazel_tools//src/conditions:windows": "@bazel_tools//tools/jdk:jni_md_header-windows",
+        "@bazel_tools//src/conditions:darwin": "@bazel_tools//tools/jdk:jni_md_header-darwin",
+        "//conditions:default": "@bazel_tools//tools/jdk:jni_md_header-linux",
     }),
-    outs = ["jni_md.h"],
-    cmd = "cp -f $< $@",
+    out = "jni_md.h",
 )
 
 genrule(
@@ -514,7 +513,11 @@ cc_library(
 cc_test(
     name = "gcs_pub_sub_test",
     srcs = ["src/ray/gcs/pubsub/test/gcs_pub_sub_test.cc"],
-    args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"],
+    args = [
+        "$(location redis-server)",
+        "$(location redis-cli)",
+        "$(location libray_redis_module.so)",
+    ],
     copts = COPTS,
     data = [
         "//:libray_redis_module.so",
@@ -720,7 +723,15 @@ cc_binary(
 cc_test(
     name = "core_worker_test",
     srcs = ["src/ray/core_worker/test/core_worker_test.cc"],
-    args = ["$(location //:plasma_store_server) $(location raylet) $(location mock_worker) $(location gcs_server) $(location redis-cli) $(location redis-server) $(location libray_redis_module.so)"],
+    args = [
+        "$(location //:plasma_store_server)",
+        "$(location raylet)",
+        "$(location mock_worker)",
+        "$(location gcs_server)",
+        "$(location redis-cli)",
+        "$(location redis-server)",
+        "$(location libray_redis_module.so)",
+    ],
     copts = COPTS,
     data = [
         "//:gcs_server",
@@ -991,7 +1002,11 @@ cc_test(
     srcs = [
         "src/ray/gcs/gcs_server/test/gcs_server_rpc_test.cc",
     ],
-    args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"],
+    args = [
+        "$(location redis-server)",
+        "$(location redis-cli)",
+        "$(location libray_redis_module.so)",
+    ],
     copts = COPTS,
     data = [
         "//:libray_redis_module.so",
@@ -1084,7 +1099,11 @@ cc_test(
     srcs = [
         "src/ray/gcs/gcs_server/test/redis_gcs_table_storage_test.cc",
     ],
-    args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"],
+    args = [
+        "$(location redis-server)",
+        "$(location redis-cli)",
+        "$(location libray_redis_module.so)",
+    ],
     copts = COPTS,
     data = [
         "//:libray_redis_module.so",
@@ -1159,7 +1178,11 @@ cc_test(
     srcs = [
         "src/ray/gcs/gcs_client/test/global_state_accessor_test.cc",
     ],
-    args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"],
+    args = [
+        "$(location redis-server)",
+        "$(location redis-cli)",
+        "$(location libray_redis_module.so)",
+    ],
     copts = COPTS,
     data = [
         "//:libray_redis_module.so",
@@ -1180,7 +1203,11 @@ cc_test(
     srcs = [
         "src/ray/gcs/gcs_client/test/service_based_gcs_client_test.cc",
     ],
-    args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"],
+    args = [
+        "$(location redis-server)",
+        "$(location redis-cli)",
+        "$(location libray_redis_module.so)",
+    ],
     copts = COPTS,
     data = [
         "//:libray_redis_module.so",
@@ -1430,7 +1457,11 @@ cc_library(
 cc_test(
     name = "redis_store_client_test",
     srcs = ["src/ray/gcs/store_client/test/redis_store_client_test.cc"],
-    args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"],
+    args = [
+        "$(location redis-server)",
+        "$(location redis-cli)",
+        "$(location libray_redis_module.so)",
+    ],
     copts = COPTS,
     data = [
         "//:libray_redis_module.so",
@@ -1485,7 +1516,11 @@ cc_library(
 cc_test(
     name = "redis_gcs_client_test",
     srcs = ["src/ray/gcs/test/redis_gcs_client_test.cc"],
-    args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"],
+    args = [
+        "$(location redis-server)",
+        "$(location redis-cli)",
+        "$(location libray_redis_module.so)",
+    ],
     copts = COPTS,
     data = [
         "//:libray_redis_module.so",
@@ -1501,7 +1536,11 @@ cc_test(
 cc_test(
     name = "redis_actor_info_accessor_test",
     srcs = ["src/ray/gcs/test/redis_actor_info_accessor_test.cc"],
-    args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"],
+    args = [
+        "$(location redis-server)",
+        "$(location redis-cli)",
+        "$(location libray_redis_module.so)",
+    ],
     copts = COPTS,
     data = [
         "//:libray_redis_module.so",
@@ -1518,7 +1557,11 @@ cc_test(
 cc_test(
     name = "redis_object_info_accessor_test",
     srcs = ["src/ray/gcs/test/redis_object_info_accessor_test.cc"],
-    args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"],
+    args = [
+        "$(location redis-server)",
+        "$(location redis-cli)",
+        "$(location libray_redis_module.so)",
+    ],
     copts = COPTS,
     data = [
         "//:libray_redis_module.so",
@@ -1535,7 +1578,11 @@ cc_test(
 cc_test(
     name = "subscription_executor_test",
     srcs = ["src/ray/gcs/test/subscription_executor_test.cc"],
-    args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"],
+    args = [
+        "$(location redis-server)",
+        "$(location redis-cli)",
+        "$(location libray_redis_module.so)",
+    ],
     copts = COPTS,
     data = [
         "//:libray_redis_module.so",
@@ -1552,7 +1599,11 @@ cc_test(
 cc_test(
     name = "redis_job_info_accessor_test",
     srcs = ["src/ray/gcs/test/redis_job_info_accessor_test.cc"],
-    args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"],
+    args = [
+        "$(location redis-server)",
+        "$(location redis-cli)",
+        "$(location libray_redis_module.so)",
+    ],
     copts = COPTS,
     data = [
         "//:libray_redis_module.so",
@@ -1569,7 +1620,11 @@ cc_test(
 cc_test(
     name = "redis_node_info_accessor_test",
     srcs = ["src/ray/gcs/test/redis_node_info_accessor_test.cc"],
-    args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"],
+    args = [
+        "$(location redis-server)",
+        "$(location redis-cli)",
+        "$(location libray_redis_module.so)",
+    ],
     copts = COPTS,
     data = [
         "//:libray_redis_module.so",
@@ -1586,7 +1641,11 @@ cc_test(
 cc_test(
     name = "asio_test",
     srcs = ["src/ray/gcs/test/asio_test.cc"],
-    args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"],
+    args = [
+        "$(location redis-server)",
+        "$(location redis-cli)",
+        "$(location libray_redis_module.so)",
+    ],
     copts = COPTS,
     data = [
         "//:libray_redis_module.so",
@@ -1822,60 +1881,90 @@ py_library(
     visibility = ["__subpackages__"],
 )
 
+copy_to_workspace(
+    name = "cp_raylet_so",
+    srcs = ["python/ray/_raylet.so"],
+    dstdir = "python/ray",
+)
+
+copy_to_workspace(
+    name = "cp_streaming",
+    srcs = ["python/ray/streaming/_streaming.so"],
+    dstdir = "python/ray/streaming",
+)
+
+copy_to_workspace(
+    name = "cp_all_py_proto",
+    srcs = [":all_py_proto"],
+    dstdir = "python/ray/core/generated",
+)
+
+copy_to_workspace(
+    name = "cp_redis",
+    srcs = [
+        ":redis-cli",
+        ":redis-server",
+    ],
+    dstdir = "python/ray/core/src/ray/thirdparty/redis/src",
+)
+
+copy_to_workspace(
+    name = "cp_libray_redis_module",
+    srcs = [":libray_redis_module.so"],
+    dstdir = "python/ray/core/src/ray/gcs/redis_module",
+)
+
+copy_to_workspace(
+    name = "cp_raylet",
+    srcs = [":raylet"],
+    dstdir = "python/ray/core/src/ray/raylet",
+)
+
+copy_to_workspace(
+    name = "cp_gcs_server",
+    srcs = [":gcs_server"],
+    dstdir = "python/ray/core/src/ray/gcs",
+)
+
+copy_to_workspace(
+    name = "cp_plasma_store_server",
+    srcs = [":plasma_store_server"],
+    dstdir = "python/ray/core/src/plasma",
+)
+
 genrule(
     name = "ray_pkg",
     srcs = [
-        "python/ray/_raylet.so",
-        "python/ray/streaming/_streaming.so",
-        "//:python_sources",
-        "//:all_py_proto",
-        "//:redis-server",
-        "//:redis-cli",
-        "//:libray_redis_module.so",
-        "//:raylet",
-        "//:gcs_server",
-        "//:plasma_store_server",
+        ":cp_raylet_so",
+        ":cp_streaming",
+        ":python_sources",
+        ":cp_all_py_proto",
+        ":cp_redis",
+        ":cp_libray_redis_module",
+        ":cp_raylet",
+        ":cp_gcs_server",
+        ":cp_plasma_store_server",
         "//streaming:copy_streaming_py_proto",
     ],
     outs = ["ray_pkg.out"],
     cmd = """
-        WORK_DIR="$$(pwd)" &&
-        RAYDIR="$$WORK_DIR/python/ray"
-        RAYLET="$(location python/ray/_raylet.so)"
-        RAYLET_TARGET_FILENAME="$${RAYLET##*/}" &&
         if [ "$${OSTYPE-}" = "msys" ]; then
-            # If Windows, we need to do things differently (e.g. use .pyd instead of .so)
-            RAYLET_TARGET_FILENAME="$${RAYLET_TARGET_FILENAME%.*}" &&
-            RAYLET_TARGET_FILENAME="$${RAYLET_TARGET_FILENAME}.pyd" &&
-            if [ -f "$${RAYLET%.*}.pdb" ]; then
-                cp -f "$${RAYLET%.*}.pdb" "$${RAYDIR}";
-            fi;
-        fi &&
-        cp -f "$${RAYLET}" "$${RAYDIR}/$${RAYLET_TARGET_FILENAME}" &&
-        cp -f $(location python/ray/streaming/_streaming.so) "$$WORK_DIR/python/ray/streaming" &&
-        mkdir -p "$$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/" &&
-        cp -f $(location //:redis-server) "$$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/" &&
-        cp -f $(location //:redis-cli) "$$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/" &&
-        mkdir -p "$$WORK_DIR/python/ray/core/src/ray/gcs/redis_module/" &&
-        cp -f $(locations //:libray_redis_module.so) "$$WORK_DIR/python/ray/core/src/ray/gcs/redis_module/" &&
-        cp -f $(location //:plasma_store_server) "$$WORK_DIR/python/ray/core/src/plasma/" &&
-        cp -f $(location //:raylet) "$$WORK_DIR/python/ray/core/src/ray/raylet/" &&
-        cp -f $(location //:gcs_server) "$$WORK_DIR/python/ray/core/src/ray/gcs/" &&
-        mkdir -p "$$WORK_DIR/python/ray/core/generated/ray/protocol/" &&
-        for f in $(locations //:all_py_proto); do
-            cp -f "$$f" "$$WORK_DIR/python/ray/core/generated/";
-        done &&
+            mv -f python/ray/_raylet.so python/ray/_raylet.pyd
+        fi
         # NOTE(hchen): Protobuf doesn't allow specifying Python package name. So we use this `sed`
         # command to change the import path in the generated file.
-        sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/gcs_pb2.py" &&
-        sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/common_pb2.py" &&
-        sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/node_manager_pb2.py" &&
-        sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/node_manager_pb2_grpc.py" &&
-        sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/reporter_pb2.py" &&
-        sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/reporter_pb2_grpc.py" &&
-        sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/core_worker_pb2.py" &&
-        sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/core_worker_pb2_grpc.py" &&
-        echo "$$WORK_DIR" > $@
+        files=(
+            python/ray/core/generated/gcs_pb2.py
+            python/ray/core/generated/common_pb2.py
+            python/ray/core/generated/node_manager_pb2.py
+            python/ray/core/generated/node_manager_pb2_grpc.py
+            python/ray/core/generated/reporter_pb2.py
+            python/ray/core/generated/reporter_pb2_grpc.py
+            python/ray/core/generated/core_worker_pb2.py
+            python/ray/core/generated/core_worker_pb2_grpc.py
+        )
+        sed -i -E 's/from src.ray.protobuf/from ./' "$${files[@]}"
+        echo "$${PWD}" > $@
     """,
     local = 1,
 )

bazel/ray.bzl

@@ -103,3 +103,41 @@ def define_java_module(
             "{auto_gen_header}": "<!-- This file is auto-generated by Bazel from pom_template.xml, do not modify it. -->",
         },
     )
+
+def copy_to_workspace(name, srcs, dstdir = ""):
+    if dstdir.startswith("/") or dstdir.startswith("\\"):
+        fail("Subdirectory must be a relative path: " + dstdir)
+    src_locations = " ".join(["$(locations %s)" % (src,) for src in srcs])
+    native.genrule(
+        name = name,
+        srcs = srcs,
+        outs = [name + ".out"],
+        # Keep this Bash script equivalent to the batch script below (or take out the batch script)
+        cmd = r"""
+            mkdir -p -- {dstdir}
+            for f in {locations}; do
+                rm -f -- {dstdir}$${{f##*/}}
+                cp -f -- "$$f" {dstdir}
+                echo $$f {dstdir}$${{f##*/}}
+            done > $@
+        """.format(
+            locations = src_locations,
+            dstdir = "." + ("/" + dstdir.replace("\\", "/")).rstrip("/") + "/",
+        ),
+        # Keep this batch script equivalent to the Bash script above (or take out the batch script)
+        cmd_bat = r"""
+            (
+                if not exist {dstdir} mkdir {dstdir}
+            ) && (
+                for %f in ({locations}) do @(
+                    (if exist {dstdir}%~nxf del /f /q {dstdir}%~nxf) &&
+                    copy /B /Y %f {dstdir} >NUL &&
+                    (echo %f {dstdir}%~nxf)
+                )
+            ) > $@
+        """.replace("\r", "").replace("\n", " ").format(
+            locations = src_locations,
+            dstdir = "." + ("\\" + dstdir.replace("/", "\\")).rstrip("\\") + "\\",
+        ),
+        local = 1,
+    )

bazel/ray_deps_build_all.bzl

@@ -1,3 +1,4 @@
+load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace")
 load("@com_github_ray_project_ray//java:dependencies.bzl", "gen_java_deps")
 load("@com_github_ray_project_ray//streaming/java:dependencies.bzl", "gen_streaming_java_deps")
 load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps")
@@ -9,6 +10,7 @@ load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_toolchains")
 
 
 def ray_deps_build_all():
+  bazel_skylib_workspace()
   gen_java_deps()
   gen_streaming_java_deps()
   checkstyle_deps()

bazel/ray_deps_setup.bzl

@@ -120,6 +120,13 @@ def ray_deps_setup():
         sha256 = "a6e372118bc961b182a3a86344c0385b6b509882929c6b12dc03bb5084c775d5",
     )
 
+    auto_http_archive(
+        name = "bazel_skylib",
+        strip_prefix = None,
+        url = "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz",
+        sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44",
+    )
+
     auto_http_archive(
         name = "com_github_checkstyle_java",
         url = "https://github.com/ray-project/checkstyle_java/archive/ef367030d1433877a3360bbfceca18a5d0791bdd.tar.gz",

streaming/BUILD.bazel

@@ -1,8 +1,9 @@
 # Bazel build
 # C/C++ documentation: https://docs.bazel.build/versions/master/be/c-cpp.html
 
+load("@bazel_skylib//rules:copy_file.bzl", "copy_file")
 load("@rules_proto_grpc//python:defs.bzl", "python_proto_compile")
-load("//bazel:ray.bzl", "COPTS")
+load("//bazel:ray.bzl", "COPTS", "copy_to_workspace")
 
 proto_library(
     name = "streaming_proto",
@@ -258,25 +259,24 @@ filegroup(
     ],
 )
 
+copy_to_workspace(
+    name = "cp_all_py_proto",
+    srcs = [":all_py_proto"],
+    dstdir = "streaming/python/generated",
+)
+
 genrule(
     name = "copy_streaming_py_proto",
     srcs = [
-        ":all_py_proto",
+        ":cp_all_py_proto",
     ],
     outs = [
         "copy_streaming_py_proto.out",
     ],
     cmd = """
-        set -e
-        WORK_DIR="$$(pwd)"
-        # Copy generated files.
-        GENERATED_DIR="$$WORK_DIR/streaming/python/generated"
-        rm -rf "$$GENERATED_DIR"
+        GENERATED_DIR="streaming/python/generated"
         mkdir -p "$$GENERATED_DIR"
         touch "$$GENERATED_DIR/__init__.py"
-        for f in $(locations //streaming:all_py_proto); do
-            cp -f "$$f" "$$GENERATED_DIR"
-        done
         sed -i -E 's/from streaming.src.protobuf/from ./' "$$GENERATED_DIR/remote_call_pb2.py"
         date > $@
     """,
@@ -285,22 +285,20 @@ genrule(
 )
 
 # Streaming java
-genrule(
+copy_file(
     name = "copy_jni_h",
-    srcs = ["@bazel_tools//tools/jdk:jni_header"],
-    outs = ["jni.h"],
-    cmd = "cp -f $< $@",
+    src = "@bazel_tools//tools/jdk:jni_header",
+    out = "jni.h",
 )
 
-genrule(
+copy_file(
     name = "copy_jni_md_h",
-    srcs = select({
-        "@bazel_tools//src/conditions:windows": ["@bazel_tools//tools/jdk:jni_md_header-windows"],
-        "@bazel_tools//src/conditions:darwin": ["@bazel_tools//tools/jdk:jni_md_header-darwin"],
-        "//conditions:default": ["@bazel_tools//tools/jdk:jni_md_header-linux"],
+    src = select({
+        "@bazel_tools//src/conditions:windows": "@bazel_tools//tools/jdk:jni_md_header-windows",
+        "@bazel_tools//src/conditions:darwin": "@bazel_tools//tools/jdk:jni_md_header-darwin",
+        "//conditions:default": "@bazel_tools//tools/jdk:jni_md_header-linux",
     }),
-    outs = ["jni_md.h"],
-    cmd = "cp -f $< $@",
+    out = "jni_md.h",
     visibility = ["//visibility:public"],
 )