diff --git a/BUILD.bazel b/BUILD.bazel index 037cff77b..ca890a77c 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,13 +1,14 @@ # Bazel build # C/C++ documentation: https://docs.bazel.build/versions/master/be/c-cpp.html +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") load("@rules_proto//proto:defs.bzl", "proto_library") load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_proto_library", "cc_test") load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library") load("@com_github_grpc_grpc//bazel:cython_library.bzl", "pyx_library") load("@rules_proto_grpc//python:defs.bzl", "python_grpc_compile") load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library") -load("//bazel:ray.bzl", "COPTS", "PYX_COPTS", "PYX_SRCS") +load("//bazel:ray.bzl", "COPTS", "PYX_COPTS", "PYX_SRCS", "copy_to_workspace") config_setting( name = "windows_msvc-cl", @@ -334,22 +335,20 @@ cc_binary( deps = [":plasma_client"], ) -genrule( +copy_file( name = "copy_jni_h", - srcs = ["@bazel_tools//tools/jdk:jni_header"], - outs = ["jni.h"], - cmd = "cp -f $< $@", + src = "@bazel_tools//tools/jdk:jni_header", + out = "jni.h", ) -genrule( +copy_file( name = "copy_jni_md_h", - srcs = select({ - "@bazel_tools//src/conditions:windows": ["@bazel_tools//tools/jdk:jni_md_header-windows"], - "@bazel_tools//src/conditions:darwin": ["@bazel_tools//tools/jdk:jni_md_header-darwin"], - "//conditions:default": ["@bazel_tools//tools/jdk:jni_md_header-linux"], + src = select({ + "@bazel_tools//src/conditions:windows": "@bazel_tools//tools/jdk:jni_md_header-windows", + "@bazel_tools//src/conditions:darwin": "@bazel_tools//tools/jdk:jni_md_header-darwin", + "//conditions:default": "@bazel_tools//tools/jdk:jni_md_header-linux", }), - outs = ["jni_md.h"], - cmd = "cp -f $< $@", + out = "jni_md.h", ) genrule( @@ -514,7 +513,11 @@ cc_library( cc_test( name = "gcs_pub_sub_test", srcs = ["src/ray/gcs/pubsub/test/gcs_pub_sub_test.cc"], - args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], + args = [ + "$(location redis-server)", + "$(location redis-cli)", + "$(location libray_redis_module.so)", + ], copts = COPTS, data = [ "//:libray_redis_module.so", @@ -720,7 +723,15 @@ cc_binary( cc_test( name = "core_worker_test", srcs = ["src/ray/core_worker/test/core_worker_test.cc"], - args = ["$(location //:plasma_store_server) $(location raylet) $(location mock_worker) $(location gcs_server) $(location redis-cli) $(location redis-server) $(location libray_redis_module.so)"], + args = [ + "$(location //:plasma_store_server)", + "$(location raylet)", + "$(location mock_worker)", + "$(location gcs_server)", + "$(location redis-cli)", + "$(location redis-server)", + "$(location libray_redis_module.so)", + ], copts = COPTS, data = [ "//:gcs_server", @@ -991,7 +1002,11 @@ cc_test( srcs = [ "src/ray/gcs/gcs_server/test/gcs_server_rpc_test.cc", ], - args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], + args = [ + "$(location redis-server)", + "$(location redis-cli)", + "$(location libray_redis_module.so)", + ], copts = COPTS, data = [ "//:libray_redis_module.so", @@ -1084,7 +1099,11 @@ cc_test( srcs = [ "src/ray/gcs/gcs_server/test/redis_gcs_table_storage_test.cc", ], - args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], + args = [ + "$(location redis-server)", + "$(location redis-cli)", + "$(location libray_redis_module.so)", + ], copts = COPTS, data = [ "//:libray_redis_module.so", @@ -1159,7 +1178,11 @@ cc_test( srcs = [ "src/ray/gcs/gcs_client/test/global_state_accessor_test.cc", ], - args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], + args = [ + "$(location redis-server)", + "$(location redis-cli)", + "$(location libray_redis_module.so)", + ], copts = COPTS, data = [ "//:libray_redis_module.so", @@ -1180,7 +1203,11 @@ cc_test( srcs = [ "src/ray/gcs/gcs_client/test/service_based_gcs_client_test.cc", ], - args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], + args = [ + "$(location redis-server)", + "$(location redis-cli)", + "$(location libray_redis_module.so)", + ], copts = COPTS, data = [ "//:libray_redis_module.so", @@ -1430,7 +1457,11 @@ cc_library( cc_test( name = "redis_store_client_test", srcs = ["src/ray/gcs/store_client/test/redis_store_client_test.cc"], - args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], + args = [ + "$(location redis-server)", + "$(location redis-cli)", + "$(location libray_redis_module.so)", + ], copts = COPTS, data = [ "//:libray_redis_module.so", @@ -1485,7 +1516,11 @@ cc_library( cc_test( name = "redis_gcs_client_test", srcs = ["src/ray/gcs/test/redis_gcs_client_test.cc"], - args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], + args = [ + "$(location redis-server)", + "$(location redis-cli)", + "$(location libray_redis_module.so)", + ], copts = COPTS, data = [ "//:libray_redis_module.so", @@ -1501,7 +1536,11 @@ cc_test( cc_test( name = "redis_actor_info_accessor_test", srcs = ["src/ray/gcs/test/redis_actor_info_accessor_test.cc"], - args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], + args = [ + "$(location redis-server)", + "$(location redis-cli)", + "$(location libray_redis_module.so)", + ], copts = COPTS, data = [ "//:libray_redis_module.so", @@ -1518,7 +1557,11 @@ cc_test( cc_test( name = "redis_object_info_accessor_test", srcs = ["src/ray/gcs/test/redis_object_info_accessor_test.cc"], - args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], + args = [ + "$(location redis-server)", + "$(location redis-cli)", + "$(location libray_redis_module.so)", + ], copts = COPTS, data = [ "//:libray_redis_module.so", @@ -1535,7 +1578,11 @@ cc_test( cc_test( name = "subscription_executor_test", srcs = ["src/ray/gcs/test/subscription_executor_test.cc"], - args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], + args = [ + "$(location redis-server)", + "$(location redis-cli)", + "$(location libray_redis_module.so)", + ], copts = COPTS, data = [ "//:libray_redis_module.so", @@ -1552,7 +1599,11 @@ cc_test( cc_test( name = "redis_job_info_accessor_test", srcs = ["src/ray/gcs/test/redis_job_info_accessor_test.cc"], - args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], + args = [ + "$(location redis-server)", + "$(location redis-cli)", + "$(location libray_redis_module.so)", + ], copts = COPTS, data = [ "//:libray_redis_module.so", @@ -1569,7 +1620,11 @@ cc_test( cc_test( name = "redis_node_info_accessor_test", srcs = ["src/ray/gcs/test/redis_node_info_accessor_test.cc"], - args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], + args = [ + "$(location redis-server)", + "$(location redis-cli)", + "$(location libray_redis_module.so)", + ], copts = COPTS, data = [ "//:libray_redis_module.so", @@ -1586,7 +1641,11 @@ cc_test( cc_test( name = "asio_test", srcs = ["src/ray/gcs/test/asio_test.cc"], - args = ["$(location redis-server) $(location redis-cli) $(location libray_redis_module.so)"], + args = [ + "$(location redis-server)", + "$(location redis-cli)", + "$(location libray_redis_module.so)", + ], copts = COPTS, data = [ "//:libray_redis_module.so", @@ -1822,60 +1881,90 @@ py_library( visibility = ["__subpackages__"], ) +copy_to_workspace( + name = "cp_raylet_so", + srcs = ["python/ray/_raylet.so"], + dstdir = "python/ray", +) + +copy_to_workspace( + name = "cp_streaming", + srcs = ["python/ray/streaming/_streaming.so"], + dstdir = "python/ray/streaming", +) + +copy_to_workspace( + name = "cp_all_py_proto", + srcs = [":all_py_proto"], + dstdir = "python/ray/core/generated", +) + +copy_to_workspace( + name = "cp_redis", + srcs = [ + ":redis-cli", + ":redis-server", + ], + dstdir = "python/ray/core/src/ray/thirdparty/redis/src", +) + +copy_to_workspace( + name = "cp_libray_redis_module", + srcs = [":libray_redis_module.so"], + dstdir = "python/ray/core/src/ray/gcs/redis_module", +) + +copy_to_workspace( + name = "cp_raylet", + srcs = [":raylet"], + dstdir = "python/ray/core/src/ray/raylet", +) + +copy_to_workspace( + name = "cp_gcs_server", + srcs = [":gcs_server"], + dstdir = "python/ray/core/src/ray/gcs", +) + +copy_to_workspace( + name = "cp_plasma_store_server", + srcs = [":plasma_store_server"], + dstdir = "python/ray/core/src/plasma", +) + genrule( name = "ray_pkg", srcs = [ - "python/ray/_raylet.so", - "python/ray/streaming/_streaming.so", - "//:python_sources", - "//:all_py_proto", - "//:redis-server", - "//:redis-cli", - "//:libray_redis_module.so", - "//:raylet", - "//:gcs_server", - "//:plasma_store_server", + ":cp_raylet_so", + ":cp_streaming", + ":python_sources", + ":cp_all_py_proto", + ":cp_redis", + ":cp_libray_redis_module", + ":cp_raylet", + ":cp_gcs_server", + ":cp_plasma_store_server", "//streaming:copy_streaming_py_proto", ], outs = ["ray_pkg.out"], cmd = """ - WORK_DIR="$$(pwd)" && - RAYDIR="$$WORK_DIR/python/ray" - RAYLET="$(location python/ray/_raylet.so)" - RAYLET_TARGET_FILENAME="$${RAYLET##*/}" && if [ "$${OSTYPE-}" = "msys" ]; then - # If Windows, we need to do things differently (e.g. use .pyd instead of .so) - RAYLET_TARGET_FILENAME="$${RAYLET_TARGET_FILENAME%.*}" && - RAYLET_TARGET_FILENAME="$${RAYLET_TARGET_FILENAME}.pyd" && - if [ -f "$${RAYLET%.*}.pdb" ]; then - cp -f "$${RAYLET%.*}.pdb" "$${RAYDIR}"; - fi; - fi && - cp -f "$${RAYLET}" "$${RAYDIR}/$${RAYLET_TARGET_FILENAME}" && - cp -f $(location python/ray/streaming/_streaming.so) "$$WORK_DIR/python/ray/streaming" && - mkdir -p "$$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/" && - cp -f $(location //:redis-server) "$$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/" && - cp -f $(location //:redis-cli) "$$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/" && - mkdir -p "$$WORK_DIR/python/ray/core/src/ray/gcs/redis_module/" && - cp -f $(locations //:libray_redis_module.so) "$$WORK_DIR/python/ray/core/src/ray/gcs/redis_module/" && - cp -f $(location //:plasma_store_server) "$$WORK_DIR/python/ray/core/src/plasma/" && - cp -f $(location //:raylet) "$$WORK_DIR/python/ray/core/src/ray/raylet/" && - cp -f $(location //:gcs_server) "$$WORK_DIR/python/ray/core/src/ray/gcs/" && - mkdir -p "$$WORK_DIR/python/ray/core/generated/ray/protocol/" && - for f in $(locations //:all_py_proto); do - cp -f "$$f" "$$WORK_DIR/python/ray/core/generated/"; - done && + mv -f python/ray/_raylet.so python/ray/_raylet.pyd + fi # NOTE(hchen): Protobuf doesn't allow specifying Python package name. So we use this `sed` # command to change the import path in the generated file. - sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/gcs_pb2.py" && - sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/common_pb2.py" && - sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/node_manager_pb2.py" && - sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/node_manager_pb2_grpc.py" && - sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/reporter_pb2.py" && - sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/reporter_pb2_grpc.py" && - sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/core_worker_pb2.py" && - sed -i -E 's/from src.ray.protobuf/from ./' "$$WORK_DIR/python/ray/core/generated/core_worker_pb2_grpc.py" && - echo "$$WORK_DIR" > $@ + files=( + python/ray/core/generated/gcs_pb2.py + python/ray/core/generated/common_pb2.py + python/ray/core/generated/node_manager_pb2.py + python/ray/core/generated/node_manager_pb2_grpc.py + python/ray/core/generated/reporter_pb2.py + python/ray/core/generated/reporter_pb2_grpc.py + python/ray/core/generated/core_worker_pb2.py + python/ray/core/generated/core_worker_pb2_grpc.py + ) + sed -i -E 's/from src.ray.protobuf/from ./' "$${files[@]}" + echo "$${PWD}" > $@ """, local = 1, ) diff --git a/bazel/ray.bzl b/bazel/ray.bzl index 613a87b81..0e41f930a 100644 --- a/bazel/ray.bzl +++ b/bazel/ray.bzl @@ -103,3 +103,41 @@ def define_java_module( "{auto_gen_header}": "", }, ) + +def copy_to_workspace(name, srcs, dstdir = ""): + if dstdir.startswith("/") or dstdir.startswith("\\"): + fail("Subdirectory must be a relative path: " + dstdir) + src_locations = " ".join(["$(locations %s)" % (src,) for src in srcs]) + native.genrule( + name = name, + srcs = srcs, + outs = [name + ".out"], + # Keep this Bash script equivalent to the batch script below (or take out the batch script) + cmd = r""" + mkdir -p -- {dstdir} + for f in {locations}; do + rm -f -- {dstdir}$${{f##*/}} + cp -f -- "$$f" {dstdir} + echo $$f {dstdir}$${{f##*/}} + done > $@ + """.format( + locations = src_locations, + dstdir = "." + ("/" + dstdir.replace("\\", "/")).rstrip("/") + "/", + ), + # Keep this batch script equivalent to the Bash script above (or take out the batch script) + cmd_bat = r""" + ( + if not exist {dstdir} mkdir {dstdir} + ) && ( + for %f in ({locations}) do @( + (if exist {dstdir}%~nxf del /f /q {dstdir}%~nxf) && + copy /B /Y %f {dstdir} >NUL && + (echo %f {dstdir}%~nxf) + ) + ) > $@ + """.replace("\r", "").replace("\n", " ").format( + locations = src_locations, + dstdir = "." + ("\\" + dstdir.replace("/", "\\")).rstrip("\\") + "\\", + ), + local = 1, + ) diff --git a/bazel/ray_deps_build_all.bzl b/bazel/ray_deps_build_all.bzl index 5c47301d8..4d4d3632e 100644 --- a/bazel/ray_deps_build_all.bzl +++ b/bazel/ray_deps_build_all.bzl @@ -1,3 +1,4 @@ +load("@bazel_skylib//:workspace.bzl", "bazel_skylib_workspace") load("@com_github_ray_project_ray//java:dependencies.bzl", "gen_java_deps") load("@com_github_ray_project_ray//streaming/java:dependencies.bzl", "gen_streaming_java_deps") load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps") @@ -9,6 +10,7 @@ load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_toolchains") def ray_deps_build_all(): + bazel_skylib_workspace() gen_java_deps() gen_streaming_java_deps() checkstyle_deps() diff --git a/bazel/ray_deps_setup.bzl b/bazel/ray_deps_setup.bzl index aaf2add07..50524259c 100644 --- a/bazel/ray_deps_setup.bzl +++ b/bazel/ray_deps_setup.bzl @@ -120,6 +120,13 @@ def ray_deps_setup(): sha256 = "a6e372118bc961b182a3a86344c0385b6b509882929c6b12dc03bb5084c775d5", ) + auto_http_archive( + name = "bazel_skylib", + strip_prefix = None, + url = "https://github.com/bazelbuild/bazel-skylib/releases/download/1.0.2/bazel-skylib-1.0.2.tar.gz", + sha256 = "97e70364e9249702246c0e9444bccdc4b847bed1eb03c5a3ece4f83dfe6abc44", + ) + auto_http_archive( name = "com_github_checkstyle_java", url = "https://github.com/ray-project/checkstyle_java/archive/ef367030d1433877a3360bbfceca18a5d0791bdd.tar.gz", diff --git a/streaming/BUILD.bazel b/streaming/BUILD.bazel index 74b14723c..816a47813 100644 --- a/streaming/BUILD.bazel +++ b/streaming/BUILD.bazel @@ -1,8 +1,9 @@ # Bazel build # C/C++ documentation: https://docs.bazel.build/versions/master/be/c-cpp.html +load("@bazel_skylib//rules:copy_file.bzl", "copy_file") load("@rules_proto_grpc//python:defs.bzl", "python_proto_compile") -load("//bazel:ray.bzl", "COPTS") +load("//bazel:ray.bzl", "COPTS", "copy_to_workspace") proto_library( name = "streaming_proto", @@ -258,25 +259,24 @@ filegroup( ], ) +copy_to_workspace( + name = "cp_all_py_proto", + srcs = [":all_py_proto"], + dstdir = "streaming/python/generated", +) + genrule( name = "copy_streaming_py_proto", srcs = [ - ":all_py_proto", + ":cp_all_py_proto", ], outs = [ "copy_streaming_py_proto.out", ], cmd = """ - set -e - WORK_DIR="$$(pwd)" - # Copy generated files. - GENERATED_DIR="$$WORK_DIR/streaming/python/generated" - rm -rf "$$GENERATED_DIR" + GENERATED_DIR="streaming/python/generated" mkdir -p "$$GENERATED_DIR" touch "$$GENERATED_DIR/__init__.py" - for f in $(locations //streaming:all_py_proto); do - cp -f "$$f" "$$GENERATED_DIR" - done sed -i -E 's/from streaming.src.protobuf/from ./' "$$GENERATED_DIR/remote_call_pb2.py" date > $@ """, @@ -285,22 +285,20 @@ genrule( ) # Streaming java -genrule( +copy_file( name = "copy_jni_h", - srcs = ["@bazel_tools//tools/jdk:jni_header"], - outs = ["jni.h"], - cmd = "cp -f $< $@", + src = "@bazel_tools//tools/jdk:jni_header", + out = "jni.h", ) -genrule( +copy_file( name = "copy_jni_md_h", - srcs = select({ - "@bazel_tools//src/conditions:windows": ["@bazel_tools//tools/jdk:jni_md_header-windows"], - "@bazel_tools//src/conditions:darwin": ["@bazel_tools//tools/jdk:jni_md_header-darwin"], - "//conditions:default": ["@bazel_tools//tools/jdk:jni_md_header-linux"], + src = select({ + "@bazel_tools//src/conditions:windows": "@bazel_tools//tools/jdk:jni_md_header-windows", + "@bazel_tools//src/conditions:darwin": "@bazel_tools//tools/jdk:jni_md_header-darwin", + "//conditions:default": "@bazel_tools//tools/jdk:jni_md_header-linux", }), - outs = ["jni_md.h"], - cmd = "cp -f $< $@", + out = "jni_md.h", visibility = ["//visibility:public"], )