From e09f63ad657d359ab8c1d8794bd016da83702e8b Mon Sep 17 00:00:00 2001 From: mehrdadn Date: Tue, 11 Feb 2020 16:49:33 -0800 Subject: [PATCH] Fix build errors and add more targets to Windows builds (#6811) * Fix common.fbs rename (due to apache/arrow/commit/bef9a1c251397311a6415d3dc362ef419d154caa) * Add missing COPTS * Use socketpair(AF_INET) if boost::asio::local is unavailable (e.g. on Windows) * Fix compile bug in service_based_gcs_client_test.cc (fix build breakage in #6686) * Work around googletest/gmock inability to specify override to avoid -Werror,-Winconsistent-missing-override * Fix missing override on IsPlasmaBuffer() * Fix missing libraries for streaming * Factor out install-toolchains.sh * Put some Bazel flags into .bazelrc * Fix jni_md.h missing inclusion * Add ~/bin to PATH for Bazel * Change echo $$(date) > $@ to date > $@ * Fix lots of unquoted paths * Add system() call checks for Windows Co-authored-by: GitHub Web Flow --- .bazelrc | 16 +++ .github/workflows/main.yml | 113 ++++-------------- BUILD.bazel | 6 +- ci/travis/build-helper.sh | 25 ++++ ci/travis/install-toolchains.sh | 28 +++++ deploy/ray-operator/Makefile | 6 +- java/BUILD.bazel | 38 +++--- src/ray/core_worker/lib/java/jni_utils.h | 2 +- src/ray/core_worker/test/core_worker_test.cc | 3 + .../test/service_based_gcs_client_test.cc | 2 +- .../test/object_manager_stress_test.cc | 6 +- .../test/object_manager_test.cc | 6 +- src/ray/raylet/client_connection_test.cc | 12 ++ .../raylet/object_manager_integration_test.cc | 7 +- streaming/BUILD.bazel | 43 +++++-- streaming/java/BUILD.bazel | 34 +++--- streaming/src/test/queue_tests_base.h | 4 + 17 files changed, 201 insertions(+), 150 deletions(-) create mode 100755 ci/travis/build-helper.sh create mode 100755 ci/travis/install-toolchains.sh diff --git a/.bazelrc b/.bazelrc index 622921140..aa8af7d79 100644 --- a/.bazelrc +++ b/.bazelrc @@ -22,6 +22,8 @@ build --per_file_copt="-\\.(asm|S)$,-.*/arrow/util/logging\\.cc@-Werror" # Ignore warnings for protobuf generated files and external projects. build --per_file_copt="-\\.(asm|S)$,\\.pb\\.cc$@-w" build --per_file_copt="-\\.(asm|S)$,external/.*@-w" +# Ignore this warning since it's impractical to fix in the relevant headers +build --per_file_copt="-\\.(asm|S)$,.*/ray/raylet/reconstruction_policy_test\\.cc@-Wno-inconsistent-missing-override" # Ignore minor warnings for host tools, which we generally can't control build --host_copt="-Wno-builtin-macro-redefined" build --host_copt="-Wno-inconsistent-missing-override" @@ -34,6 +36,20 @@ build:iwyu --experimental_action_listener=//:iwyu_cpp # bazel_common/tools/maven/pom_file.bzl with Bazel 1.0 build --incompatible_depset_is_not_iterable=false +# Print relative paths when possible +build:windows --attempt_to_print_relative_paths +# Save disk space by hardlinking cache hits instead of copying +build:windows --experimental_repository_cache_hardlinks +# Avoid duplicate noise in the build output +build:windows --experimental_ui_deduplicate +# Clean the environment before building, to make builds more deterministic +build:windows --incompatible_strict_action_env +# For colored output (seems necessary on Windows) +build:windows --color=yes +# For compiler colored output (seems necessary on Windows) +build:windows --per_file_copt="-\\.(asm|S)$@-fansi-escape-codes" +build:windows --per_file_copt="-\\.(asm|S)$@-fcolor-diagnostics" + # Thread sanitizer configuration: build:tsan --strip=never build:tsan --copt -fsanitize=thread diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d71505c04..68d8ff465 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -42,31 +42,16 @@ jobs: CC: ${{ matrix.compiler }} run: | # TODO(mehrdadn): Replace this with the same build script as below, once we factor that out - set -euo pipefail - PATH="${HOME}/bin:${PATH}" - main() { - local startflags=() - startflags+=(--batch) - startflags+=(--nodeep_execroot) - local cmdflags=() - cmdflags+=(--attempt_to_print_relative_paths) - cmdflags+=(--color=yes) - cmdflags+=(--experimental_repository_cache_hardlinks) - cmdflags+=(--experimental_ui_deduplicate) - cmdflags+=(--incompatible_strict_action_env) - cmdflags+=(--keep_going) - cmdflags+=(--per_file_copt='-\.(asm|S)$@-fansi-escape-codes') - cmdflags+=(--per_file_copt='-\.(asm|S)$@-fcolor-diagnostics') - cmdflags+=(--show_progress_rate_limit=5) - cmdflags+=(--show_task_finish) - cmdflags+=(--show_timestamps) - cmdflags+=(--symlink_prefix=/) - cmdflags+=(--verbose_failures) - local packages=() - packages+=("//:ray_pkg") - bazel "${startflags[@]}" build "${cmdflags[@]}" "${packages[@]}" "$@" - } - main --compilation_mode=fastbuild --config=iwyu "$@" + . ./ci/travis/build-helper.sh prep_build_env + bazel --batch build \ + --compilation_mode=fastbuild \ + --config=iwyu \ + --keep_going \ + --show_progress_rate_limit=5 \ + --show_task_finish \ + --show_timestamps \ + --verbose_failures \ + "//:*" # TODO(mehrdadn): Should be "//:*", but we get a linking error with _streaming.so build: name: ${{ matrix.name }} runs-on: ${{ matrix.os }} @@ -96,26 +81,9 @@ jobs: - name: Install C/C++ toolchains if: matrix.compiler == 'clang' || matrix.compiler == 'clang-cl' shell: bash - run: | - set -euo pipefail - if [ "${OSTYPE}" = "msys" ]; then - export MSYS2_ARG_CONV_EXCL="*" # Don't let MSYS2 attempt to auto-translate arguments that look like paths - # Ideally we should be able to use the Chocolatey package manager: - # choco install --no-progress llvm - # However, it frequently gives HTTP 503 errors, so we just download and install manually. - urldir="https://releases.llvm.org" - arch=64 - if [ "${HOSTTYPE}" = "${HOSTTYPE%64}" ]; then arch=32; fi - target="./LLVM-${LLVM_VERSION_WINDOWS}-win${arch}.exe" - curl -s -L -R -o "${target}" "http://releases.llvm.org/${LLVM_VERSION_WINDOWS}/${target##*/}" - chmod +x "${target}" - "${target}" /S - rm -f -- "${target}" - elif 1>&- command -v pacman; then - sudo pacman -S --needed --noconfirm --noprogressbar clang - elif 1>&- command -v apt-get; then - sudo apt-get -q -y install clang - fi + env: + CC: ${{ matrix.compiler }} + run: ./ci/travis/install-toolchains.sh "${CC%%-*}" - name: Install MSYS2 (only needed if some shell commands are missing) if: false && runner.os == 'Windows' uses: numworks/setup-msys2@v1 @@ -124,50 +92,11 @@ jobs: - name: Perform build shell: bash run: | - set -euo pipefail - main() { - if [ "${OSTYPE}" = "msys" ]; then - export MSYS2_ARG_CONV_EXCL="*" # Don't let MSYS2 attempt to auto-translate arguments that look like paths - local latest_python_bin="" - for latest_python_bin in /proc/registry/HKEY_LOCAL_MACHINE/Software/Python/PythonCore/*/InstallPath/@; do - if [ -f "${latest_python_bin}" ]; then - latest_python_bin="$(tr -d '\0' < "${latest_python_bin}")" - latest_python_bin="${latest_python_bin}\\" - else - latest_python_bin="" - fi - done - latest_python_bin="${latest_python_bin}python.exe" - if [ -f "${latest_python_bin}" ]; then - export PYTHON2_BIN_PATH="${latest_python_bin}" PYTHON3_BIN_PATH="${latest_python_bin}" - fi - fi - # NOTE: Only options that are _user preferences_ (i.e. not repository-specific) should go here. - # These are options that people may disagree on having on their own machines, but which are useful for our builds. - bazel_output_root="${HOME}/bazel" - if [ "${OSTYPE}" = "msys" ]; then - bazel_output_root="$(cygpath -w -- "${bazel_output_root}")" - fi - local startflags=() - startflags+=(--batch) - startflags+=(--nodeep_execroot) - #startflags+=(--output_user_root="${bazel_output_root}") - local cmdflags=() - cmdflags+=(--attempt_to_print_relative_paths) - cmdflags+=(--color=yes) - cmdflags+=(--experimental_repository_cache_hardlinks) - cmdflags+=(--experimental_ui_deduplicate) - cmdflags+=(--incompatible_strict_action_env) - cmdflags+=(--keep_going) - cmdflags+=(--per_file_copt='-\.(asm|S)$@-fansi-escape-codes') - cmdflags+=(--per_file_copt='-\.(asm|S)$@-fcolor-diagnostics') - cmdflags+=(--show_progress_rate_limit=5) - cmdflags+=(--show_task_finish) - cmdflags+=(--show_timestamps) - cmdflags+=(--symlink_prefix=/) - cmdflags+=(--verbose_failures) - local packages=() - packages+=("//:ray_pkg") - bazel "${startflags[@]}" build "${cmdflags[@]}" "${packages[@]}" "$@" - } - main "$@" + . ./ci/travis/build-helper.sh prep_build_env + bazel --batch build \ + --keep_going \ + --show_progress_rate_limit=5 \ + --show_task_finish \ + --show_timestamps \ + --verbose_failures \ + "//:ray_pkg" diff --git a/BUILD.bazel b/BUILD.bazel index 040cff984..658bd9fb2 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1027,7 +1027,7 @@ FLATC_ARGS = [ flatbuffer_cc_library( name = "common_fbs", - srcs = ["@plasma//:cpp/src/plasma/format/common.fbs"], + srcs = ["@plasma//:cpp/src/plasma/common.fbs"], flatc_args = FLATC_ARGS, out_prefix = "src/ray/common/", ) @@ -1114,6 +1114,7 @@ cc_binary( "@bazel_tools//src/conditions:darwin": ["@bazel_tools//tools/jdk:jni_md_header-darwin"], "//conditions:default": ["@bazel_tools//tools/jdk:jni_md_header-linux"], }), + copts = COPTS, includes = [ "src", "external/bazel_tools/tools/jdk/include", @@ -1139,6 +1140,7 @@ cc_binary( "//:core_worker_lib", "//:src/ray/ray_exported_symbols.lds", "//:src/ray/ray_version_script.lds", + "//streaming:jni", ], ) @@ -1288,7 +1290,7 @@ genrule( outs = ["ray_pkg.out"], cmd = """ set -x && - WORK_DIR=$$(pwd) && + WORK_DIR="$$(pwd)" && cp -f $(location python/ray/_raylet.so) "$$WORK_DIR/python/ray" && mkdir -p "$$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/" && cp -f $(location //:redis-server) "$$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/" && diff --git a/ci/travis/build-helper.sh b/ci/travis/build-helper.sh new file mode 100755 index 000000000..27a2bed34 --- /dev/null +++ b/ci/travis/build-helper.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +prep_build_env() { + export PATH="${PATH}:${HOME}/bin" + if [ "${OSTYPE}" = "msys" ]; then + export MSYS2_ARG_CONV_EXCL="*" # Don't let MSYS2 attempt to auto-translate arguments that look like paths + local latest_python_bin="" + for latest_python_bin in /proc/registry/HKEY_LOCAL_MACHINE/Software/Python/PythonCore/*/InstallPath/@; do + if [ -f "${latest_python_bin}" ]; then + latest_python_bin="$(tr -d '\0' < "${latest_python_bin}")" + latest_python_bin="${latest_python_bin}\\" + else + latest_python_bin="" + fi + done + latest_python_bin="${latest_python_bin}python.exe" + if [ -f "${latest_python_bin}" ]; then + export PYTHON2_BIN_PATH="${latest_python_bin}" PYTHON3_BIN_PATH="${latest_python_bin}" + fi + fi +} + +if [ 0 -lt "$#" ]; then + "$@" +fi diff --git a/ci/travis/install-toolchains.sh b/ci/travis/install-toolchains.sh new file mode 100755 index 000000000..fa2c3e938 --- /dev/null +++ b/ci/travis/install-toolchains.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -euo pipefail + +install_clang() { + if [ "${OSTYPE}" = "msys" ]; then + export MSYS2_ARG_CONV_EXCL="*" # Don't let MSYS2 attempt to auto-translate arguments that look like paths + # Ideally we should be able to use the Chocolatey package manager: + # choco install --no-progress llvm + # However, it frequently gives HTTP 503 errors, so we just download and install manually. + urldir="https://releases.llvm.org" + arch=64 + if [ "${HOSTTYPE}" = "${HOSTTYPE%64}" ]; then arch=32; fi + local version + version="${LLVM_VERSION_WINDOWS}" + target="./LLVM-${version}-win${arch}.exe" + curl -s -L -R -o "${target}" "http://releases.llvm.org/${version}/${target##*/}" + chmod +x "${target}" + "${target}" /S + rm -f -- "${target}" + elif 1>&- command -v pacman; then + sudo pacman -S --needed --noconfirm --noprogressbar clang + elif 1>&- command -v apt-get; then + sudo apt-get -q -y install clang + fi +} + +install_"$@" diff --git a/deploy/ray-operator/Makefile b/deploy/ray-operator/Makefile index 5731b590b..dbfc3d366 100644 --- a/deploy/ray-operator/Makefile +++ b/deploy/ray-operator/Makefile @@ -64,11 +64,11 @@ controller-gen: ifeq (, $(shell which controller-gen)) @{ \ set -e ;\ - CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\ - cd $$CONTROLLER_GEN_TMP_DIR ;\ + CONTROLLER_GEN_TMP_DIR=$$(mktemp -d)" ;\ + cd "$$CONTROLLER_GEN_TMP_DIR" ;\ go mod init tmp ;\ go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.2 ;\ - rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ + rm -rf "$$CONTROLLER_GEN_TMP_DIR" ;\ } CONTROLLER_GEN=$(GOBIN)/controller-gen else diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 90e376b92..cddf543ef 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -173,21 +173,21 @@ genrule( outs = ["gen_maven_deps.out"], cmd = """ set -x - WORK_DIR=$$(pwd) + WORK_DIR="$$(pwd)" # Copy protobuf-generated files. - rm -rf $$WORK_DIR/java/runtime/src/main/java/org/ray/runtime/generated + rm -rf "$$WORK_DIR/java/runtime/src/main/java/org/ray/runtime/generated" for f in $(locations //java:all_java_proto); do - unzip $$f -x META-INF/MANIFEST.MF -d $$WORK_DIR/java/runtime/src/main/java + unzip "$$f" -x META-INF/MANIFEST.MF -d "$$WORK_DIR/java/runtime/src/main/java" done # Copy native dependecies. - NATIVE_DEPS_DIR=$$WORK_DIR/java/runtime/native_dependencies/ - rm -rf $$NATIVE_DEPS_DIR - mkdir -p $$NATIVE_DEPS_DIR + NATIVE_DEPS_DIR="$$WORK_DIR/java/runtime/native_dependencies/" + rm -rf "$$NATIVE_DEPS_DIR" + mkdir -p "$$NATIVE_DEPS_DIR" for f in $(locations //java:java_native_deps); do - chmod +w $$f - cp $$f $$NATIVE_DEPS_DIR + chmod +w "$$f" + cp "$$f" "$$NATIVE_DEPS_DIR" done - echo $$(date) > $@ + date > $@ """, local = 1, tags = ["no-cache"], @@ -202,12 +202,12 @@ genrule( outs = ["copy_pom_file.out"], cmd = """ set -x - WORK_DIR=$$(pwd) - cp -f $(location //java:org_ray_ray_api_pom) $$WORK_DIR/java/api/pom.xml - cp -f $(location //java:org_ray_ray_runtime_pom) $$WORK_DIR/java/runtime/pom.xml - cp -f $(location //java:org_ray_ray_tutorial_pom) $$WORK_DIR/java/tutorial/pom.xml - cp -f $(location //java:org_ray_ray_test_pom) $$WORK_DIR/java/test/pom.xml - echo $$(date) > $@ + WORK_DIR="$$(pwd)" + cp -f $(location //java:org_ray_ray_api_pom) "$$WORK_DIR/java/api/pom.xml" + cp -f $(location //java:org_ray_ray_runtime_pom) "$$WORK_DIR/java/runtime/pom.xml" + cp -f $(location //java:org_ray_ray_tutorial_pom) "$$WORK_DIR/java/tutorial/pom.xml" + cp -f $(location //java:org_ray_ray_test_pom) "$$WORK_DIR/java/test/pom.xml" + date > $@ """, local = 1, tags = ["no-cache"], @@ -234,10 +234,10 @@ genrule( ], outs = ["ray_java_pkg.out"], cmd = """ - WORK_DIR=$$(pwd) - rm -rf $$WORK_DIR/python/ray/jars && mkdir -p $$WORK_DIR/python/ray/jars - cp -f $(location //java:ray_dist_deploy.jar) $$WORK_DIR/python/ray/jars/ray_dist.jar - echo $$(date) > $@ + WORK_DIR="$$(pwd)" + rm -rf "$$WORK_DIR/python/ray/jars" && mkdir -p "$$WORK_DIR/python/ray/jars" + cp -f $(location //java:ray_dist_deploy.jar) "$$WORK_DIR/python/ray/jars/ray_dist.jar" + date > $@ """, local = 1, tags = ["no-cache"], diff --git a/src/ray/core_worker/lib/java/jni_utils.h b/src/ray/core_worker/lib/java/jni_utils.h index e9f59fbe4..abbad3236 100644 --- a/src/ray/core_worker/lib/java/jni_utils.h +++ b/src/ray/core_worker/lib/java/jni_utils.h @@ -181,7 +181,7 @@ class JavaByteArrayBuffer : public ray::Buffer { bool OwnsData() const override { return true; } - bool IsPlasmaBuffer() const { return false; } + bool IsPlasmaBuffer() const override { return false; } ~JavaByteArrayBuffer() { env_->ReleaseByteArrayElements(java_byte_array_, native_bytes_, JNI_ABORT); diff --git a/src/ray/core_worker/test/core_worker_test.cc b/src/ray/core_worker/test/core_worker_test.cc index 5ebf25a68..2f49cc8ee 100644 --- a/src/ray/core_worker/test/core_worker_test.cc +++ b/src/ray/core_worker/test/core_worker_test.cc @@ -75,6 +75,9 @@ std::string MetadataToString(std::shared_ptr obj) { class CoreWorkerTest : public ::testing::Test { public: CoreWorkerTest(int num_nodes) : gcs_options_("127.0.0.1", 6379, "") { +#ifdef _WIN32 + RAY_CHECK(false) << "port system() calls to Windows before running this test"; +#endif // flush redis first. flushall_redis(); diff --git a/src/ray/gcs/gcs_client/test/service_based_gcs_client_test.cc b/src/ray/gcs/gcs_client/test/service_based_gcs_client_test.cc index c5a278afb..2b323cc28 100644 --- a/src/ray/gcs/gcs_client/test/service_based_gcs_client_test.cc +++ b/src/ray/gcs/gcs_client/test/service_based_gcs_client_test.cc @@ -156,7 +156,7 @@ class ServiceBasedGcsGcsClientTest : public RedisServiceManagerForTest { std::vector nodes; RAY_CHECK_OK(gcs_client_->Nodes().AsyncGetAll( [&nodes, &promise](Status status, const std::vector &result) { - assert(result); + assert(!result.empty()); nodes.assign(result.begin(), result.end()); promise.set_value(status.ok()); })); diff --git a/src/ray/object_manager/test/object_manager_stress_test.cc b/src/ray/object_manager/test/object_manager_stress_test.cc index 5b76be9fd..6c8c3372a 100644 --- a/src/ray/object_manager/test/object_manager_stress_test.cc +++ b/src/ray/object_manager/test/object_manager_stress_test.cc @@ -68,7 +68,11 @@ class MockServer { class TestObjectManagerBase : public ::testing::Test { public: - TestObjectManagerBase() {} + TestObjectManagerBase() { +#ifdef _WIN32 + RAY_CHECK(false) << "port system() calls to Windows before running this test"; +#endif + } std::string StartStore(const std::string &id) { std::string store_id = "/tmp/store"; diff --git a/src/ray/object_manager/test/object_manager_test.cc b/src/ray/object_manager/test/object_manager_test.cc index 19a5f07d5..632863d71 100644 --- a/src/ray/object_manager/test/object_manager_test.cc +++ b/src/ray/object_manager/test/object_manager_test.cc @@ -62,7 +62,11 @@ class MockServer { class TestObjectManagerBase : public ::testing::Test { public: - TestObjectManagerBase() {} + TestObjectManagerBase() { +#ifdef _WIN32 + RAY_CHECK(false) << "port system() calls to Windows before running this test"; +#endif + } std::string StartStore(const std::string &id) { std::string store_id = "/tmp/store"; diff --git a/src/ray/raylet/client_connection_test.cc b/src/ray/raylet/client_connection_test.cc index db6ca53e5..1f0b867e0 100644 --- a/src/ray/raylet/client_connection_test.cc +++ b/src/ray/raylet/client_connection_test.cc @@ -6,6 +6,11 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +#if !defined(BOOST_ASIO_HAS_LOCAL_SOCKETS) +#include +#include +#endif + #include "ray/common/client_connection.h" namespace ray { @@ -15,7 +20,14 @@ class ClientConnectionTest : public ::testing::Test { public: ClientConnectionTest() : io_service_(), in_(io_service_), out_(io_service_), error_message_type_(1) { +#if defined(BOOST_ASIO_HAS_LOCAL_SOCKETS) boost::asio::local::connect_pair(in_, out_); +#else + int pair[2] = {}; // TODO(mehrdadn): This should be type SOCKET for Windows + RAY_CHECK(socketpair(AF_INET, SOCK_STREAM, 0, pair) == 0); + in_.assign(local_stream_protocol::v4(), pair[0]); + out_.assign(local_stream_protocol::v4(), pair[1]); +#endif } ray::Status WriteBadMessage(std::shared_ptr conn, diff --git a/src/ray/raylet/object_manager_integration_test.cc b/src/ray/raylet/object_manager_integration_test.cc index 99763c54f..1c2eecdb2 100644 --- a/src/ray/raylet/object_manager_integration_test.cc +++ b/src/ray/raylet/object_manager_integration_test.cc @@ -17,7 +17,12 @@ std::string store_executable; // TODO(hme): Get this working once the dust settles. class TestObjectManagerBase : public ::testing::Test { public: - TestObjectManagerBase() { RAY_LOG(INFO) << "TestObjectManagerBase: started."; } + TestObjectManagerBase() { + RAY_LOG(INFO) << "TestObjectManagerBase: started."; +#ifdef _WIN32 + RAY_CHECK(false) << "port system() calls to Windows before running this test"; +#endif + } std::string StartStore(const std::string &id) { std::string store_id = "/tmp/store"; diff --git a/streaming/BUILD.bazel b/streaming/BUILD.bazel index 51c3a40b8..3ab194701 100644 --- a/streaming/BUILD.bazel +++ b/streaming/BUILD.bazel @@ -175,6 +175,7 @@ cc_binary( srcs = glob(["src/test/*.h"]) + [ "src/test/streaming_queue_tests.cc", ], + copts = COPTS, deps = test_common_deps, ) @@ -183,6 +184,7 @@ cc_test( srcs = [ "src/test/ring_buffer_tests.cc", ], + copts = COPTS, includes = [ "streaming/src/test", ], @@ -194,6 +196,7 @@ cc_test( srcs = [ "src/test/message_serialization_tests.cc", ], + copts = COPTS, deps = test_common_deps, ) @@ -202,6 +205,7 @@ cc_test( srcs = [ "src/test/mock_transfer_tests.cc", ], + copts = COPTS, deps = test_common_deps, ) @@ -210,6 +214,7 @@ cc_test( srcs = [ "src/test/streaming_util_tests.cc", ], + copts = COPTS, deps = test_common_deps, ) @@ -237,16 +242,16 @@ genrule( cmd = """ set -e set -x - WORK_DIR=$$(pwd) + WORK_DIR="$$(pwd)" # Copy generated files. - GENERATED_DIR=$$WORK_DIR/streaming/python/generated - rm -rf $$GENERATED_DIR - mkdir -p $$GENERATED_DIR - touch $$GENERATED_DIR/__init__.py + GENERATED_DIR="$$WORK_DIR/streaming/python/generated" + rm -rf "$$GENERATED_DIR" + mkdir -p "$$GENERATED_DIR" + touch "$$GENERATED_DIR/__init__.py" for f in $(locations //streaming:streaming_py_proto); do - cp $$f $$GENERATED_DIR + cp "$$f" "$$GENERATED_DIR" done - echo $$(date) > $@ + date > $@ """, local = 1, visibility = ["//visibility:public"], @@ -269,6 +274,20 @@ genrule( }), outs = ["jni_md.h"], cmd = "cp -f $< $@", + visibility = ["//visibility:public"], +) + +cc_library( + name = "jni", + hdrs = [ + ":jni.h", + ":jni_md.h", + ], + copts = COPTS, + includes = [ + ".", # needed for `include ` + ], + visibility = ["//visibility:public"], ) cc_binary( @@ -276,18 +295,18 @@ cc_binary( srcs = glob([ "src/lib/java/*.cc", "src/lib/java/*.h", - ]) + [ - ":jni.h", # needed for `include "jni.h"` - ":jni_md.h", - ], + ]), + copts = COPTS, includes = [ - ".", # needed for `include ` "src", ], linkshared = 1, linkstatic = 1, visibility = ["//visibility:public"], deps = [ + ":jni", ":streaming_lib", + "//:core_worker_lib", + "//:ray_util", ], ) diff --git a/streaming/java/BUILD.bazel b/streaming/java/BUILD.bazel index adb4e338a..c740546e7 100644 --- a/streaming/java/BUILD.bazel +++ b/streaming/java/BUILD.bazel @@ -159,10 +159,10 @@ genrule( outs = ["copy_pom_file.out"], cmd = """ set -x - WORK_DIR=$$(pwd) - cp -f $(location //streaming/java:org_ray_ray_streaming-api_pom) $$WORK_DIR/streaming/java/streaming-api/pom.xml - cp -f $(location //streaming/java:org_ray_ray_streaming-runtime_pom) $$WORK_DIR/streaming/java/streaming-runtime/pom.xml - echo $$(date) > $@ + WORK_DIR="$$(pwd)" + cp -f $(location //streaming/java:org_ray_ray_streaming-api_pom) "$$WORK_DIR/streaming/java/streaming-api/pom.xml" + cp -f $(location //streaming/java:org_ray_ray_streaming-runtime_pom) "$$WORK_DIR/streaming/java/streaming-runtime/pom.xml" + date > $@ """, local = 1, tags = ["no-cache"], @@ -177,15 +177,15 @@ genrule( outs = ["cp_java_generated.out"], cmd = """ set -x - WORK_DIR=$$(pwd) - GENERATED_DIR=$$WORK_DIR/streaming/java/streaming-runtime/src/main/java/org/ray/streaming/runtime/generated - rm -rf $$GENERATED_DIR - mkdir -p $$GENERATED_DIR + WORK_DIR="$$(pwd)" + GENERATED_DIR="$$WORK_DIR/streaming/java/streaming-runtime/src/main/java/org/ray/streaming/runtime/generated" + rm -rf "$$GENERATED_DIR" + mkdir -p "$$GENERATED_DIR" # Copy protobuf-generated files. for f in $(locations //streaming/java:all_java_proto); do - unzip $$f -x META-INF/MANIFEST.MF -d $$WORK_DIR/streaming/java/streaming-runtime/src/main/java + unzip "$$f" -x META-INF/MANIFEST.MF -d "$$WORK_DIR/streaming/java/streaming-runtime/src/main/java" done - echo $$(date) > $@ + date > $@ """, local = 1, tags = ["no-cache"], @@ -201,16 +201,16 @@ genrule( outs = ["gen_maven_deps.out"], cmd = """ set -x - WORK_DIR=$$(pwd) + WORK_DIR="$$(pwd)" # Copy native dependencies. - NATIVE_DEPS_DIR=$$WORK_DIR/streaming/java/streaming-runtime/native_dependencies/ - rm -rf $$NATIVE_DEPS_DIR - mkdir -p $$NATIVE_DEPS_DIR + NATIVE_DEPS_DIR="$$WORK_DIR/streaming/java/streaming-runtime/native_dependencies/" + rm -rf "$$NATIVE_DEPS_DIR" + mkdir -p "$$NATIVE_DEPS_DIR" for f in $(locations //streaming/java:java_native_deps); do - chmod +w $$f - cp $$f $$NATIVE_DEPS_DIR + chmod +w "$$f" + cp "$$f" "$$NATIVE_DEPS_DIR" done - echo $$(date) > $@ + date > $@ """, local = 1, tags = ["no-cache"], diff --git a/streaming/src/test/queue_tests_base.h b/streaming/src/test/queue_tests_base.h index 6e3da6561..6b9e627eb 100644 --- a/streaming/src/test/queue_tests_base.h +++ b/streaming/src/test/queue_tests_base.h @@ -21,6 +21,10 @@ class StreamingQueueTestBase : public ::testing::TestWithParam { store_executable_(store_exe), actor_executable_(actor_exe), node_manager_port_(port) { +#ifdef _WIN32 + RAY_CHECK(false) << "port system() calls to Windows before running this test"; +#endif + // flush redis first. flushall_redis();