diff --git a/.travis.yml b/.travis.yml index 25812baf2..a72cdbcfb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -63,6 +63,16 @@ matrix: # Make sure that the README is formatted properly. - cd python - python setup.py check --restructuredtext --strict --metadata + - cd .. + # Run Bazel linter Buildifier. + - wget https://dl.google.com/go/go1.11.linux-amd64.tar.gz + - tar -xf go1.11.linux-amd64.tar.gz + - mkdir $HOME/go_dir + - export GOROOT=`pwd`/go + - export GOPATH="$HOME/go_dir" + - export PATH="$GOROOT/bin:$GOPATH/bin:$PATH" + - go get github.com/bazelbuild/buildtools/buildifier + - ./.travis/bazel-format.sh - os: linux dist: trusty diff --git a/.travis/bazel-format.sh b/.travis/bazel-format.sh new file mode 100755 index 000000000..ec5fa5bb0 --- /dev/null +++ b/.travis/bazel-format.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# Before running this script, please make sure golang is installed +# and buildifier is also installed. The example is showed in .travis.yml. +set -e + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) + +function usage() +{ + echo "Usage: bazel-format.sh []" + echo + echo "Options:" + echo " -h|--help print the help info" + echo " -c|--check check whether there are format issues in bazel files" + echo " -f|--fix fix all the format issue directly" + echo +} + +RUN_TYPE=diff + +# Parse options +while [[ $# > 0 ]]; do + key="$1" + case $key in + -h|--help) + usage + exit 0 + ;; + -c|--check) + RUN_TYPE=diff + ;; + -f|--fix) + RUN_TYPE=fix + ;; + *) + echo "ERROR: unknown option \"$key\"" + echo + usage + exit -1 + ;; + esac + shift +done + +pushd $ROOT_DIR/.. +BAZEL_FILES="bazel/BUILD bazel/BUILD.plasma bazel/ray.bzl BUILD.bazel WORKSPACE" +buildifier -mode=$RUN_TYPE -diff_command="diff -u" $BAZEL_FILES +popd diff --git a/BUILD.bazel b/BUILD.bazel index 2f58699f4..b2cda35dd 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -10,11 +10,11 @@ COPTS = ["-DRAY_USE_GLOG"] cc_binary( name = "raylet", srcs = ["src/ray/raylet/main.cc"], + copts = COPTS, deps = [ ":ray_util", ":raylet_lib", ], - copts = COPTS, ) cc_binary( @@ -24,25 +24,29 @@ cc_binary( "src/ray/raylet/monitor.h", "src/ray/raylet/monitor_main.cc", ], + copts = COPTS, deps = [ ":gcs", ":ray_util", ], - copts = COPTS, ) cc_library( name = "raylet_lib", - srcs = glob([ - "src/ray/raylet/*.cc" - ], exclude = [ - "src/ray/raylet/mock_gcs_client.cc", - "src/ray/raylet/monitor_main.cc", - "src/ray/raylet/*_test.cc", - ]), + srcs = glob( + [ + "src/ray/raylet/*.cc", + ], + exclude = [ + "src/ray/raylet/mock_gcs_client.cc", + "src/ray/raylet/monitor_main.cc", + "src/ray/raylet/*_test.cc", + ], + ), hdrs = glob([ "src/ray/raylet/*.h", ]), + copts = COPTS, deps = [ ":gcs", ":gcs_fbs", @@ -51,73 +55,72 @@ cc_library( ":ray_common", ":ray_util", "@boost//:asio", - "@plasma//:plasma_client", "@com_google_googletest//:gtest", + "@plasma//:plasma_client", ], - copts = COPTS, ) cc_test( name = "lineage_cache_test", srcs = ["src/ray/raylet/lineage_cache_test.cc"], + copts = COPTS, deps = [ - "@com_google_googletest//:gtest_main", ":node_manager_fbs", ":raylet_lib", + "@com_google_googletest//:gtest_main", ], - copts = COPTS, ) cc_test( name = "reconstruction_policy_test", srcs = ["src/ray/raylet/reconstruction_policy_test.cc"], + copts = COPTS, deps = [ - "@com_google_googletest//:gtest_main", ":node_manager_fbs", ":object_manager", - ":raylet_lib" + ":raylet_lib", + "@com_google_googletest//:gtest_main", ], - copts = COPTS, ) cc_test( name = "worker_pool_test", srcs = ["src/ray/raylet/worker_pool_test.cc"], - deps = [ - "@com_google_googletest//:gtest_main", - ":raylet_lib", - ], copts = COPTS, + deps = [ + ":raylet_lib", + "@com_google_googletest//:gtest_main", + ], ) cc_test( name = "logging_test", srcs = ["src/ray/util/logging_test.cc"], - deps = [ - "@com_google_googletest//:gtest_main", - ":ray_util", - ], copts = COPTS, + deps = [ + ":ray_util", + "@com_google_googletest//:gtest_main", + ], ) cc_test( name = "task_dependency_manager_test", srcs = ["src/ray/raylet/task_dependency_manager_test.cc"], - deps = [ - "@com_google_googletest//:gtest_main", - ":raylet_lib" - ], copts = COPTS, + deps = [ + ":raylet_lib", + "@com_google_googletest//:gtest_main", + ], ) cc_test( name = "task_test", srcs = ["src/ray/raylet/task_test.cc"], - deps = [ - "@com_google_googletest//:gtest_main", - ":raylet_lib", - ], copts = COPTS, + deps = [ + ":raylet_lib", + "@com_google_googletest//:gtest_main", + ], ) cc_library( @@ -128,6 +131,7 @@ cc_library( hdrs = glob([ "src/ray/object_manager/*.h", ]), + copts = COPTS, includes = [ "src", ], @@ -137,67 +141,70 @@ cc_library( ":ray_common", ":ray_util", "@boost//:asio", - "@plasma//:plasma_client" + "@plasma//:plasma_client", ], - copts = COPTS, ) cc_binary( name = "object_manager_test", testonly = 1, srcs = ["src/ray/object_manager/test/object_manager_test.cc"], - deps = [ - "@com_google_googletest//:gtest_main", - ":object_manager", - ], copts = COPTS, + deps = [ + ":object_manager", + "@com_google_googletest//:gtest_main", + ], ) cc_binary( name = "object_manager_stress_test", testonly = 1, srcs = ["src/ray/object_manager/test/object_manager_stress_test.cc"], - deps = [ - "@com_google_googletest//:gtest_main", - ":object_manager", - ], copts = COPTS, + deps = [ + ":object_manager", + "@com_google_googletest//:gtest_main", + ], ) cc_library( name = "ray_util", - srcs = glob([ - "src/ray/*.cc", - "src/ray/util/*.cc", - ], exclude = [ - "src/ray/util/logging_test.cc", - "src/ray/util/signal_test.cc" - ]), + srcs = glob( + [ + "src/ray/*.cc", + "src/ray/util/*.cc", + ], + exclude = [ + "src/ray/util/logging_test.cc", + "src/ray/util/signal_test.cc", + ], + ), hdrs = glob([ "src/ray/*.h", "src/ray/util/*.h", ]), + copts = COPTS, includes = [ "src", ], deps = [ - "@plasma//:plasma_client", - "@com_github_google_glog//:glog", - ":sha256" + ":sha256", + "@com_github_google_glog//:glog", + "@plasma//:plasma_client", ], - copts = COPTS, ) cc_library( name = "ray_common", srcs = [ "src/ray/common/client_connection.cc", - "src/ray/common/common_protocol.cc" + "src/ray/common/common_protocol.cc", ], hdrs = [ "src/ray/common/client_connection.h", - "src/ray/common/common_protocol.h" + "src/ray/common/common_protocol.h", ], + copts = COPTS, includes = [ "src/ray/gcs/format", ], @@ -207,7 +214,6 @@ cc_library( ":ray_util", "@boost//:asio", ], - copts = COPTS, ) cc_library( @@ -223,12 +229,15 @@ cc_library( cc_library( name = "hiredis", - srcs = glob([ - "src/ray/thirdparty/ae/ae.c", - "src/ray/thirdparty/hiredis/*.c", - ], exclude = [ - "src/ray/thirdparty/hiredis/test.c" - ]), + srcs = glob( + [ + "src/ray/thirdparty/ae/ae.c", + "src/ray/thirdparty/hiredis/*.c", + ], + exclude = [ + "src/ray/thirdparty/hiredis/test.c", + ], + ), hdrs = glob([ "src/ray/thirdparty/ae/*.h", "src/ray/thirdparty/hiredis/*.h", @@ -238,57 +247,60 @@ cc_library( "src/ray/thirdparty/ae/ae_epoll.c", ]), includes = [ - "src/ray/thirdparty/hiredis", "src/ray/thirdparty/ae", + "src/ray/thirdparty/hiredis", ], ) cc_library( name = "gcs", - srcs = glob([ - "src/ray/gcs/*.cc" - ], exclude = [ - "src/ray/gcs/*_test.cc" - ]), + srcs = glob( + [ + "src/ray/gcs/*.cc", + ], + exclude = [ + "src/ray/gcs/*_test.cc", + ], + ), hdrs = glob([ "src/ray/gcs/*.h", "src/ray/gcs/format/*.h", ]), + copts = COPTS, includes = [ "src/ray/gcs/format", ], deps = [ ":gcs_fbs", - ":node_manager_fbs", - ":ray_util", - ":ray_common", ":hiredis", + ":node_manager_fbs", + ":ray_common", + ":ray_util", "@boost//:asio", ], - copts = COPTS, ) cc_binary( name = "gcs_client_test", testonly = 1, srcs = ["src/ray/gcs/client_test.cc"], + copts = COPTS, deps = [ ":gcs", "@com_google_googletest//:gtest_main", ], - copts = COPTS, ) cc_binary( name = "asio_test", testonly = 1, srcs = ["src/ray/gcs/asio_test.cc"], + copts = COPTS, deps = [ ":gcs", - "@com_google_googletest//:gtest_main", ":ray_util", + "@com_google_googletest//:gtest_main", ], - copts = COPTS, ) FLATC_ARGS = [ @@ -308,7 +320,7 @@ flatbuffer_cc_library( name = "common_fbs", srcs = ["@plasma//:cpp/src/plasma/format/common.fbs"], flatc_args = FLATC_ARGS, - out_prefix = "src/ray/common/" + out_prefix = "src/ray/common/", ) flatbuffer_cc_library( @@ -333,31 +345,34 @@ pyx_library( "python/ray/__init__.py", "python/ray/_raylet.pyx", "python/ray/includes/*.pxd", - "python/ray/includes/*.pxi" + "python/ray/includes/*.pxi", ]), - deps = ["//:raylet_lib"] + deps = ["//:raylet_lib"], ) cc_binary( name = "raylet_library_java.so", srcs = [ - "@bazel_tools//tools/jdk:current_java_runtime", - "src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.h", - "src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.cc", "src/ray/id.h", + "src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.cc", + "src/ray/raylet/lib/java/org_ray_runtime_raylet_RayletClientImpl.h", "src/ray/raylet/raylet_client.h", - "src/ray/util/logging.h" - ], + "src/ray/util/logging.h", + "@bazel_tools//tools/jdk:current_java_runtime", + ], includes = [ "src", - "external/local_jdk/include"] + select({ - "@bazel_tools//src/conditions:darwin": ["external/local_jdk/include/darwin"], - "//conditions:default": ["external/local_jdk/include/linux"] - }), + "external/local_jdk/include", + ] + select({ + "@bazel_tools//src/conditions:darwin": ["external/local_jdk/include/darwin"], + "//conditions:default": ["external/local_jdk/include/linux"], + }), linkshared = 1, linkstatic = 1, - deps = ["@plasma//:plasma_client", - "//:raylet_lib"], + deps = [ + "//:raylet_lib", + "@plasma//:plasma_client", + ], ) genrule( @@ -370,17 +385,17 @@ genrule( filegroup( name = "raylet_library_java", - visibility = ["//visibility:public"], srcs = select({ "@bazel_tools//src/conditions:darwin": [":raylet_library_java.dylib"], "//conditions:default": [":raylet_library_java.so"], - }) + }), + visibility = ["//visibility:public"], ) flatbuffer_py_library( name = "python_gcs_fbs", srcs = [ - "src/ray/gcs/format/gcs.fbs" + "src/ray/gcs/format/gcs.fbs", ], outs = [ "ActorState.py", @@ -417,7 +432,7 @@ flatbuffer_py_library( flatbuffer_py_library( name = "python_node_manager_fbs", srcs = [ - "src/ray/raylet/format/node_manager.fbs" + "src/ray/raylet/format/node_manager.fbs", ], outs = [ "ray/protocol/DisconnectClient.py", @@ -436,13 +451,13 @@ flatbuffer_py_library( "ray/protocol/Task.py", "ray/protocol/TaskExecutionSpecification.py", "ray/protocol/WaitReply.py", - "ray/protocol/WaitRequest.py" + "ray/protocol/WaitRequest.py", ], - out_prefix = "python/ray/core/generated/", include_paths = [ - "src/ray/gcs/format/" + "src/ray/gcs/format/", ], - includes = ["src/ray/gcs/format/gcs.fbs"] + includes = ["src/ray/gcs/format/gcs.fbs"], + out_prefix = "python/ray/core/generated/", ) filegroup( @@ -458,34 +473,34 @@ filegroup( "python/ray/experimental/*.py", "python/ray/internal/*.py", "python/ray/WebUI.ipynb", - "python/ray/workers/default_worker.py" - ]) + "python/ray/workers/default_worker.py", + ]), ) genrule( name = "redis", + outs = ["redis-server"], cmd = """ set -x && curl -sL \"https://github.com/antirez/redis/archive/5.0.3.tar.gz\" | tar xz --strip-components=1 -C . && make && mv ./src/redis-server $(location redis-server) """, - outs = ["redis-server"] ) cc_library( name = "ray_redis_module", - hdrs = [ - "src/ray/gcs/redis_module/redismodule.h", - "src/ray/gcs/redis_module/redis_string.h" - ], srcs = [ "src/ray/gcs/redis_module/ray_redis_module.cc", - "src/ray/util/logging.cc" + "src/ray/util/logging.cc", + ], + hdrs = [ + "src/ray/gcs/redis_module/redis_string.h", + "src/ray/gcs/redis_module/redismodule.h", ], deps = [ - ":ray_common" - ] + ":ray_common", + ], ) genrule( @@ -521,5 +536,5 @@ genrule( mkdir -p $(location ray_pkg)/java_lib && cp $(location @plasma//:plasma_client_java) $(location ray_pkg)/java_lib && cp $(location //:raylet_library_java) $(location ray_pkg)/java_lib - """ + """, ) diff --git a/WORKSPACE b/WORKSPACE index 7e307d7f9..00f77852e 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -7,28 +7,29 @@ git_repository( ) load("@com_github_nelhage_rules_boost//:boost/boost.bzl", "boost_deps") + boost_deps() git_repository( name = "com_github_google_flatbuffers", - remote = "https://github.com/google/flatbuffers.git", commit = "63d51afd1196336a7d1f56a988091ef05deb1c62", + remote = "https://github.com/google/flatbuffers.git", ) git_repository( name = "com_google_googletest", - remote = "https://github.com/google/googletest", commit = "3306848f697568aacf4bcca330f6bdd5ce671899", + remote = "https://github.com/google/googletest", ) git_repository( - name = "com_github_gflags_gflags", + name = "com_github_gflags_gflags", remote = "https://github.com/gflags/gflags.git", tag = "v2.2.2", ) new_git_repository( - name = "com_github_google_glog", + name = "com_github_google_glog", build_file = "//bazel:BUILD.glog", remote = "https://github.com/google/glog.git", tag = "v0.3.5", @@ -37,16 +38,17 @@ new_git_repository( new_git_repository( name = "plasma", build_file = "@//bazel:BUILD.plasma", - remote = "https://github.com/ray-project/arrow", commit = "6a27c660ea700febf6fd73b2e851ab96e9315134", + remote = "https://github.com/ray-project/arrow", ) new_git_repository( - name="cython", - build_file="@//bazel:BUILD.cython", - remote = "https://github.com/cython/cython", + name = "cython", + build_file = "@//bazel:BUILD.cython", commit = "49414dbc7ddc2ca2979d6dbe1e44714b10d72e7e", + remote = "https://github.com/cython/cython", ) load("@//bazel:python_configure.bzl", "python_configure") -python_configure(name="local_config_python") + +python_configure(name = "local_config_python") diff --git a/bazel/BUILD.plasma b/bazel/BUILD.plasma index a8199e92c..edb92a3ac 100644 --- a/bazel/BUILD.plasma +++ b/bazel/BUILD.plasma @@ -2,51 +2,38 @@ load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library") cc_library( name = "arrow", + srcs = [ + "cpp/src/arrow/buffer.cc", + "cpp/src/arrow/memory_pool.cc", + "cpp/src/arrow/status.cc", + "cpp/src/arrow/util/io-util.cc", + "cpp/src/arrow/util/logging.cc", + "cpp/src/arrow/util/thread-pool.cc", + ], hdrs = [ "cpp/src/arrow/buffer.h", + "cpp/src/arrow/io/interfaces.h", "cpp/src/arrow/memory_pool.h", "cpp/src/arrow/status.h", - "cpp/src/arrow/io/interfaces.h", "cpp/src/arrow/util/bit-util.h", "cpp/src/arrow/util/io-util.h", "cpp/src/arrow/util/logging.h", "cpp/src/arrow/util/macros.h", "cpp/src/arrow/util/string_builder.h", "cpp/src/arrow/util/string_view.h", - "cpp/src/arrow/util/type_traits.h", "cpp/src/arrow/util/thread-pool.h", + "cpp/src/arrow/util/type_traits.h", "cpp/src/arrow/util/visibility.h", "cpp/src/arrow/util/windows_compatibility.h", "cpp/src/arrow/vendored/string_view.hpp", - "cpp/src/arrow/vendored/xxhash/xxhash.h", "cpp/src/arrow/vendored/xxhash/xxhash.c", - ], - srcs = [ - "cpp/src/arrow/buffer.cc", - "cpp/src/arrow/memory_pool.cc", - "cpp/src/arrow/status.cc", - "cpp/src/arrow/util/logging.cc", - "cpp/src/arrow/util/thread-pool.cc", - "cpp/src/arrow/util/io-util.cc", + "cpp/src/arrow/vendored/xxhash/xxhash.h", ], strip_include_prefix = "cpp/src", ) cc_library( name = "plasma_client", - visibility = ["//visibility:public"], - hdrs = [ - "cpp/src/plasma/client.h", - "cpp/src/plasma/common.h", - "cpp/src/plasma/compat.h", - "cpp/src/plasma/fling.h", - "cpp/src/plasma/io.h", - "cpp/src/plasma/malloc.h", - "cpp/src/plasma/plasma.h", - "cpp/src/plasma/protocol.h", - "cpp/src/plasma/common_generated.h", - "cpp/src/plasma/plasma_generated.h", - ], srcs = [ "cpp/src/plasma/client.cc", "cpp/src/plasma/common.cc", @@ -56,23 +43,41 @@ cc_library( "cpp/src/plasma/plasma.cc", "cpp/src/plasma/protocol.cc", ], - deps = [":common_fbs", ":plasma_fbs", ":arrow"], + hdrs = [ + "cpp/src/plasma/client.h", + "cpp/src/plasma/common.h", + "cpp/src/plasma/common_generated.h", + "cpp/src/plasma/compat.h", + "cpp/src/plasma/fling.h", + "cpp/src/plasma/io.h", + "cpp/src/plasma/malloc.h", + "cpp/src/plasma/plasma.h", + "cpp/src/plasma/plasma_generated.h", + "cpp/src/plasma/protocol.h", + ], strip_include_prefix = "cpp/src", + visibility = ["//visibility:public"], + deps = [ + ":arrow", + ":common_fbs", + ":plasma_fbs", + ], ) cc_binary( name = "plasma_client_java.so", srcs = [ - "@bazel_tools//tools/jdk:current_java_runtime", "cpp/src/plasma/lib/java/org_apache_arrow_plasma_PlasmaClientJNI.cc", - "cpp/src/plasma/lib/java/org_apache_arrow_plasma_PlasmaClientJNI.h" - ], + "cpp/src/plasma/lib/java/org_apache_arrow_plasma_PlasmaClientJNI.h", + "@bazel_tools//tools/jdk:current_java_runtime", + ], includes = [ "cpp/src", - "../../external/local_jdk/include"] + select({ - "@bazel_tools//src/conditions:darwin": ["../../external/local_jdk/include/darwin"], - "//conditions:default": ["../../external/local_jdk/include/linux"] - }), + "../../external/local_jdk/include", + ] + select({ + "@bazel_tools//src/conditions:darwin": ["../../external/local_jdk/include/darwin"], + "//conditions:default": ["../../external/local_jdk/include/linux"], + }), linkshared = 1, linkstatic = 1, deps = [":plasma_client"], @@ -88,20 +93,24 @@ genrule( filegroup( name = "plasma_client_java", - visibility = ["//visibility:public"], srcs = select({ "@bazel_tools//src/conditions:darwin": [":plasma_client_java.dylib"], "//conditions:default": [":plasma_client_java.so"], - }) + }), + visibility = ["//visibility:public"], ) cc_library( name = "plasma_lib", + srcs = [ + "cpp/src/plasma/dlmalloc.cc", + "cpp/src/plasma/events.cc", + "cpp/src/plasma/eviction_policy.cc", + ], hdrs = [ "cpp/src/plasma/events.h", "cpp/src/plasma/eviction_policy.h", "cpp/src/plasma/store.h", - "cpp/src/plasma/thirdparty/dlmalloc.c", "cpp/src/plasma/thirdparty/ae/ae.h", "cpp/src/plasma/thirdparty/ae/ae_epoll.c", "cpp/src/plasma/thirdparty/ae/ae_evport.c", @@ -109,24 +118,25 @@ cc_library( "cpp/src/plasma/thirdparty/ae/ae_select.c", "cpp/src/plasma/thirdparty/ae/config.h", "cpp/src/plasma/thirdparty/ae/zmalloc.h", + "cpp/src/plasma/thirdparty/dlmalloc.c", ], - srcs = [ - "cpp/src/plasma/dlmalloc.cc", - "cpp/src/plasma/eviction_policy.cc", - "cpp/src/plasma/events.cc", - ], - deps = [":plasma_client"], strip_include_prefix = "cpp/src", + deps = [":plasma_client"], ) - cc_binary( name = "plasma_store_server", + srcs = glob( + [ + "cpp/src/plasma/store.cc", + "cpp/src/plasma/thirdparty/ae/ae.c", + ], + exclude = [ + "cpp/src/plasma/dlmalloc.cc", + "cpp/src/plasma/events.cc", + ], + ), visibility = ["//visibility:public"], - srcs = glob([ - "cpp/src/plasma/store.cc", - "cpp/src/plasma/thirdparty/ae/ae.c", - ], exclude = ["cpp/src/plasma/dlmalloc.cc", "cpp/src/plasma/events.cc"]), deps = [":plasma_lib"], ) @@ -137,18 +147,18 @@ FLATC_ARGS = [ ] flatbuffer_cc_library( - name="common_fbs", - srcs=["cpp/src/plasma/format/common.fbs"], - flatc_args=FLATC_ARGS, - out_prefix="cpp/src/plasma/" + name = "common_fbs", + srcs = ["cpp/src/plasma/format/common.fbs"], + flatc_args = FLATC_ARGS, + out_prefix = "cpp/src/plasma/", ) flatbuffer_cc_library( - name="plasma_fbs", - srcs=["cpp/src/plasma/format/plasma.fbs"], - flatc_args=FLATC_ARGS, - out_prefix="cpp/src/plasma/", - includes = ["cpp/src/plasma/format/common.fbs"] + name = "plasma_fbs", + srcs = ["cpp/src/plasma/format/plasma.fbs"], + flatc_args = FLATC_ARGS, + includes = ["cpp/src/plasma/format/common.fbs"], + out_prefix = "cpp/src/plasma/", ) exports_files(["cpp/src/plasma/format/common.fbs"])