Upgrade arrow to latest master (#4858)

bazel/BUILD.plasma

@@ -25,11 +25,13 @@ cc_library(
     name = "arrow",
     srcs = [
         "cpp/src/arrow/buffer.cc",
+        "cpp/src/arrow/io/interfaces.cc",
         "cpp/src/arrow/memory_pool.cc",
         "cpp/src/arrow/status.cc",
         "cpp/src/arrow/util/io-util.cc",
         "cpp/src/arrow/util/logging.cc",
         "cpp/src/arrow/util/memory.cc",
+        "cpp/src/arrow/util/string_builder.cc",
         "cpp/src/arrow/util/thread-pool.cc",
     ],
     hdrs = [
@@ -42,6 +44,7 @@ cc_library(
         "cpp/src/arrow/util/logging.h",
         "cpp/src/arrow/util/macros.h",
         "cpp/src/arrow/util/memory.h",
+        "cpp/src/arrow/util/stl.h",
         "cpp/src/arrow/util/string_builder.h",
         "cpp/src/arrow/util/string_view.h",
         "cpp/src/arrow/util/thread-pool.h",
@@ -53,6 +56,9 @@ cc_library(
         "cpp/src/arrow/vendored/xxhash/xxhash.h",
     ],
     strip_include_prefix = "cpp/src",
+    deps = [
+        "@boost//:filesystem",
+    ],
 )
 
 cc_library(

bazel/ray_deps_setup.bzl

@@ -3,9 +3,9 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
 def ray_deps_setup():
     RULES_JVM_EXTERNAL_TAG = "1.2"
-   
+
     RULES_JVM_EXTERNAL_SHA = "e5c68b87f750309a79f59c2b69ead5c3221ffa54ff9496306937bfa1c9c8c86b"
-   
+
     http_archive(
         name = "rules_jvm_external",
         sha256 = RULES_JVM_EXTERNAL_SHA,
@@ -18,7 +18,7 @@ def ray_deps_setup():
         strip_prefix = "bazel-common-f1115e0f777f08c3cdb115526c4e663005bec69b",
         url = "https://github.com/google/bazel-common/archive/f1115e0f777f08c3cdb115526c4e663005bec69b.zip",
     )
- 
+
     BAZEL_SKYLIB_TAG = "0.6.0"
 
     http_archive(
@@ -26,64 +26,64 @@ def ray_deps_setup():
         strip_prefix = "bazel-skylib-%s" % BAZEL_SKYLIB_TAG,
         url = "https://github.com/bazelbuild/bazel-skylib/archive/%s.tar.gz" % BAZEL_SKYLIB_TAG,
     )
-   
+
     git_repository(
         name = "com_github_checkstyle_java",
         commit = "85f37871ca03b9d3fee63c69c8107f167e24e77b",
         remote = "https://github.com/ruifangChen/checkstyle_java",
     )
-   
+
     git_repository(
         name = "com_github_nelhage_rules_boost",
         commit = "5171b9724fbb39c5fdad37b9ca9b544e8858d8ac",
         remote = "https://github.com/ray-project/rules_boost",
     )
-   
+
     git_repository(
         name = "com_github_google_flatbuffers",
         commit = "63d51afd1196336a7d1f56a988091ef05deb1c62",
         remote = "https://github.com/google/flatbuffers.git",
     )
-   
+
     git_repository(
         name = "com_google_googletest",
         commit = "3306848f697568aacf4bcca330f6bdd5ce671899",
         remote = "https://github.com/google/googletest",
     )
-   
+
     git_repository(
         name = "com_github_gflags_gflags",
         remote = "https://github.com/gflags/gflags.git",
         tag = "v2.2.2",
     )
-   
+
     new_git_repository(
         name = "com_github_google_glog",
         build_file = "@//bazel:BUILD.glog",
         commit = "5c576f78c49b28d89b23fbb1fc80f54c879ec02e",
         remote = "https://github.com/google/glog",
     )
-   
+
     new_git_repository(
         name = "plasma",
         build_file = "@//bazel:BUILD.plasma",
-        commit = "d00497b38be84fd77c40cbf77f3422f2a81c44f9",
+        commit = "9fcc12fc094b85ec2e3e9798bae5c8151d14df5e",
         remote = "https://github.com/apache/arrow",
     )
-   
+
     new_git_repository(
         name = "cython",
         build_file = "@//bazel:BUILD.cython",
         commit = "49414dbc7ddc2ca2979d6dbe1e44714b10d72e7e",
         remote = "https://github.com/cython/cython",
     )
-   
+
     http_archive(
         name = "io_opencensus_cpp",
         strip_prefix = "opencensus-cpp-3aa11f20dd610cb8d2f7c62e58d1e69196aadf11",
         urls = ["https://github.com/census-instrumentation/opencensus-cpp/archive/3aa11f20dd610cb8d2f7c62e58d1e69196aadf11.zip"],
     )
-   
+
     # OpenCensus depends on Abseil so we have to explicitly pull it in.
     # This is how diamond dependencies are prevented.
     git_repository(
@@ -96,7 +96,7 @@ def ray_deps_setup():
     http_archive(
         name = "com_github_jupp0r_prometheus_cpp",
         strip_prefix = "prometheus-cpp-master",
-   
+
         # TODO(qwang): We should use the repository of `jupp0r` here when this PR
         # `https://github.com/jupp0r/prometheus-cpp/pull/225` getting merged.
         urls = ["https://github.com/jovany-wang/prometheus-cpp/archive/master.zip"],

build.sh

@@ -101,8 +101,8 @@ pushd "$BUILD_DIR"
 # generated from https://github.com/ray-project/arrow-build from
 # the commit listed in the command.
 $PYTHON_EXECUTABLE -m pip install \
-    --target="$ROOT_DIR/python/ray/pyarrow_files" pyarrow==0.12.0.RAY \
-    --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/ca1fa51f0901f5a4298f0e4faea00f24e5dd7bb7/index.html
+    --target="$ROOT_DIR/python/ray/pyarrow_files" pyarrow==0.14.0.RAY \
+    --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/9f35817b35f9d0614a736a497d70de2cf07fed52/index.html
 export PYTHON_BIN_PATH="$PYTHON_EXECUTABLE"
 
 if [ "$RAY_BUILD_JAVA" == "YES" ]; then

python/ray/rllib/tests/test_env_with_subprocess.py

@@ -80,6 +80,7 @@ if __name__ == "__main__":
             },
         },
     })
+    time.sleep(5.0)
     leaked = leaked_processes()
     assert not leaked, "LEAKED PROCESSES: {}".format(leaked)
     assert not os.path.exists(UNIQUE_FILE_0), "atexit handler not called"