Fix pyarrow version (#3760)

cmake/Modules/ArrowExternalProject.cmake

@@ -22,6 +22,9 @@ set(arrow_URL https://github.com/ray-project/arrow.git)
 # the patch is available at
 # https://github.com/ray-project/arrow/commit/c347cd571e51723fc8512922f1b3a8e45e45b169
 # See the discussion in https://github.com/apache/arrow/pull/3177
+# WARNING: If the arrow version is updated, you need to also update the
+# SETUPTOOLS_SCM_PRETEND_VERSION version string in the ThirdpartyToolchain.cmake
+# file
 set(arrow_TAG c347cd571e51723fc8512922f1b3a8e45e45b169)
 
 set(ARROW_INSTALL_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/external/arrow-install)

cmake/Modules/BoostExternalProject.cmake

@@ -64,6 +64,6 @@ if (RAY_BUILD_BOOST)
     BUILD_IN_SOURCE 1
     BUILD_BYPRODUCTS ${Boost_BUILD_PRODUCTS}
     CONFIGURE_COMMAND ./bootstrap.sh
-    BUILD_COMMAND bash -c "./b2 cxxflags=-fPIC cflags=-fPIC variant=release link=static --with-filesystem --with-system --with-thread --with-atomic --with-chrono --with-date_time --with-regex -j8 install --prefix=${Boost_INSTALL_PREFIX} > /dev/null"
+    BUILD_COMMAND bash -c "./b2 cxxflags=\"-fPIC -D_GLIBCXX_USE_CXX11_ABI=0\" cflags=-fPIC variant=release link=static --with-filesystem --with-system --with-thread --with-atomic --with-chrono --with-date_time --with-regex -j8 install --prefix=${Boost_INSTALL_PREFIX} > /dev/null"
     INSTALL_COMMAND "")
 endif ()

cmake/Modules/ThirdpartyToolchain.cmake

@@ -118,6 +118,7 @@ if ("${CMAKE_RAY_LANG_PYTHON}" STREQUAL "YES")
 
     # PYARROW_PARALLEL= , so it will add -j to pyarrow build
     set(pyarrow_ENV
+      "SETUPTOOLS_SCM_PRETEND_VERSION=0.11.1-RAY"
       "PKG_CONFIG_PATH=${ARROW_LIBRARY_DIR}/pkgconfig"
       "PYARROW_WITH_PLASMA=1"
       "PYARROW_WITH_TENSORFLOW=1"

python/build-wheel-macos.sh

@@ -67,7 +67,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do
     $PIP_CMD install --upgrade setuptools
     # Install setuptools_scm because otherwise when building the wheel for
     # Python 3.6, we see an error.
-    $PIP_CMD install -q setuptools_scm==2.1.0
+    $PIP_CMD install -q setuptools_scm==3.1.0
     # Fix the numpy version because this will be the oldest numpy version we can
     # support.
     $PIP_CMD install -q numpy==$NUMPY_VERSION cython==0.29.0

test/runtest.py

@@ -8,9 +8,11 @@ import os
 import random
 import re
 import setproctitle
+import shutil
 import string
 import subprocess
 import sys
+import tempfile
 import threading
 import time
 from collections import defaultdict, namedtuple, OrderedDict
@@ -2585,3 +2587,16 @@ def test_ray_stack(shutdown_only):
     if not success:
         raise Exception("Failed to find necessary information with "
                         "'ray stack'")
+
+
+def test_pandas_parquet_serialization():
+    # Only test this if pandas is installed
+    pytest.importorskip("pandas")
+
+    import pandas as pd
+
+    tempdir = tempfile.mkdtemp()
+    filename = os.path.join(tempdir, "parquet-test")
+    pd.DataFrame({"col1": [0, 1], "col2": [0, 1]}).to_parquet(filename)
+    # Clean up
+    shutil.rmtree(tempdir)