From 05c4e3fb2a3b36ceea90cac4104d8466a0cc2ac5 Mon Sep 17 00:00:00 2001
From: Barak Michener <me@barakmich.com>
Date: Tue, 3 Nov 2020 19:36:32 -0800
Subject: [PATCH] [build] Build wheels with manylinux2014 (#11621)

* necessary changes

* Split bazel install

* manylinux2014

* change references to manylinux2014

* Fix lint

* port alex's docker build changes

* fix config issue

* remove extra manylinux2010 requirement script

* revert SHA overwrite

* wip

* incompatible_linklibs

* fix nits
---
 .bazelrc                                      |  6 +++++
 build-docker.sh                               |  4 ++--
 ci/travis/build-docker-images.py              |  1 -
 ci/travis/ci.sh                               |  4 ++--
 ci/travis/install-bazel.sh                    |  1 +
 doc/dev/RELEASE_PROCESS.rst                   |  6 ++---
 doc/dev/download_wheels.sh                    |  6 ++---
 .../newsreader/ray-project/cluster.yaml       |  2 +-
 doc/source/development.rst                    |  2 +-
 doc/source/installation.rst                   |  6 ++---
 python/README-building-wheels.md              |  4 ++--
 ...linux1.sh => build-wheel-manylinux2014.sh} | 24 ++++++++++++-------
 python/ray/autoscaler/aws/defaults.yaml       |  2 +-
 python/ray/autoscaler/aws/example-full.yaml   |  2 +-
 .../autoscaler/aws/example-gpu-docker.yaml    |  4 ++--
 python/ray/autoscaler/aws/example-java.yaml   |  2 +-
 python/ray/autoscaler/azure/defaults.yaml     |  2 +-
 python/ray/autoscaler/azure/example-full.yaml |  2 +-
 .../autoscaler/azure/example-gpu-docker.yaml  |  2 +-
 python/ray/autoscaler/azure/example-gpu.yaml  |  2 +-
 python/ray/autoscaler/gcp/defaults.yaml       |  2 +-
 python/ray/autoscaler/gcp/example-full.yaml   |  2 +-
 .../autoscaler/gcp/example-gpu-docker.yaml    |  4 ++--
 python/ray/autoscaler/local/example-full.yaml |  2 +-
 python/ray/nightly-wheels.yaml                |  6 ++---
 .../util/sgd/tf/examples/tf-example-sgd.yaml  |  2 +-
 .../util/sgd/torch/examples/example-sgd.yaml  |  2 +-
 .../torch/examples/image_models/cluster.yaml  |  2 +-
 .../torch/examples/segmentation/example.yaml  |  2 +-
 .../sgd/torch/examples/sgd-development.yaml   |  2 +-
 .../torch/examples/transformers/cluster.yaml  |  2 +-
 python/setup.py                               |  4 +++-
 release/long_running_distributed_tests/run.sh |  2 +-
 release/long_running_tests/run.sh             |  2 +-
 release/microbenchmark/run.sh                 |  2 +-
 release/rllib_tests/regression_tests/run.sh   |  2 +-
 release/rllib_tests/stress_tests/run.sh       |  2 +-
 release/rllib_tests/unit_gpu_tests/run.sh     |  2 +-
 release/stress_tests/cluster.yaml             |  2 +-
 release/stress_tests/run.sh                   |  2 +-
 40 files changed, 74 insertions(+), 58 deletions(-)
 rename python/{build-wheel-manylinux1.sh => build-wheel-manylinux2014.sh} (76%)

diff --git a/.bazelrc b/.bazelrc
index f342bf475..67bbf92d1 100644
--- a/.bazelrc
+++ b/.bazelrc
@@ -7,6 +7,7 @@ build --enable_platform_specific_config
 build --action_env=PATH
 # For --compilation_mode=dbg, consider enabling checks in the standard library as well (below).
 build --compilation_mode=opt
+#build --cxxopt="-std=c++11"
 # This workaround is needed to prevent Bazel from compiling the same file twice (once PIC and once not).
 build:linux --force_pic
 build:macos --force_pic
@@ -35,6 +36,7 @@ build:msvc     --per_file_copt="-\\.(asm|S)$@-WX"
 # Ignore warnings for protobuf generated files and external projects.
 build --per_file_copt="\\.pb\\.cc$@-w"
 build --per_file_copt="-\\.(asm|S)$,external/.*@-w"
+#build --per_file_copt="external/.*@-Wno-unused-result"
 # Ignore minor warnings for host tools, which we generally can't control
 build:clang-cl --host_copt="-Wno-inconsistent-missing-override"
 build:clang-cl --host_copt="-Wno-microsoft-unqualified-friend"
@@ -60,6 +62,10 @@ build:windows --color=yes
 build:clang-cl --per_file_copt="-\\.(asm|S)$@-fansi-escape-codes"
 build:clang-cl --per_file_copt="-\\.(asm|S)$@-fcolor-diagnostics"
 
+build:manylinux2010 --copt="-Wno-unused-result"
+build:manylinux2010 --linkopt="-lrt"
+
+
 # Debug build flags. Uncomment in '-c dbg' builds to enable checks in the C++ standard library:
 #build:linux --cxxopt="-D_GLIBCXX_DEBUG=1"
 #build:linux --cxxopt="-D_GLIBCXX_DEBUG_PEDANTIC=1"
diff --git a/build-docker.sh b/build-docker.sh
index 33c0b5792..973afaebe 100755
--- a/build-docker.sh
+++ b/build-docker.sh
@@ -7,7 +7,7 @@ set -x
 
 GPU=""
 BASE_IMAGE="ubuntu:focal"
-WHEEL_URL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux1_x86_64.whl"
+WHEEL_URL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl"
 PYTHON_VERSION=""
 
 while [[ $# -gt 0 ]]
@@ -90,4 +90,4 @@ if [ $BUILD_EXAMPLES ]; then
     fi
 fi
 
-rm -rf "$WHEEL_DIR"
\ No newline at end of file
+rm -rf "$WHEEL_DIR"
diff --git a/ci/travis/build-docker-images.py b/ci/travis/build-docker-images.py
index 71b2a9a44..2e5f66655 100644
--- a/ci/travis/build-docker-images.py
+++ b/ci/travis/build-docker-images.py
@@ -105,7 +105,6 @@ def _build_cpu_gpu_images(image_name) -> List[str]:
                 start = datetime.datetime.now()
                 current_iter = start
                 for line in output:
-                    # print(line)
                     if datetime.datetime.now(
                     ) - current_iter >= datetime.timedelta(minutes=5):
                         current_iter = datetime.datetime.now()
diff --git a/ci/travis/ci.sh b/ci/travis/ci.sh
index 2da2e4a02..2b58e11fe 100755
--- a/ci/travis/ci.sh
+++ b/ci/travis/ci.sh
@@ -289,8 +289,8 @@ build_wheels() {
 
       # This command should be kept in sync with ray/python/README-building-wheels.md,
       # except the "${MOUNT_BAZEL_CACHE[@]}" part.
-      suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" \
-        rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh
+      docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" \
+      quay.io/pypa/manylinux2014_x86_64 /ray/python/build-wheel-manylinux2014.sh
       ;;
     darwin*)
       # This command should be kept in sync with ray/python/README-building-wheels.md.
diff --git a/ci/travis/install-bazel.sh b/ci/travis/install-bazel.sh
index 4f64a677b..9431594fc 100755
--- a/ci/travis/install-bazel.sh
+++ b/ci/travis/install-bazel.sh
@@ -44,6 +44,7 @@ esac
   fi
 )
 
+export PATH=/opt/python/cp36-cp36m/bin:$PATH
 python="$(command -v python3 || command -v python || echo python)"
 version="$("${python}" -s -c "import runpy, sys; runpy.run_path(sys.argv.pop(), run_name='__api__')" bazel_version "${ROOT_DIR}/../../python/setup.py")"
 if [ "${OSTYPE}" = "msys" ]; then
diff --git a/doc/dev/RELEASE_PROCESS.rst b/doc/dev/RELEASE_PROCESS.rst
index 127f11a99..49ff43a27 100644
--- a/doc/dev/RELEASE_PROCESS.rst
+++ b/doc/dev/RELEASE_PROCESS.rst
@@ -129,9 +129,9 @@ This document describes the process for creating new releases.
        export RAY_VERSION=...  # e.g., 0.7.0
 
        # Linux Wheels
-       pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-manylinux1_x86_64.whl
-       pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-manylinux1_x86_64.whl
-       pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-manylinux1_x86_64.whl
+       pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-manylinux2014_x86_64.whl
+       pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-manylinux2014_x86_64.whl
+       pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-manylinux2014_x86_64.whl
 
        # Mac Wheels
        pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-macosx_10_13_intel.whl
diff --git a/doc/dev/download_wheels.sh b/doc/dev/download_wheels.sh
index ddb3d36df..163e87921 100644
--- a/doc/dev/download_wheels.sh
+++ b/doc/dev/download_wheels.sh
@@ -11,9 +11,9 @@ if [[ -z "$RAY_VERSION" ]]; then
 fi
 
 # Make sure Linux wheels are downloadable without errors.
-wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-manylinux1_x86_64.whl"
-wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-manylinux1_x86_64.whl"
-wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-manylinux1_x86_64.whl"
+wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-manylinux2014_x86_64.whl"
+wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-manylinux2014_x86_64.whl"
+wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp38-cp38-manylinux2014_x86_64.whl"
 # Make sure macos wheels are downloadable without errors.
 wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp36-cp36m-macosx_10_13_intel.whl"
 wget "https://s3-us-west-2.amazonaws.com/ray-wheels/releases/$RAY_VERSION/$RAY_HASH/ray-$RAY_VERSION-cp37-cp37m-macosx_10_13_intel.whl"
diff --git a/doc/examples/newsreader/ray-project/cluster.yaml b/doc/examples/newsreader/ray-project/cluster.yaml
index 49fe9b833..6b69fe05a 100644
--- a/doc/examples/newsreader/ray-project/cluster.yaml
+++ b/doc/examples/newsreader/ray-project/cluster.yaml
@@ -24,7 +24,7 @@ setup_commands:
   - echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.bashrc
   - sudo apt -y update
   - sudo apt -y install npm
-  - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
+  - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
 
 # How Ray will authenticate with newly launched nodes.
 auth:
diff --git a/doc/source/development.rst b/doc/source/development.rst
index 0e5d29b06..8719440e7 100644
--- a/doc/source/development.rst
+++ b/doc/source/development.rst
@@ -23,7 +23,7 @@ RLlib, Tune, Autoscaler, and most Python files do not require you to build and c
 
 .. code-block:: shell
 
-    pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-manylinux1_x86_64.whl
+    pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
 
 2. Fork and clone the project to your machine. Connect your repository to the upstream (main project) ray repository.
 
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
index c7bbec768..4b75c8957 100644
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,9 +51,9 @@ instead of the ones above:
 `Linux Python 3.6`_  `MacOS Python 3.6`_  `Windows Python 3.6`_
 ===================  ===================  ======================
 
-.. _`Linux Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-manylinux1_x86_64.whl
-.. _`Linux Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux1_x86_64.whl
-.. _`Linux Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
+.. _`Linux Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
+.. _`Linux Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
+.. _`Linux Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
 
 .. _`MacOS Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-macosx_10_13_x86_64.whl
 .. _`MacOS Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-macosx_10_13_intel.whl
diff --git a/python/README-building-wheels.md b/python/README-building-wheels.md
index 34158d5e1..346af7e9f 100644
--- a/python/README-building-wheels.md
+++ b/python/README-building-wheels.md
@@ -1,4 +1,4 @@
-# Building manylinux1 wheels
+# Building manylinux2014 wheels
 
 **WARNING:** To cause everything to be rebuilt, this script will delete ALL changes to the
 repository, including both changes to tracked files, and ANY untracked files.
@@ -9,7 +9,7 @@ produce .whl files owned by root.
 Inside the root directory (i.e., one level above this python directory), run
 
 ```
-docker run -e TRAVIS_COMMIT=<commit_number_to_use> --rm -w /ray -v `pwd`:/ray -ti rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh
+docker run -e TRAVIS_COMMIT=<commit_number_to_use> --rm -w /ray -v `pwd`:/ray -ti quay.io/pypa/manylinux2014_x86_64  /ray/python/build-wheel-manylinux2014.sh
 ```
 
 The wheel files will be placed in the .whl directory.
diff --git a/python/build-wheel-manylinux1.sh b/python/build-wheel-manylinux2014.sh
similarity index 76%
rename from python/build-wheel-manylinux1.sh
rename to python/build-wheel-manylinux2014.sh
index 11bf572b9..1791573c6 100755
--- a/python/build-wheel-manylinux1.sh
+++ b/python/build-wheel-manylinux2014.sh
@@ -22,18 +22,25 @@ NUMPY_VERSIONS=("1.14.5"
                 "1.14.5"
                 "1.14.5")
 
-sudo apt-get install unzip
-/ray/ci/travis/install-bazel.sh
+yum -y install unzip zip sudo
+yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel xz
+
+/ray/ci/travis/install-bazel.sh
+# Put bazel into the PATH if building Bazel from source
+# export PATH=/root/bazel-3.2.0/output:$PATH:/root/bin
+
+# If converting down to manylinux2010, the following configuration should
+# be set for bazel
+#echo "build --config=manylinux2010" >> /root/.bazelrc
+echo "build --incompatible_linkopts_to_linklibs" >> /root/.bazelrc
 
-# Put bazel into the PATH
-export PATH=$PATH:/root/bin
 
 # Install and use the latest version of Node.js in order to build the dashboard.
 set +x
 curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.34.0/install.sh | bash
 source "$HOME"/.nvm/nvm.sh
-nvm install $NODE_VERSION
-nvm use node
+nvm install "$NODE_VERSION"
+nvm use "$NODE_VERSION"
 
 # Build the dashboard so its static assets can be included in the wheel.
 # TODO(mfitton): switch this back when deleting old dashboard code.
@@ -66,7 +73,8 @@ for ((i=0; i<${#PYTHONS[@]}; ++i)); do
       exit 1
     fi
 
-    PATH=/opt/python/${PYTHON}/bin:$PATH /opt/python/"${PYTHON}"/bin/python setup.py bdist_wheel
+    PATH=/opt/python/${PYTHON}/bin:/root/bazel-3.2.0/output:$PATH \
+    /opt/python/"${PYTHON}"/bin/python setup.py bdist_wheel
     # In the future, run auditwheel here.
     mv dist/*.whl ../.whl/
   popd
@@ -76,6 +84,6 @@ done
 # hack, we should use auditwheel instead.
 for path in .whl/*.whl; do
   if [ -f "${path}" ]; then
-    mv "${path}" "${path//linux/manylinux1}"
+    mv "${path}" "${path//linux/manylinux2014}"
   fi
 done
diff --git a/python/ray/autoscaler/aws/defaults.yaml b/python/ray/autoscaler/aws/defaults.yaml
index 3ecd2ea5a..a274a9fc8 100644
--- a/python/ray/autoscaler/aws/defaults.yaml
+++ b/python/ray/autoscaler/aws/defaults.yaml
@@ -123,7 +123,7 @@ setup_commands:
     # has your Ray repo pre-cloned. Then, you can replace the pip installs
     # below with a git checkout <your_sha> (and possibly a recompile).
     - echo 'export PATH="$HOME/anaconda3/envs/tensorflow_p36/bin:$PATH"' >> ~/.bashrc
-    - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
+    - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
     # Consider uncommenting these if you also want to run apt-get commands during setup
     # - sudo pkill -9 apt-get || true
     # - sudo pkill -9 dpkg || true
diff --git a/python/ray/autoscaler/aws/example-full.yaml b/python/ray/autoscaler/aws/example-full.yaml
index f75e1ce88..7343cee0b 100644
--- a/python/ray/autoscaler/aws/example-full.yaml
+++ b/python/ray/autoscaler/aws/example-full.yaml
@@ -139,7 +139,7 @@ setup_commands: []
     # has your Ray repo pre-cloned. Then, you can replace the pip installs
     # below with a git checkout <your_sha> (and possibly a recompile).
     # Uncomment the following line if you want to run the nightly version of ray (as opposed to the latest)
-    # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux1_x86_64.whl
+    # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
 
 # Custom commands that will be run on the head node after common setup.
 head_setup_commands: []
diff --git a/python/ray/autoscaler/aws/example-gpu-docker.yaml b/python/ray/autoscaler/aws/example-gpu-docker.yaml
index 67b5e5b22..b65e56464 100644
--- a/python/ray/autoscaler/aws/example-gpu-docker.yaml
+++ b/python/ray/autoscaler/aws/example-gpu-docker.yaml
@@ -101,8 +101,8 @@ file_mounts: {
 # List of shell commands to run to set up nodes.
 # NOTE: rayproject/ray:latest has ray latest bundled
 setup_commands: []
-    # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
-    # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux1_x86_64.whl
+    # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
+    # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
 
 # Custom commands that will be run on the head node after common setup.
 head_setup_commands:
diff --git a/python/ray/autoscaler/aws/example-java.yaml b/python/ray/autoscaler/aws/example-java.yaml
index a217c4009..e9bd11ad3 100644
--- a/python/ray/autoscaler/aws/example-java.yaml
+++ b/python/ray/autoscaler/aws/example-java.yaml
@@ -63,7 +63,7 @@ initialization_commands:
 setup_commands:
     - sudo apt-get install -y python3 python3-pip
     - python3 -m pip install --upgrade pip
-    - python3 -m pip install https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
+    - python3 -m pip install https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
 # Custom commands that will be run on the head node after common setup.
 head_setup_commands:
     - python3 -m pip install boto3==1.4.8  # 1.4.8 adds InstanceMarketOptions
diff --git a/python/ray/autoscaler/azure/defaults.yaml b/python/ray/autoscaler/azure/defaults.yaml
index 8fbf11930..75cb0e182 100644
--- a/python/ray/autoscaler/azure/defaults.yaml
+++ b/python/ray/autoscaler/azure/defaults.yaml
@@ -123,7 +123,7 @@ setup_commands:
     - echo 'eval "$(conda shell.bash hook)"' >> ~/.bashrc
     # - echo 'conda activate py37_pytorch' >> ~/.bashrc
     - echo 'conda activate py37_tensorflow' >> ~/.bashrc
-    - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux1_x86_64.whl
+    - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
     # Consider uncommenting these if you also want to run apt-get commands during setup
     # - sudo pkill -9 apt-get || true
     # - sudo pkill -9 dpkg || true
diff --git a/python/ray/autoscaler/azure/example-full.yaml b/python/ray/autoscaler/azure/example-full.yaml
index b007b0b85..c5466ccda 100644
--- a/python/ray/autoscaler/azure/example-full.yaml
+++ b/python/ray/autoscaler/azure/example-full.yaml
@@ -139,7 +139,7 @@ setup_commands:
     # Uncomment the following line if you want to run the nightly version of ray (as opposed to the latest)
     - echo 'eval "$(conda shell.bash hook)"' >> ~/.bashrc
     - echo 'conda activate py37_tensorflow' >> ~/.bashrc
-    - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux1_x86_64.whl
+    - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
 
 # Custom commands that will be run on the head node after common setup.
 head_setup_commands:
diff --git a/python/ray/autoscaler/azure/example-gpu-docker.yaml b/python/ray/autoscaler/azure/example-gpu-docker.yaml
index 264c8f229..262eff1a8 100644
--- a/python/ray/autoscaler/azure/example-gpu-docker.yaml
+++ b/python/ray/autoscaler/azure/example-gpu-docker.yaml
@@ -78,7 +78,7 @@ file_mounts: {
 # List of shell commands to run to set up nodes.
 # NOTE: rayproject/ray:latest has ray latest bundled
 setup_commands: []
-#     - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux1_x86_64.whl
+#     - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
  
 # Custom commands that will be run on the head node after common setup.
 head_setup_commands:
diff --git a/python/ray/autoscaler/azure/example-gpu.yaml b/python/ray/autoscaler/azure/example-gpu.yaml
index 55133e50b..06cf42e6e 100644
--- a/python/ray/autoscaler/azure/example-gpu.yaml
+++ b/python/ray/autoscaler/azure/example-gpu.yaml
@@ -108,7 +108,7 @@ setup_commands:
     - echo 'eval "$(conda shell.bash hook)"' >> ~/.bashrc
     # - echo 'conda activate py37_pytorch' >> ~/.bashrc
     - echo 'conda activate py37_tensorflow' >> ~/.bashrc
-    - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux1_x86_64.whl
+    - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
     # Consider uncommenting these if you also want to run apt-get commands during setup
     # - sudo pkill -9 apt-get || true
     # - sudo pkill -9 dpkg || true
diff --git a/python/ray/autoscaler/gcp/defaults.yaml b/python/ray/autoscaler/gcp/defaults.yaml
index a119c6fcd..09794f0b1 100644
--- a/python/ray/autoscaler/gcp/defaults.yaml
+++ b/python/ray/autoscaler/gcp/defaults.yaml
@@ -142,7 +142,7 @@ setup_commands:
       && echo 'export PATH="$HOME/anaconda3/bin:$PATH"' >> ~/.profile
 
     # Install ray
-    - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux1_x86_64.whl
+    - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
 
 
 # Custom commands that will be run on the head node after common setup.
diff --git a/python/ray/autoscaler/gcp/example-full.yaml b/python/ray/autoscaler/gcp/example-full.yaml
index ef21fd27a..ba0b4fc87 100644
--- a/python/ray/autoscaler/gcp/example-full.yaml
+++ b/python/ray/autoscaler/gcp/example-full.yaml
@@ -148,7 +148,7 @@ setup_commands: []
     # has your Ray repo pre-cloned. Then, you can replace the pip installs
     # below with a git checkout <your_sha> (and possibly a recompile).
     # Uncomment the following line if you want to run the nightly version of ray (as opposed to the latest)
-    # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux1_x86_64.whl
+    # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
 
 
 # Custom commands that will be run on the head node after common setup.
diff --git a/python/ray/autoscaler/gcp/example-gpu-docker.yaml b/python/ray/autoscaler/gcp/example-gpu-docker.yaml
index fbac3f11e..ad6cd71de 100644
--- a/python/ray/autoscaler/gcp/example-gpu-docker.yaml
+++ b/python/ray/autoscaler/gcp/example-gpu-docker.yaml
@@ -131,8 +131,8 @@ initialization_commands:
 # List of shell commands to run to set up nodes.
 # NOTE: rayproject/ray:latest has ray latest bundled
 setup_commands: []
-    # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
-    # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux1_x86_64.whl
+    # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
+    # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
 
 # Custom commands that will be run on the head node after common setup.
 head_setup_commands:
diff --git a/python/ray/autoscaler/local/example-full.yaml b/python/ray/autoscaler/local/example-full.yaml
index 8213fbce3..55794ce36 100644
--- a/python/ray/autoscaler/local/example-full.yaml
+++ b/python/ray/autoscaler/local/example-full.yaml
@@ -94,7 +94,7 @@ setup_commands: []
     # has your Ray repo pre-cloned. Then, you can replace the pip installs
     # below with a git checkout <your_sha> (and possibly a recompile).
     # Uncomment the following line if you want to run the nightly version of ray (as opposed to the latest)
-    # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux1_x86_64.whl
+    # - pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
 
 # Custom commands that will be run on the head node after common setup.
 head_setup_commands: []
diff --git a/python/ray/nightly-wheels.yaml b/python/ray/nightly-wheels.yaml
index e7bf1c6a5..f2d52651a 100644
--- a/python/ray/nightly-wheels.yaml
+++ b/python/ray/nightly-wheels.yaml
@@ -1,7 +1,7 @@
 linux:
-  "3.8": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-manylinux1_x86_64.whl
-  "3.7": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux1_x86_64.whl
-  "3.6": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
+  "3.8": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-manylinux2014_x86_64.whl
+  "3.7": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl
+  "3.6": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
 
 darwin:
   "3.8": https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-macosx_10_13_x86_64.whl
diff --git a/python/ray/util/sgd/tf/examples/tf-example-sgd.yaml b/python/ray/util/sgd/tf/examples/tf-example-sgd.yaml
index 7078bbc3a..392d91937 100644
--- a/python/ray/util/sgd/tf/examples/tf-example-sgd.yaml
+++ b/python/ray/util/sgd/tf/examples/tf-example-sgd.yaml
@@ -48,7 +48,7 @@ worker_nodes:
 
 setup_commands:
     - conda install setuptools=45.1.0=py36_0 wrapt=1.11.2 --yes # workaround to fix wrapt error
-    - ray &> /dev/null || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
+    - ray &> /dev/null || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
     - pip install -U ray[tune]
     - pip install tensorflow-gpu==2.1.0
 
diff --git a/python/ray/util/sgd/torch/examples/example-sgd.yaml b/python/ray/util/sgd/torch/examples/example-sgd.yaml
index 2fb3cb036..743ec2488 100644
--- a/python/ray/util/sgd/torch/examples/example-sgd.yaml
+++ b/python/ray/util/sgd/torch/examples/example-sgd.yaml
@@ -43,7 +43,7 @@ worker_nodes:
         #     MaxPrice: "9.0"
 
 setup_commands:
-    - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
+    - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
     - pip install -U ipdb ray[rllib] torch torchvision
     # Install apex.
     # - rm -rf apex || true
diff --git a/python/ray/util/sgd/torch/examples/image_models/cluster.yaml b/python/ray/util/sgd/torch/examples/image_models/cluster.yaml
index 9646f0b48..14577161f 100644
--- a/python/ray/util/sgd/torch/examples/image_models/cluster.yaml
+++ b/python/ray/util/sgd/torch/examples/image_models/cluster.yaml
@@ -62,7 +62,7 @@ worker_nodes:
 
 setup_commands:
     # This replaces the standard anaconda Ray installation
-    - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
+    - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
     # Uncomment this and the filemount to update the Ray installation with your local Ray code
     # - rm -rf ./anaconda3/lib/python3.6/site-packages/ray/util/sgd/
     # - cp -rf ~/sgd ./anaconda3/lib/python3.6/site-packages/ray/util/
diff --git a/python/ray/util/sgd/torch/examples/segmentation/example.yaml b/python/ray/util/sgd/torch/examples/segmentation/example.yaml
index 8ad621d5e..c90c62a7d 100644
--- a/python/ray/util/sgd/torch/examples/segmentation/example.yaml
+++ b/python/ray/util/sgd/torch/examples/segmentation/example.yaml
@@ -42,7 +42,7 @@ worker_nodes:
 
 setup_commands:
     # This replaces the standard anaconda Ray installation
-    - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
+    - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
     # Uncomment this and the filemount to update the Ray installation with your local Ray code
     # - rm -rf ./anaconda3/lib/python3.6/site-packages/ray/util/sgd/
     # - cp -rf ~/sgd ./anaconda3/lib/python3.6/site-packages/ray/util/
diff --git a/python/ray/util/sgd/torch/examples/sgd-development.yaml b/python/ray/util/sgd/torch/examples/sgd-development.yaml
index 8c07fc6bc..d3c089cc8 100644
--- a/python/ray/util/sgd/torch/examples/sgd-development.yaml
+++ b/python/ray/util/sgd/torch/examples/sgd-development.yaml
@@ -62,7 +62,7 @@ worker_nodes:
 
 setup_commands:
     # This replaces the standard anaconda Ray installation
-    - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
+    - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
     # Uncomment this and the filemount to update the Ray installation with your local Ray code
     # - rm -rf ./anaconda3/lib/python3.6/site-packages/ray/util/sgd/
     # - cp -rf ~/sgd ./anaconda3/lib/python3.6/site-packages/ray/util/
diff --git a/python/ray/util/sgd/torch/examples/transformers/cluster.yaml b/python/ray/util/sgd/torch/examples/transformers/cluster.yaml
index e2a168a81..2aeb7087d 100644
--- a/python/ray/util/sgd/torch/examples/transformers/cluster.yaml
+++ b/python/ray/util/sgd/torch/examples/transformers/cluster.yaml
@@ -47,7 +47,7 @@ worker_nodes:
 
 setup_commands:
     # This replaces the standard anaconda Ray installation
-    - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
+    - ray || pip install -U https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl
     - pip install -q tqdm
 
     # Installing this without -U to make sure we don't replace the existing Ray installation
diff --git a/python/setup.py b/python/setup.py
index 0ed74933e..3bba134f5 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -178,7 +178,8 @@ def is_invalid_windows_platform():
 # (~/.bazel/bin/bazel) if it isn't found.
 def bazel_invoke(invoker, cmdline, *args, **kwargs):
     home = os.path.expanduser("~")
-    candidates = ["bazel"]
+    first_candidate = os.getenv("BAZEL_PATH", "bazel")
+    candidates = [first_candidate]
     if sys.platform == "win32":
         mingw_dir = os.getenv("MINGW_DIR")
         if mingw_dir:
@@ -191,6 +192,7 @@ def bazel_invoke(invoker, cmdline, *args, **kwargs):
             result = invoker([cmd] + cmdline, *args, **kwargs)
             break
         except IOError:
+            raise
             if i >= len(candidates) - 1:
                 raise
     return result
diff --git a/release/long_running_distributed_tests/run.sh b/release/long_running_distributed_tests/run.sh
index a2b7070f8..386416a08 100755
--- a/release/long_running_distributed_tests/run.sh
+++ b/release/long_running_distributed_tests/run.sh
@@ -42,7 +42,7 @@ echo "commit: $commit"
 echo "branch: $ray_branch"
 echo "workload: $workload"
 
-wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl"
+wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux2014_x86_64.whl"
 
 conda uninstall -y terminado || true
 pip install -U pip
diff --git a/release/long_running_tests/run.sh b/release/long_running_tests/run.sh
index e9288de1c..a376a8124 100644
--- a/release/long_running_tests/run.sh
+++ b/release/long_running_tests/run.sh
@@ -48,7 +48,7 @@ echo "commit: $commit"
 echo "branch: $ray_branch"
 echo "workload: $workload"
 
-wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl"
+wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux2014_x86_64.whl"
 
 echo set-window-option -g mouse on > ~/.tmux.conf
 echo 'termcapinfo xterm* ti@:te@' > ~/.screenrc
diff --git a/release/microbenchmark/run.sh b/release/microbenchmark/run.sh
index 7aa2fd09d..cbb04fe6f 100755
--- a/release/microbenchmark/run.sh
+++ b/release/microbenchmark/run.sh
@@ -47,7 +47,7 @@ echo "commit: $commit"
 echo "branch: $ray_branch"
 echo "workload: $workload"
 
-wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp38-cp38-manylinux1_x86_64.whl"
+wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp38-cp38-manylinux2014_x86_64.whl"
 
 
 echo set-window-option -g mouse on > ~/.tmux.conf
diff --git a/release/rllib_tests/regression_tests/run.sh b/release/rllib_tests/regression_tests/run.sh
index cd4afe1d0..ce932a732 100755
--- a/release/rllib_tests/regression_tests/run.sh
+++ b/release/rllib_tests/regression_tests/run.sh
@@ -41,7 +41,7 @@ echo "commit: $commit"
 echo "branch: $ray_branch"
 echo "workload: ignored"
 
-wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl"
+wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux2014_x86_64.whl"
 
 conda uninstall -y terminado
 source activate tensorflow_p36 && pip install -U pip
diff --git a/release/rllib_tests/stress_tests/run.sh b/release/rllib_tests/stress_tests/run.sh
index 1608e32de..b038de9fb 100755
--- a/release/rllib_tests/stress_tests/run.sh
+++ b/release/rllib_tests/stress_tests/run.sh
@@ -42,7 +42,7 @@ echo "commit: $commit"
 echo "branch: $ray_branch"
 echo "workload: ignored"
 
-wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl"
+wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux2014_x86_64.whl"
 
 conda uninstall -y terminado
 source activate tensorflow_p36 && pip install -U pip
diff --git a/release/rllib_tests/unit_gpu_tests/run.sh b/release/rllib_tests/unit_gpu_tests/run.sh
index 1d0a554a8..db468e789 100755
--- a/release/rllib_tests/unit_gpu_tests/run.sh
+++ b/release/rllib_tests/unit_gpu_tests/run.sh
@@ -42,7 +42,7 @@ echo "commit: $commit"
 echo "branch: $ray_branch"
 echo "workload: ignored"
 
-wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl"
+wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux2014_x86_64.whl"
 
 conda uninstall -y terminado
 source activate tensorflow_p36 && pip install -U pip
diff --git a/release/stress_tests/cluster.yaml b/release/stress_tests/cluster.yaml
index 11ac15b88..6f7449ae0 100644
--- a/release/stress_tests/cluster.yaml
+++ b/release/stress_tests/cluster.yaml
@@ -101,7 +101,7 @@ setup_commands: []
     - pip install terminado
     - pip install boto3==1.4.8 cython==0.29.0
     # - cd ray/python; git checkout master; git pull; pip install -e . --verbose
-    - "pip install https://s3-us-west-2.amazonaws.com/ray-wheels/{{ray_branch}}/{{commit}}/ray-{{ray_version}}-cp36-cp36m-manylinux1_x86_64.whl"
+    - "pip install https://s3-us-west-2.amazonaws.com/ray-wheels/{{ray_branch}}/{{commit}}/ray-{{ray_version}}-cp36-cp36m-manylinux2014_x86_64.whl"
 
 # Custom commands that will be run on the head node after common setup.
 head_setup_commands: []
diff --git a/release/stress_tests/run.sh b/release/stress_tests/run.sh
index 183a89a03..f58eacfc8 100644
--- a/release/stress_tests/run.sh
+++ b/release/stress_tests/run.sh
@@ -38,7 +38,7 @@ echo "commit: $commit"
 echo "branch: $ray_branch"
 echo "workload: $workload"
 
-wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux1_x86_64.whl"
+wheel="https://s3-us-west-2.amazonaws.com/ray-wheels/$ray_branch/$commit/ray-$ray_version-cp36-cp36m-manylinux2014_x86_64.whl"
 
 # Install Anaconda.
 wget --quiet https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh || true