Factor out Travis 'install' sections for use with GitHub Actions (#7988)

2026-07-17 11:32:33 +08:00 · 2020-04-15 08:10:22 -07:00
parent 428516056a
commit ba00c29b67
21 changed files with 519 additions and 384 deletions
@@ -17,7 +17,7 @@ jobs:
      - name: Checkout repository
        uses: actions/checkout@v1
        with:
-          fetch-depth: 1
+          fetch-depth: 0
      - name: Run Jenkins Tests
        shell: bash -e -o pipefail -l {0}
        env:
@@ -11,48 +11,38 @@ jobs:
      matrix:
        # Useful info: https://help.github.com/en/articles/workflow-syntax-for-github-actions
        include:
-          - name: ubuntu-clang
+          - name: ubuntu
            os: ubuntu-16.04
-            compiler: clang
-          - name: windows-clang
+          - name: windows
            os: windows-2019
-            compiler: clang-cl
-          - name: macos-clang
+          - name: macos
            os: macos-10.15
-            compiler: clang
+    env:
+      PYTHON: 3.6
    steps:
-    - name: Configure Git
+    - name: Pre-checkout system configuration
      # We add -l to process .bashrc, but need -e -o pipefail for consistency with GitHub Actions's default behavior.
      shell: bash -e -o pipefail -l {0}
      run: |
+        # Note: only configure settings here that are required _before_ the repository is cloned (such as Git settings).
+        if [ ! -f ~/.profile ] && [ ! -f ~/.bash_profile ]; then  # missing on some machines, but needed for ~/.bashrc
+          echo 'if [ -n "${BASH_VERSION}" ] && [ -f ~/.bashrc ]; then . ~/.bashrc; fi' >> ~/.profile
+        fi
        git config --global core.symlinks true && git config --global core.autocrlf false
        if command -v dpkg > /dev/null; then sudo dpkg-reconfigure debconf -f noninteractive -p high; fi
    - name: Checkout repository
      uses: actions/checkout@v2
      with:
-        fetch-depth: 1
-    - name: Setup Bazel
+        # we need full history to diff against the original for linting etc.
+        fetch-depth: 0
+    - name: Run CI script
      shell: bash -e -o pipefail -l {0}
      env:
        BAZEL_CACHE_CREDENTIAL_B64: ${{ secrets.BAZEL_CACHE_CREDENTIAL_B64 }}
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: ./ci/travis/install-bazel.sh
-    - name: Install C/C++ toolchains
-      shell: bash -e -o pipefail -l {0}
-      env:
-        CC: ${{ matrix.compiler }}
-      run: ./ci/travis/install-toolchains.sh "${CC%%-*}"
-    - name: Install MSYS2 (only needed if some shell commands are missing)
-      if: false && runner.os == 'Windows'
-      uses: numworks/setup-msys2@v1
-      with:
-        update: false
-    - name: Perform build
-      shell: bash -e -o pipefail -l {0}
      run: |
-        bazel build \
-          --keep_going \
-          "//:*"
+        . ./ci/travis/ci.sh init
+        . ./ci/travis/ci.sh build
    - name: Run Clang Include-What-You-Use
      continue-on-error: true
      if: runner.os == 'Linux'
@@ -8,27 +8,35 @@ matrix:
      env:
        - PYTHON=3.6
        - PYTHONWARNINGS=ignore
+        - RAY_DEFAULT_BUILD=1
+        - RAY_CYTHON_EXAMPLES=1
+      before_install:
+        - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED
+      install:
+        - . ./ci/travis/ci.sh build

    - os: osx
      osx_image: xcode7
      env:
        - PYTHON=3.6
        - PYTHONWARNINGS=ignore
+        - RAY_DEFAULT_BUILD=1
+        - RAY_CYTHON_EXAMPLES=1
+      before_install:
+        - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED
+      install:
+        - . ./ci/travis/ci.sh build

    - os: linux
      env:
        - JDK='Oracle JDK 8'
        - PYTHON=3.6 PYTHONWARNINGS=ignore
        - RAY_INSTALL_JAVA=1
+      before_install:
+        - . ./ci/travis/ci.sh init RAY_CI_JAVA_AFFECTED
      install:
-        - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
-        - if [ $RAY_CI_JAVA_AFFECTED != "1" ]; then exit; fi
-        - ./ci/travis/install-bazel.sh
-        - ./ci/suppress_output ./ci/travis/install-dependencies.sh
-        - export PATH="$HOME/miniconda/bin:$PATH"
-        - ./ci/suppress_output ./ci/travis/install-ray.sh
+        - . ./ci/travis/ci.sh build
      script:
-        - if [ $RAY_CI_JAVA_AFFECTED != "1" ]; then exit; fi
        - ./java/test.sh

    - os: linux
@@ -37,14 +45,10 @@ matrix:
        - JDK='Oracle JDK 8'
        - RAY_INSTALL_JAVA=1
        - PYTHON=3.6 PYTHONWARNINGS=ignore
+      before_install:
+        - . ./ci/travis/ci.sh init RAY_CI_STREAMING_PYTHON_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
      install:
-        - python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py
-        - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
-        - if [[ $RAY_CI_STREAMING_PYTHON_AFFECTED != "1" && $RAY_CI_STREAMING_JAVA_AFFECTED != "1" ]]; then exit; fi
-        - ./ci/travis/install-bazel.sh
-        - ./ci/suppress_output ./ci/travis/install-dependencies.sh
-        - export PATH="$HOME/miniconda/bin:$PATH"
-        - ./ci/suppress_output ./ci/travis/install-ray.sh
+        - . ./ci/travis/ci.sh build
      script:
        # Streaming cpp test.
        - if [ $RAY_CI_STREAMING_CPP_AFFECTED == "1" ]; then bazel test --config=ci --test_output=all //streaming:all && bash streaming/src/test/run_streaming_queue_test.sh; fi
@@ -58,17 +62,12 @@ matrix:
        - PYTHON=3.6 PYTHONWARNINGS=ignore
        - RAY_INSTALL_JAVA=1
        - RAY_GCS_SERVICE_ENABLED=false
+        - RAY_CYTHON_EXAMPLES=1
+      before_install:
+        - . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED
      install:
-        - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
-        - if [ $RAY_CI_ONLY_RLLIB_AFFECTED == "1" ]; then exit; fi
-        - ./ci/travis/install-bazel.sh
-        - ./ci/suppress_output ./ci/travis/install-dependencies.sh
-        - export PATH="$HOME/miniconda/bin:$PATH"
-        - ./ci/suppress_output ./ci/travis/install-ray.sh
-        - ./ci/suppress_output ./ci/travis/install-cython-examples.sh
-        - eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=master bash)"
+        - . ./ci/travis/ci.sh build
      script:
-        - if [ $RAY_CI_ONLY_RLLIB_AFFECTED == "1" ]; then exit; fi
        - ./ci/suppress_output bash src/ray/test/run_core_worker_tests.sh
        - ./ci/suppress_output bash streaming/src/test/run_streaming_queue_test.sh
        - ./java/test.sh
@@ -80,18 +79,12 @@ matrix:
        - PYTHON=3.6 PYTHONWARNINGS=ignore
        - RAY_INSTALL_JAVA=1
        - RAY_GCS_SERVICE_ENABLED=false
+        - RAY_CYTHON_EXAMPLES=1
+      before_install:
+        - . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED
      install:
-        - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
-        - if [ $RAY_CI_ONLY_RLLIB_AFFECTED == "1" ]; then exit; fi
-        - ./ci/travis/install-bazel.sh
-        - ./ci/suppress_output ./ci/travis/install-dependencies.sh
-        - export PATH="$HOME/miniconda/bin:$PATH"
-        - ./ci/suppress_output ./ci/travis/install-ray.sh
-        - ./ci/suppress_output ./ci/travis/install-cython-examples.sh
-        - eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=master bash)"
+        - . ./ci/travis/ci.sh build
      script:
-        - if [ $RAY_CI_ONLY_RLLIB_AFFECTED == "1" ]; then exit; fi
-        - export PATH="$HOME/miniconda/bin:$PATH"
        - ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/tests/...

    - os: linux
@@ -99,14 +92,13 @@ matrix:
        - LINT=1
        - PYTHONWARNINGS=ignore
      before_install:
-        - sudo apt-get update -qq
-      install: []
+        - . ./ci/travis/ci.sh init
+      install:
+        - . ./ci/travis/ci.sh build
      script:
        - ./ci/travis/check-git-clang-format-output.sh
        # Try generating Sphinx documentation. To do this, we need to install
        # Ray first.
-        - ./ci/travis/install-dependencies.sh
-        - export PATH="$HOME/miniconda/bin:$PATH"
        - cd doc
        # readthedocs has an antiquated build env.
        # This is a best effort to reproduce it locally to avoid doc build failures
@@ -147,24 +139,11 @@ matrix:
        - LINUX_WHEELS=1
        - PYTHONWARNINGS=ignore
        - RAY_INSTALL_JAVA=1
+      before_install:
+        - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED
      install:
-        - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
-        - if [ $RAY_CI_LINUX_WHEELS_AFFECTED != "1" ]; then exit; fi
-        - ./ci/suppress_output ./ci/travis/install-dependencies.sh
-
-        # Mount bazel cache dir to the docker container.
-        # For the linux wheel build, we use a shared cache between all
-        # wheels, but not between different travis runs, because that
-        # caused timeouts in the past. See the "cache: false" line below.
-        - export MOUNT_BAZEL_CACHE="-v $HOME/ray-bazel-cache:/root/ray-bazel-cache -e TRAVIS=true -e TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST -e encrypted_1c30b31fe1ee_key=$encrypted_1c30b31fe1ee_key -e encrypted_1c30b31fe1ee_iv=$encrypted_1c30b31fe1ee_iv"
-
-        # This command should be kept in sync with ray/python/README-building-wheels.md,
-        # except the `$MOUNT_BAZEL_CACHE` part.
-
-        - ./ci/suppress_output docker run --rm -w /ray -v `pwd`:/ray $MOUNT_BAZEL_CACHE -e TRAVIS_COMMIT=$TRAVIS_COMMIT -ti rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh
-
+        - . ./ci/travis/ci.sh build
      script:
-        - if [ $RAY_CI_LINUX_WHEELS_AFFECTED != "1" ]; then exit; fi
        # Explicitly sleep 60 seconds for logs to go through
        - ./ci/travis/test-wheels.sh || { cat /tmp/ray/session_latest/logs/* && sleep 60 && false; }
      cache: false
@@ -176,16 +155,11 @@ matrix:
        - MAC_WHEELS=1
        - PYTHONWARNINGS=ignore
        - RAY_INSTALL_JAVA=1
+      before_install:
+        - . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED
      install:
-        - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
-        - if [ $RAY_CI_MACOS_WHEELS_AFFECTED != "1" ]; then exit; fi
-
-        - ./ci/suppress_output ./ci/travis/install-dependencies.sh
-        # This command should be kept in sync with ray/python/README-building-wheels.md.
-        - ./ci/suppress_output ./python/build-wheel-macos.sh
+        - . ./ci/travis/ci.sh build
      script:
-        - if [ $RAY_CI_MACOS_WHEELS_AFFECTED != "1" ]; then exit; fi
-
        # Explicitly sleep 60 seconds for logs to go through
        - ./ci/travis/test-wheels.sh || { cat /tmp/ray/session_latest/logs/* && sleep 60 && false; }

@@ -199,15 +173,11 @@ matrix:
        - TORCH_VERSION=1.4
        - PYTHON=3.6
        - PYTHONWARNINGS=ignore
+      before_install:
+        - . ./ci/travis/ci.sh init RAY_CI_RLLIB_AFFECTED
      install:
-        - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
-        - if [ $RAY_CI_RLLIB_AFFECTED != "1" ]; then exit; fi
-        - ./ci/travis/install-bazel.sh
-        - ./ci/travis/install-dependencies.sh
-        - export PATH="$HOME/miniconda/bin:$PATH"
-        - ./ci/suppress_output ./ci/travis/install-ray.sh
+        - . ./ci/travis/ci.sh build
      script:
-        - if [ $RAY_CI_RLLIB_AFFECTED != "1" ]; then exit; fi
        - travis_wait 90 bazel test --config=ci --test_output=streamed --build_tests_only --test_tag_filters=learning_tests rllib/...

    # RLlib: Learning tests with tf=1.x (from rllib/tuned_examples/regression_tests/*.yaml).
@@ -221,15 +191,11 @@ matrix:
        - TORCH_VERSION=1.4
        - PYTHON=3.6
        - PYTHONWARNINGS=ignore
+      before_install:
+        - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
      install:
-        - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
-        - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
-        - ./ci/travis/install-bazel.sh
-        - ./ci/travis/install-dependencies.sh
-        - export PATH="$HOME/miniconda/bin:$PATH"
-        - ./ci/suppress_output ./ci/travis/install-ray.sh
+        - . ./ci/travis/ci.sh build
      script:
-        - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
        - travis_wait 90 bazel test --config=ci --test_output=streamed --build_tests_only --test_tag_filters=learning_tests rllib/...

    # RLlib: Quick Agent train.py runs (compilation & running, no(!) learning).
@@ -243,15 +209,11 @@ matrix:
        - TFP_VERSION=0.8
        - TORCH_VERSION=1.4
        - PYTHONWARNINGS=ignore
+      before_install:
+        - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
      install:
-        - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
-        - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
-        - ./ci/travis/install-bazel.sh
-        - ./ci/travis/install-dependencies.sh
-        - export PATH="$HOME/miniconda/bin:$PATH"
-        - ./ci/suppress_output ./ci/travis/install-ray.sh
+        - . ./ci/travis/ci.sh build
      script:
-        - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
        - travis_wait 60 bazel test --config=ci --build_tests_only --test_tag_filters=quick_train rllib/...
        # Test everything that does not have any of the "main" labels:
        # "learning_tests|quick_train|examples|tests_dir".
@@ -267,15 +229,11 @@ matrix:
        - TFP_VERSION=0.8
        - TORCH_VERSION=1.4
        - PYTHONWARNINGS=ignore
+      before_install:
+        - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
      install:
-        - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
-        - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
-        - ./ci/travis/install-bazel.sh
-        - ./ci/travis/install-dependencies.sh
-        - export PATH="$HOME/miniconda/bin:$PATH"
-        - ./ci/suppress_output ./ci/travis/install-ray.sh
+        - . ./ci/travis/ci.sh build
      script:
-        - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_A,examples_B rllib/...
        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_C rllib/...
        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_E,examples_L,examples_M,examples_P rllib/...
@@ -291,15 +249,11 @@ matrix:
        - TFP_VERSION=0.8
        - TORCH_VERSION=1.4
        - PYTHONWARNINGS=ignore
+      before_install:
+        - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
      install:
-        - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
-        - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
-        - ./ci/travis/install-bazel.sh
-        - ./ci/travis/install-dependencies.sh
-        - export PATH="$HOME/miniconda/bin:$PATH"
-        - ./ci/suppress_output ./ci/travis/install-ray.sh
+        - . ./ci/travis/ci.sh build
      script:
-        - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I rllib/...

    # RLlib: tests_dir: Everything in rllib/tests/ directory (J-Z).
@@ -312,15 +266,11 @@ matrix:
        - TFP_VERSION=0.8
        - TORCH_VERSION=1.4
        - PYTHONWARNINGS=ignore
+      before_install:
+        - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
      install:
-        - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
-        - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
-        - ./ci/travis/install-bazel.sh
-        - ./ci/travis/install-dependencies.sh
-        - export PATH="$HOME/miniconda/bin:$PATH"
-        - ./ci/suppress_output ./ci/travis/install-ray.sh
+        - . ./ci/travis/ci.sh build
      script:
-        - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi
        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_J,tests_dir_K,tests_dir_L,tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z rllib/...

    # Cpp worker test
@@ -328,32 +278,14 @@ matrix:
      env:
        - TESTSUITE=cpp_worker
        - PYTHON=3.6
+      before_install:
+        - . ./ci/travis/ci.sh init
      install:
-        - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
-        - ./ci/travis/install-bazel.sh
-        - ./ci/suppress_output ./ci/travis/install-dependencies.sh
-        - export PATH="$HOME/miniconda/bin:$PATH"
-        - ./ci/suppress_output ./ci/travis/install-ray.sh
+        - . ./ci/travis/ci.sh build
      script:
        - bazel test --config=ci //cpp:all --build_tests_only --test_output=streamed

-install:
-  - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py`
-  - if [ $RAY_CI_SERVE_AFFECTED != "1" ] && [ $RAY_CI_TUNE_AFFECTED != "1" ] && [ $RAY_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
-
-  - ./ci/travis/install-bazel.sh
-  - ./ci/suppress_output ./ci/travis/install-dependencies.sh
-  - export PATH="$HOME/miniconda/bin:$PATH"
-  - ./ci/suppress_output ./ci/travis/install-ray.sh
-  - ./ci/suppress_output ./ci/travis/install-cython-examples.sh
-
-  - eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=master bash)"
-
 script:
-  - if [ $RAY_CI_SERVE_AFFECTED != "1" ] && [ $RAY_CI_TUNE_AFFECTED != "1" ] && [ $RAY_CI_PYTHON_AFFECTED != "1" ]; then exit; fi
-
-  - export PATH="$HOME/miniconda/bin:$PATH"
-
  # raylet integration tests
  - ./ci/suppress_output bash src/ray/test/run_core_worker_tests.sh
  - ./ci/suppress_output bash src/ray/test/run_object_manager_tests.sh
@@ -1520,7 +1520,6 @@ genrule(
    ],
    outs = ["ray_pkg.out"],
    cmd = """
-        set -x &&
        WORK_DIR="$$(pwd)" &&
        RAYDIR="$$WORK_DIR/python/ray"
        RAYLET="$(location python/ray/_raylet.so)"
@@ -123,7 +123,7 @@ pushd $WORK_DIR
 git clone https://github.com/suquark/pickle5-backport
 pushd pickle5-backport
  git checkout 8ffe41ceba9d5e2ce8a98190f6b3d2f3325e5a72
-  "$PYTHON_EXECUTABLE" setup.py bdist_wheel
+  CC=gcc "$PYTHON_EXECUTABLE" setup.py bdist_wheel
  unzip -o dist/*.whl -d "$ROOT_DIR/python/ray/pickle5_files"
 popd
 popd
@@ -0,0 +1,51 @@
+# CI process
+
+_This document is a work-in-progress._
+_Please double-check file/function/etc. names for changes, as this document may be out of sync._
+
+### Dependencies
+
+All dependencies (e.g. `apt`, `pip`) should be installed in `install_dependencies()`, following the same pattern as
+those that already exist.
+
+Once a dependency is added/removed, please ensure that if `reload_env` (or similar) is updated if it exists, as CI
+systems differ on when `~/.bashrc` et al. are reloaded, if at all. (And they are not necessarily idempotent.)
+
+### Bazel, environment variables, and caching
+
+Any environment variables passed to Bazel actions (e.g. `PATH`) should be idempotent to hit the Bazel cache.
+
+If a different `PATH` gets passed to a Bazel action, Bazel will not hit the cache, and you might trigger a full rebuild
+when you really expect an incremental (or no-op) build for an option (say `pip install -e .` after `bazel build //...`).
+
+### Invocation
+
+The CI system (such as Travis) must _source_ (_not_ execute) `ci/travis/ci.sh` and pass the action(s) to execute.
+The script either handles the work or dispatches it to other script(s) as it deems appropriate.
+This helps ensure any environment setup/teardown is handled appropriately.
+
+### Development best practices & pitfalls (read before adding a new script)
+
+Before adding new scripts, please read this section.
+
+First, please consider modifying an existing script instead (e.g. add your code as a separate function).
+Adding new scripts has a number of pitfalls that easily take hours (even days) to track down and fix:
+
+- When _calling_ other scripts (as executables), environment variables (like `PATH`) _cannot_ propagate back up to the
+  caller. Often, the caller expects such variables to be updated.
+
+- When _sourcing_ other scripts, global state (`ROOT_DIR`, `main`, `set -e`, etc.) may be overwritten silently, causing
+  unexpected behavior.
+
+The following practices can avoid such pitfalls while maintaining intuitive control flow:
+
+- Put all environment-modifying functions in the _same_ shell script, so that their invocation behaves intuitively.
+  (The sheer length of the script is a secondary concern and can be mitigated by keeping functions modular.)
+
+- Avoid adding new scripts if possible. If it's necessary that you do so, call them instead of sourcing them.
+
+- Always add code inside a function, not at global scope. Use `local` for variables where it makes sense.
+  However, be careful and know the shell rules: for example, e.g. `local x=$(false)` succeeds even under `set -e`.
+
+Ultimately, it's best to _only_ add new scripts if they might need to be executed directly by _non-CI_ code,
+as in that case, they should probably not use CI entrypoints (which assume exclusive control over the machine).
@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+
+{ SHELLOPTS_STACK="${SHELLOPTS_STACK-}|$(set +o); set -$-"; } 2> /dev/null  # Push caller's shell options (quietly)
+
+unset -f cd  # Travis defines this on Mac for RVM, but it floods the trace log and isn't relevant for us
+
+set -eo pipefail && if [ -n "${OSTYPE##darwin*}" ]; then set -ux; fi  # some options interfere with Travis's RVM on Mac
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)"
+WORKSPACE_DIR="${ROOT_DIR}/../.."
+
+# If provided the names of one or more environment variables, returns success if any of them is triggered.
+# Usage: should_run_job [VAR_NAME]...
+should_run_job() {
+  local skip=0
+  if [ -n "${1-}" ]; then  # were any triggers provided? (if not, then the job will always run)
+    local envvar active_triggers=()
+    for envvar in "$@"; do
+      if [ "${!envvar}" = 1 ]; then
+        active_triggers+="${envvar}=${!envvar}"  # success! we found at least one of the given triggers is occurring
+      fi
+    done
+    if [ 0 -eq "${#active_triggers[@]}" ]; then
+      echo "Job is not triggered by any of $1; skipping job."
+      skip=1
+    else
+      echo "Job is triggered by: ${#active_triggers[*]}"
+    fi
+  fi
+  return "${skip}"
+}
+
+reload_env() {
+  # TODO: We should really just use a new login shell instead of doing this manually.
+  # Otherwise we might source a script that isn't idempotent (e.g. one that blindly prepends to PATH).
+  { local set_x="${-//[^x]/}"; } 2> /dev/null  # save set -x to suppress noise
+  set +x
+  local to_add="$HOME/miniconda/bin" old_path=":${PATH}:"
+  if [ "${old_path}" = "${old_path##*:${to_add}:*}" ]; then
+    PATH="${to_add}:$PATH"
+  fi
+  test -z "${set_x}" || set -x  # restore set -x
+  if [ "${OSTYPE}" = msys ]; then
+    PATH="${PATH// :/:}"  # HACK: Work around https://github.com/actions/virtual-environments/issues/635#issue-589318442
+  fi
+  PYTHON3_BIN_PATH=python
+  export PATH PYTHON3_BIN_PATH
+}
+
+# Initializes the environment for the current job. Performs the following tasks:
+# - Calls 'exit 0' to quickly exit if provided a list of job names and none of them has been triggered.
+# - Sets variables to indicate the job names that have been triggered.
+#   Note: Please avoid exporting these variables. Instead, source any callees that need to use them.
+#   This helps reduce implicit coupling of callees to their parents, as they will be unable to run when not sourced, (especially with set -u).
+# - Installs dependencies for the current job.
+# - Exports any environment variables necessary to run the build.
+# Usage: init [JOB_NAMES]
+# - JOB_NAMES (optional): Comma-separated list of job names to trigger on.
+init() {
+  local job_names="${1-}"
+
+  local variable_definitions
+  variable_definitions=($(python "${ROOT_DIR}"/determine_tests_to_run.py))
+  { declare "${variable_definitions[@]}"; } > /dev/null 2> /dev/null
+
+  if ! (set +x && should_run_job ${job_names//,/ }); then
+    exit 0
+  fi
+
+  if [ "${OSTYPE}" = msys ]; then
+    export USE_CLANG_CL=1
+  fi
+
+  local wheels="${LINUX_WHEELS-}${MAC_WHEELS-}"
+  if [ -z "${wheels}" ]; then  # NOT building wheels
+    "${ROOT_DIR}"/install-bazel.sh
+  fi
+  . "${ROOT_DIR}"/install-dependencies.sh
+  reload_env  # We just modified our environment; reload it so we can continue
+}
+
+build() {
+  local wheels="${LINUX_WHEELS-}${MAC_WHEELS-}"
+  if [ -z "${wheels}" ]; then  # NOT building wheels
+    if [ "${LINT-}" != 1 ]; then  # NOT linting
+      bazel build -k "//:*"   # Do a full build first to ensure it passes
+      "${ROOT_DIR}"/install-ray.sh
+    fi
+  fi
+
+  if [ "${RAY_CYTHON_EXAMPLES-}" = 1 ]; then
+    "${ROOT_DIR}"/install-cython-examples.sh
+  fi
+
+  if [ "${RAY_DEFAULT_BUILD-}" = 1 ]; then
+    eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=master bash)"
+  fi
+
+  if [ "${LINUX_WHEELS-}" = 1 ]; then
+    # Mount bazel cache dir to the docker container.
+    # For the linux wheel build, we use a shared cache between all
+    # wheels, but not between different travis runs, because that
+    # caused timeouts in the past. See the "cache: false" line below.
+    local MOUNT_BAZEL_CACHE=(-v "${HOME}/ray-bazel-cache":/root/ray-bazel-cache -e TRAVIS=true -e TRAVIS_PULL_REQUEST="${TRAVIS_PULL_REQUEST}" -e encrypted_1c30b31fe1ee_key="${encrypted_1c30b31fe1ee_key-}" -e encrypted_1c30b31fe1ee_iv="${encrypted_1c30b31fe1ee_iv-}")
+
+    # This command should be kept in sync with ray/python/README-building-wheels.md,
+    # except the "${MOUNT_BAZEL_CACHE[@]}" part.
+    "${WORKSPACE_DIR}"/ci/suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" -ti rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh
+  fi
+
+  if [ "${MAC_WHEELS-}" = 1 ]; then
+    # This command should be kept in sync with ray/python/README-building-wheels.md.
+    "${WORKSPACE_DIR}"/ci/suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh
+  fi
+}
+
+"$@"
+
+{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null  # Pop caller's shell options (quietly)
@@ -3,11 +3,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function

+import json
 import os
 import re
 import subprocess
 import sys
-from functools import partial
 from pprint import pformat


@@ -25,7 +25,7 @@ def list_changed_files(commit_range):
        list: List of changed files within the commit range
    """

-    command = ["git", "diff", "--name-only", commit_range]
+    command = ["git", "diff", "--name-only", commit_range, "--"]
    out = subprocess.check_output(command)
    return [s.strip() for s in out.decode().splitlines() if s is not None]

@@ -45,9 +45,19 @@ if __name__ == "__main__":
    RAY_CI_STREAMING_PYTHON_AFFECTED = 0
    RAY_CI_STREAMING_JAVA_AFFECTED = 0

-    if os.environ["TRAVIS_EVENT_TYPE"] == "pull_request":
+    event_type = None
+    for key in ["GITHUB_EVENT_NAME", "TRAVIS_EVENT_TYPE"]:
+        event_type = os.getenv(key, event_type)

-        files = list_changed_files(os.environ["TRAVIS_COMMIT_RANGE"])
+    if event_type == "pull_request":
+
+        commit_range = os.getenv("TRAVIS_COMMIT_RANGE")
+        if commit_range is None:
+            with open(os.environ["GITHUB_EVENT_PATH"], "rb") as f:
+                event = json.loads(f.read())
+            base = event["pull_request"]["base"]["sha"]
+            commit_range = "{}...{}".format(base, event["after"])
+        files = list_changed_files(commit_range)

        print(pformat(files), file=sys.stderr)

@@ -138,25 +148,20 @@ if __name__ == "__main__":
        RAY_CI_ONLY_RLLIB_AFFECTED = 1

    # Log the modified environment variables visible in console.
-    for output_stream in [sys.stdout, sys.stderr]:
-        _print = partial(print, file=output_stream)
-        _print("export RAY_CI_TUNE_AFFECTED={}".format(RAY_CI_TUNE_AFFECTED))
-        _print("export RAY_CI_ONLY_RLLIB_AFFECTED={}"
-               .format(RAY_CI_ONLY_RLLIB_AFFECTED))
-        _print("export RAY_CI_RLLIB_AFFECTED={}".format(RAY_CI_RLLIB_AFFECTED))
-        _print("export RAY_CI_RLLIB_FULL_AFFECTED={}".format(
-            RAY_CI_RLLIB_FULL_AFFECTED))
-        _print("export RAY_CI_SERVE_AFFECTED={}".format(RAY_CI_SERVE_AFFECTED))
-        _print("export RAY_CI_JAVA_AFFECTED={}".format(RAY_CI_JAVA_AFFECTED))
-        _print(
-            "export RAY_CI_PYTHON_AFFECTED={}".format(RAY_CI_PYTHON_AFFECTED))
-        _print("export RAY_CI_LINUX_WHEELS_AFFECTED={}"
-               .format(RAY_CI_LINUX_WHEELS_AFFECTED))
-        _print("export RAY_CI_MACOS_WHEELS_AFFECTED={}"
-               .format(RAY_CI_MACOS_WHEELS_AFFECTED))
-        _print("export RAY_CI_STREAMING_CPP_AFFECTED={}"
-               .format(RAY_CI_STREAMING_CPP_AFFECTED))
-        _print("export RAY_CI_STREAMING_PYTHON_AFFECTED={}"
-               .format(RAY_CI_STREAMING_PYTHON_AFFECTED))
-        _print("export RAY_CI_STREAMING_JAVA_AFFECTED={}"
-               .format(RAY_CI_STREAMING_JAVA_AFFECTED))
+    print(" ".join([
+        "RAY_CI_TUNE_AFFECTED={}".format(RAY_CI_TUNE_AFFECTED),
+        "RAY_CI_ONLY_RLLIB_AFFECTED={}".format(RAY_CI_ONLY_RLLIB_AFFECTED),
+        "RAY_CI_RLLIB_AFFECTED={}".format(RAY_CI_RLLIB_AFFECTED),
+        "RAY_CI_RLLIB_FULL_AFFECTED={}".format(RAY_CI_RLLIB_FULL_AFFECTED),
+        "RAY_CI_SERVE_AFFECTED={}".format(RAY_CI_SERVE_AFFECTED),
+        "RAY_CI_JAVA_AFFECTED={}".format(RAY_CI_JAVA_AFFECTED),
+        "RAY_CI_PYTHON_AFFECTED={}".format(RAY_CI_PYTHON_AFFECTED),
+        "RAY_CI_LINUX_WHEELS_AFFECTED={}".format(RAY_CI_LINUX_WHEELS_AFFECTED),
+        "RAY_CI_MACOS_WHEELS_AFFECTED={}".format(RAY_CI_MACOS_WHEELS_AFFECTED),
+        "RAY_CI_STREAMING_CPP_AFFECTED={}".format(
+            RAY_CI_STREAMING_CPP_AFFECTED),
+        "RAY_CI_STREAMING_PYTHON_AFFECTED={}".format(
+            RAY_CI_STREAMING_PYTHON_AFFECTED),
+        "RAY_CI_STREAMING_JAVA_AFFECTED={}".format(
+            RAY_CI_STREAMING_JAVA_AFFECTED),
+    ]))
@@ -61,9 +61,7 @@ fi

 if [ "${TRAVIS-}" = true ]; then
  # Use bazel disk cache if this script is running in Travis.
-  mkdir -p "${HOME}/ray-bazel-cache"
  cat <<EOF >> "${HOME}/.bazelrc"
-build --disk_cache="${HOME}/ray-bazel-cache"
 build --show_timestamps  # Travis doesn't have an option to show timestamps, but GitHub Actions does
 # If we are in Travis, most of the compilation result will be cached.
 # This means we are I/O bounded. By default, Bazel set the number of concurrent
@@ -74,26 +72,6 @@ build --jobs=50
 EOF
 fi
 if [ -n "${GITHUB_WORKFLOW-}" ]; then
-  cat <<"EOF" >> "${HOME}/.profile"
-# Set up environment variables the CI user needs on login to run Bazel on each platform.
-if [ "${OSTYPE}" = "msys" ]; then
-  export USE_CLANG_CL=1
-  export MSYS2_ARG_CONV_EXCL="*"  # Don't let MSYS2 attempt to auto-translate arguments that look like paths
-  latest_python_bin=""  # Detect the system Python from the registry
-  for latest_python_bin in /proc/registry/HKEY_LOCAL_MACHINE/Software/Python/PythonCore/*/InstallPath/@; do
-    if [ -f "${latest_python_bin}" ]; then
-      read -r latest_python_bin < "${latest_python_bin}"
-      latest_python_bin="${latest_python_bin}\\"
-    else
-      latest_python_bin=""
-    fi
-  done
-  latest_python_bin="${latest_python_bin}python.exe"
-  if [ -f "${latest_python_bin}" ]; then
-    export PYTHON2_BIN_PATH="${latest_python_bin}" PYTHON3_BIN_PATH="${latest_python_bin}"
-  fi
-fi
-EOF
  cat <<EOF >> "${HOME}/.bazelrc"
 --output_base=".bazel-out"  # On GitHub Actions, staying on the same volume seems to be faster
 EOF
@@ -103,6 +81,7 @@ if [ "${TRAVIS-}" = true ] || [ -n "${GITHUB_WORKFLOW-}" ]; then
 # CI output doesn't scroll, so don't use curses
 build --color=yes
 build --curses=no
+build --disk_cache="$(test "${OSTYPE}" = msys || echo ~/ray-bazel-cache)"
 build --progress_report_interval=60
 # Use ray google cloud cache
 build --remote_cache="https://storage.googleapis.com/ray-bazel-cache"
@@ -1,30 +1,15 @@
 #!/usr/bin/env bash

-# Cause the script to exit if a single command fails
-set -e
+set -euxo pipefail

 ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)

-echo "PYTHON is $PYTHON"
+install_cython_examples() {
+  (
+    cd "${ROOT_DIR}/../../doc/examples/cython"
+    pip install scipy
+    python setup.py install --user
+  )
+}

-cython_examples="$ROOT_DIR/../../doc/examples/cython"
-
-if [[ "$PYTHON" == "3.6" ]]; then
-   export PATH="$HOME/miniconda/bin:$PATH"
-
-   pushd $cython_examples
-   pip install scipy
-   python setup.py install --user
-   popd
-
-elif [[ "$LINT" == "1" ]]; then
-   export PATH="$HOME/miniconda/bin:$PATH"
-
-   pushd $cython_examples
-   python setup.py install --user
-   popd
-
-else
-   echo "Unrecognized Python version."
-   exit 1
-fi
+install_cython_examples "$@"
@@ -1,122 +1,176 @@
 #!/usr/bin/env bash

-ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+{ SHELLOPTS_STACK="${SHELLOPTS_STACK-}|$(set +o); set -$-"; } 2> /dev/null  # Push caller's shell options (quietly)

-echo "PYTHON is $PYTHON"
+set -euxo pipefail

-# Make sure all important package versions are static (via env variables
-# or assign default values to them).
-tf_version="$TF_VERSION"
-if [[ $tf_version == "" ]]; then tf_version="2.0.0b1"; fi
-echo "tf_version is $tf_version"
-tfp_version="$TFP_VERSION"
-if [[ tfp_version == "" ]]; then tfp_version="0.8"; fi
-echo "tfp_version is $tfp_version"
-torch_version="$TORCH_VERSION"
-if [[ torch_version == "" ]]; then torch_version="1.4"; fi
-echo "torch_version is $torch_version"
+ROOT_DIR=$(builtin cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+WORKSPACE_DIR="${ROOT_DIR}/../.."

-platform=""
-case "${OSTYPE}" in
-  linux*)
-    platform="linux"
-    ;;
-  darwin*)
-    platform="macosx"
-    ;;
-  msys*)
-    platform="win32"
-    ;;
-esac
-if [ -z "$platform" ]; then
-  echo "Unrecognized platform."
-  exit 1
-fi
+pkg_install_helper() {
+  case "${OSTYPE}" in
+    darwin*) brew install "$@";;
+    linux*) sudo apt-get install -qq -o=Dpkg::Use-Pty=0 "$@" | grep --line-buffered -v "^\(Preparing to unpack\|Unpacking\|Processing triggers for\) ";;
+    *) false;;
+  esac
+}

-echo "Platform is $platform."
+install_base() {
+  case "${OSTYPE}" in
+    linux*)
+      sudo apt-get update -qq
+      pkg_install_helper build-essential curl unzip tmux gdb libunwind-dev python3-pip python3-setuptools
+      if [ "${LINUX_WHEELS-}" = 1 ]; then
+        pkg_install_helper docker
+        sudo usermod -a -G docker travis
+      fi
+      if [ -n "${PYTHON-}" ]; then
+        "${ROOT_DIR}/install-strace.sh" || true
+      fi
+      ;;
+  esac
+}

-# Upgrade pip and other packages to avoid incompatibility ERRORS.
-pip install --upgrade pip # setuptools cloudpickle urllib3
+install_miniconda() {
+  local miniconda_version="3-4.5.4" miniconda_platform="" exe_suffix=".sh"
+  case "${OSTYPE}" in
+    linux*) miniconda_platform=Linux;;
+    darwin*) miniconda_platform=MacOSX;;
+    msys*) miniconda_platform=Windows; exe_suffix=".exe";;
+  esac
+  local miniconda_url="https://repo.continuum.io/miniconda/Miniconda${miniconda_version}-${miniconda_platform}-${HOSTTYPE}${exe_suffix}"
+  local miniconda_target="./${miniconda_url##*/}"
+  local miniconda_dir="$HOME/miniconda"
+  curl -s -L -o "${miniconda_target}" "${miniconda_url}"
+  chmod +x "${miniconda_target}"
+  case "${OSTYPE}" in
+    msys*)
+      miniconda_dir="${miniconda_dir}/bin"  # HACK: Compensate for python.exe being in the installation root on Windows
+      MSYS2_ARG_CONV_EXCL="*" "${miniconda_target}" /S /D="$(cygpath -w -- "${miniconda_dir}")"
+      ;;
+    *)
+      "${miniconda_target}" -b -p "${miniconda_dir}" | grep --line-buffered -v "^\(installing: \|installation finished\.\)"
+      ;;
+  esac
+  { local set_x="${-//[^x]/}"; } 2> /dev/null  # save set -x to suppress noise
+  set +x
+  local source_line='PYTHON3_BIN_PATH=python; PATH="$HOME/miniconda/bin:$PATH"; export PYTHON3_BIN_PATH PATH;'
+  test -f ~/.bashrc && grep -x -q -F "${source_line}" -- ~/.bashrc || echo "${source_line}" >> ~/.bashrc
+  test -z "${set_x}" || set -x  # restore set -x
+  eval "${source_line}"
+  python -m pip install --upgrade --quiet pip
+}

-# If we're in a CI environment, do some configuration
-if [ "${TRAVIS-}" = true ] || [ -n "${GITHUB_WORKFLOW-}" ]; then
-  pip config --user set global.disable-pip-version-check True
-  pip config --user set global.no-color True
-  pip config --user set global.progress_bar off
-  pip config --user set global.quiet True
-fi
+install_nvm() {
+  local NVM_HOME="${HOME}/.nvm"
+  if [ "${OSTYPE}" = msys ]; then
+    local version="1.1.7"
+    if [ ! -f "${NVM_HOME}/nvm.sh" ]; then
+      mkdir -p -- "${NVM_HOME}"
+      export NVM_SYMLINK="${PROGRAMFILES}\nodejs"
+      (
+        cd "${NVM_HOME}"
+        local target="./nvm-${version}.zip"
+        curl -s -L -o "${target}" "https://github.com/coreybutler/nvm-windows/releases/download/${version}/nvm-noinstall.zip"
+        unzip -q -- "${target}"
+        rm -f -- "${target}"
+        printf "%s\r\n" "root: $(cygpath -w -- "${NVM_HOME}")" "path: ${NVM_SYMLINK}" > settings.txt
+      )
+      printf "%s\n" "export NVM_HOME=\"$(cygpath -w -- "${NVM_HOME}")\"" 'nvm() { "${NVM_HOME}/nvm.exe" "$@"; }' > "${NVM_HOME}/nvm.sh"
+    fi
+  else
+    test -f "${NVM_HOME}/nvm.sh"  # double-check NVM is already available on other platforms
+  fi
+}

-if [[ "$PYTHON" == "3.6" ]] && [[ "$platform" == "linux" ]]; then
-  sudo apt-get update
-  sudo apt-get install -y build-essential curl unzip tmux gdb libunwind-dev
-  # Install miniconda.
-  wget -q https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv
-  bash miniconda.sh -b -p $HOME/miniconda
-  export PATH="$HOME/miniconda/bin:$PATH"
-  "${ROOT_DIR}/install-strace.sh" || true
-  pip install scipy tensorflow==$tf_version \
-    cython==0.29.0 gym \
-    opencv-python-headless pyyaml pandas==0.24.2 requests \
-    feather-format lxml openpyxl xlrd py-spy pytest pytest-timeout networkx tabulate aiohttp \
-    uvicorn dataclasses pygments werkzeug kubernetes flask grpcio pytest-sugar pytest-rerunfailures pytest-asyncio \
-    blist scikit-learn numba
-elif [[ "$PYTHON" == "3.6" ]] && [[ "$platform" == "macosx" ]]; then
-  # Install miniconda.
-  wget -q https://repo.continuum.io/miniconda/Miniconda3-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv
-  bash miniconda.sh -b -p $HOME/miniconda
-  export PATH="$HOME/miniconda/bin:$PATH"
-  pip install scipy tensorflow==$tf_version \
-    cython==0.29.0 gym \
-    opencv-python-headless pyyaml pandas==0.24.2 requests \
-    feather-format lxml openpyxl xlrd py-spy pytest pytest-timeout networkx tabulate aiohttp \
-    uvicorn dataclasses pygments werkzeug kubernetes flask grpcio pytest-sugar pytest-rerunfailures pytest-asyncio \
-    blist scikit-learn numba
-elif [[ "$LINT" == "1" ]]; then
-  sudo apt-get update
-  sudo apt-get install -y build-essential curl unzip
-  # Install miniconda.
-  wget -q https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv
-  bash miniconda.sh -b -p $HOME/miniconda
-  export PATH="$HOME/miniconda/bin:$PATH"
-  # Install Python linting tools.
-  pip install flake8==3.7.7 flake8-comprehensions flake8-quotes==2.0.0
-  # Install TypeScript and HTML linting tools.
-  pushd "$ROOT_DIR/../../python/ray/dashboard/client"
-    source "$HOME/.nvm/nvm.sh"
-    nvm install node
-    nvm use node
-    npm ci
-  popd
-elif [[ "$LINUX_WHEELS" == "1" ]]; then
-  sudo apt-get install docker
-  sudo usermod -a -G docker travis
-elif [[ "$MAC_WHEELS" == "1" ]]; then
-  :
-else
-  echo "Unrecognized environment."
-  exit 1
-fi
+install_pip() {
+  local python=python
+  if command -v python3 > /dev/null; then
+    python=python3
+  fi

-# Install modules needed in all jobs.
-pip install dm-tree
+  if "${python}" -m pip --version || "${python}" -m ensurepip; then  # If pip is present, configure it
+    "${python}" -m pip install --upgrade --quiet pip

-# Additional RLlib dependencies.
-if [[ "$RLLIB_TESTING" == "1" ]]; then
-  pip install tensorflow-probability==$tfp_version gast==0.2.2 \
-    torch==$torch_version torchvision \
-    atari_py gym[atari] lz4 smart_open
-fi
+    # If we're in a CI environment, do some configuration
+    if [ "${TRAVIS-}" = true ] || [ -n "${GITHUB_WORKFLOW-}" ]; then
+      "${python}" -W ignore -m pip config -q --user set global.disable-pip-version-check True
+      "${python}" -W ignore -m pip config -q --user set global.no-color True
+      "${python}" -W ignore -m pip config -q --user set global.progress_bar off
+      "${python}" -W ignore -m pip config -q --user set global.quiet True
+    fi
+  fi
+}

-# Additional streaming dependencies.
-if [[ "$RAY_CI_STREAMING_PYTHON_AFFECTED" == "1" ]]; then
-  pip install -q msgpack>=0.6.2
-fi
+install_node() {
+  if [ "${OSTYPE}" = msys ]; then
+    { echo "WARNING: Skipping running Node.js due to module incompatibilities with Windows"; } 2> /dev/null
+  else
+    # Install the latest version of Node.js in order to build the dashboard.
+    (
+      set +x # suppress set -x since it'll get very noisy here
+      . "${HOME}/.nvm/nvm.sh"
+      nvm install node
+      nvm use --silent node
+      npm config set loglevel warn  # make NPM quieter
+    )
+  fi
+}

-if [[ "$PYTHON" == "3.6" ]] || [[ "$MAC_WHEELS" == "1" ]]; then
-  # Install the latest version of Node.js in order to build the dashboard.
-  source "$HOME/.nvm/nvm.sh"
-  nvm install node
-fi
+run_npm() {
+  npm ci
+}

-pip install psutil setproctitle \
-        --target="$ROOT_DIR/../../python/ray/thirdparty_files"
+install_dependencies() {
+
+  install_base
+  if [ -n "${GITHUB_WORKFLOW-}" ]; then  # Keep Travis's built-in compilers and only use this for GitHub Actions (for now)
+    "${ROOT_DIR}"/install-toolchains.sh
+  fi
+  install_nvm
+  install_pip
+
+  if [ -n "${PYTHON-}" ]; then
+    install_miniconda
+    pip_packages=(scipy tensorflow=="${TF_VERSION:-2.0.0b1}" cython==0.29.0 gym opencv-python-headless pyyaml \
+      pandas==0.24.2 requests feather-format lxml openpyxl xlrd py-spy pytest pytest-timeout networkx tabulate aiohttp \
+      uvicorn dataclasses pygments werkzeug kubernetes flask grpcio pytest-sugar pytest-rerunfailures pytest-asyncio \
+      scikit-learn numba)
+    if [ "${OSTYPE}" != msys ]; then
+      # These packages aren't Windows-compatible
+      pip_packages+=(blist)  # https://github.com/DanielStutzbach/blist/issues/81#issue-391460716
+    fi
+    CC=gcc pip install "${pip_packages[@]}"
+  elif [ "${LINT-}" = 1 ]; then
+    install_miniconda
+    pip install flake8==3.7.7 flake8-comprehensions flake8-quotes==2.0.0  # Python linters
+    pushd "${WORKSPACE_DIR}/python/ray/dashboard/client"  # TypeScript & HTML linters
+      install_node
+      run_npm
+    popd
+  fi
+
+  # Install modules needed in all jobs.
+  pip install dm-tree
+
+  # Additional RLlib dependencies.
+  if [ "${RLLIB_TESTING-}" = 1 ]; then
+    pip install tensorflow-probability=="${TFP_VERSION-0.8}" gast==0.2.2 torch=="${TORCH_VERSION-1.4}" torchvision \
+      atari_py gym[atari] lz4 smart_open
+  fi
+
+  # Additional streaming dependencies.
+  if [ "${RAY_CI_STREAMING_PYTHON_AFFECTED}" = 1 ]; then
+    pip install msgpack>=0.6.2
+  fi
+
+  if [ -n "${PYTHON-}" ] || [ "${MAC_WHEELS-}" = 1 ]; then
+    install_node
+  fi
+
+  CC=gcc pip install psutil setproctitle --target="${WORKSPACE_DIR}/python/ray/thirdparty_files"
+}
+
+install_dependencies "$@"
+
+{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null  # Pop caller's shell options (quietly)
@@ -1,27 +1,35 @@
 #!/usr/bin/env bash

-# Cause the script to exit if a single command fails.
-set -e
+set -euxo pipefail

 ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
+WORKSPACE_DIR="${ROOT_DIR}/../.."

-echo "PYTHON is $PYTHON"
-
-if [[ "$PYTHON" == "3.6" ]]; then
-  export PATH="$HOME/miniconda/bin:$PATH"
-
-  pushd "$ROOT_DIR/../../python"
-    pushd ray/dashboard/client
-      source $HOME/.nvm/nvm.sh
-      nvm use node
+build_dashboard_front_end() {
+  if [ "${OSTYPE}" = msys ]; then
+    { echo "WARNING: Not building dashboard front-end due to NPM package incompatibilities with Windows"; } 2> /dev/null
+  else
+    (
+      cd ray/dashboard/client
+      set +x  # suppress set -x since it'll get very noisy here
+      . "${HOME}/.nvm/nvm.sh"
+      nvm use --silent node
      npm ci
-      npm run build
-    popd
-    pip install -e . --verbose
-  popd
+      npm run -s build
+    )
+  fi
+}

-else
-  echo "Unrecognized Python version."
-  exit 1
-fi
+install_ray() {
+  (
+    cd "${WORKSPACE_DIR}"/python
+    build_dashboard_front_end
+    if [ "${OSTYPE}" = msys ]; then
+      "${WORKSPACE_DIR}"/ci/keep_alive pip install -v -e . || echo "WARNING: Ignoring Ray package build failure on Windows for now" 1>&2
+    else
+      "${WORKSPACE_DIR}"/ci/keep_alive pip install -v -e .
+    fi
+  )
+}

+install_ray "$@"
@@ -1,22 +1,22 @@
 #!/usr/bin/env bash

-set -eu
+set -euxo pipefail

-case "$(uname -s)" in
-  Linux*)
-    strace -qq -k -e trace=exit /bin/true 1> /dev/null 2> /dev/null || {
-      echo "This Linux distribution doesn't appear to support strace -k." "Attempting to build & install a recent version..." 1>&2
-      git -c advice.detachedHead=false clone -q --depth=1 "https://github.com/strace/strace" -b v5.5 && (
-        cd strace &&
-        ./bootstrap > /dev/null &&
-        CPPFLAGS="-w ${CPPFLAGS-}" ./configure --quiet --with-libunwind --enable-mpers=no &&
-        make -s -j"$(getconf _NPROCESSORS_ONLN || echo 1)" &&
-        sudo make -s install
-      )
-    }
-    ;;
-  *)
-    # Unable to install on other platforms
-    false
-    ;;
-esac
+install_strace() {
+  case "${OSTYPE}" in
+    linux*)
+      if ! strace -qq -k -e trace=exit /bin/true 1> /dev/null 2> /dev/null; then
+        { echo "This Linux distribution doesn't appear to support strace -k." "Attempting to build & install a recent version..."; } 2> /dev/null
+        git -c advice.detachedHead=false clone -q --depth=1 "https://github.com/strace/strace" -b v5.5 && (
+          cd strace &&
+          ./bootstrap > /dev/null &&
+          CPPFLAGS="-w ${CPPFLAGS-}" ./configure --quiet --with-libunwind --enable-mpers=no &&
+          make -s -j"$(getconf _NPROCESSORS_ONLN || echo 1)" &&
+          sudo make -s install
+        ) > /dev/null
+      fi;;
+    *) false;;
+  esac
+}
+
+install_strace "$@"
@@ -4,13 +4,12 @@ set -euxo pipefail

 LLVM_VERSION="9.0.0"

-install_clang() {
+install_toolchains() {
  local osversion="" url="" urlbase="https://releases.llvm.org" targetdir="/usr/local"
  case "${OSTYPE}" in
    msys)
-      export MSYS2_ARG_CONV_EXCL="*"  # Don't let MSYS2 attempt to auto-translate arguments that look like paths
      osversion=win
-      if [ ! "${HOSTTYPE}" = "${HOSTTYPE%64}" ]; then
+      if [ "${HOSTTYPE}" != "${HOSTTYPE%64}" ]; then
        osversion="${osversion}64"
      else
        osversion="${osversion}32"
@@ -31,7 +30,7 @@ install_clang() {
  curl -s -L -R "${url}" | if [ "${OSTYPE}" = "msys" ]; then
    local target="./${url##*/}"
    install /dev/stdin "${target}"
-    7z x "${target}" -o"${targetdir}"
+    7z x -bsp0 -bso0 "${target}" -o"${targetdir}"
    rm -f -- "${target}"
  else
    sudo tar -x -J --strip-components=1 -C "${targetdir}"
@@ -40,4 +39,4 @@ install_clang() {
  "${targetdir}"/bin/clang --version 1>&2
 }

-install_"$@"
+install_toolchains "$@"
@@ -175,7 +175,6 @@ genrule(
    ],
    outs = ["gen_maven_deps.out"],
    cmd = """
-        set -x
        WORK_DIR="$$(pwd)"
        # Copy protobuf-generated files.
        rm -rf "$$WORK_DIR/java/runtime/src/main/java/io/ray/runtime/generated"
@@ -204,7 +203,6 @@ genrule(
    ],
    outs = ["copy_pom_file.out"],
    cmd = """
-        set -x
        WORK_DIR="$$(pwd)"
        cp -f $(location //java:io_ray_ray_api_pom) "$$WORK_DIR/java/api/pom.xml"
        cp -f $(location //java:io_ray_ray_runtime_pom) "$$WORK_DIR/java/runtime/pom.xml"
@@ -46,5 +46,5 @@ popd

 pushd $ROOT_DIR
 echo "Testing maven install."
-mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN clean install -DskipTests
+mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN --no-transfer-progress clean install -DskipTests
 popd
@@ -1,6 +1,12 @@
 #include <limits.h>
 #include <sys/time.h>

+#ifdef timezone
+#pragma push_macro("timezone")  // Work around https://bugs.python.org/issue34657
+#undef timezone
+#define timezone timezone
+#endif
+
 int gettimeofday(struct timeval *tv, struct timezone *tz) {
  // Free implementation from: https://stackoverflow.com/a/26085827
  SYSTEMTIME systime;
@@ -16,3 +22,7 @@ int gettimeofday(struct timeval *tv, struct timezone *tz) {
  tv->tv_usec = static_cast<int>(systime.wMilliseconds * 1000);
  return 0;
 }
+
+#ifdef timezone
+#pragma pop_macro("timezone")
+#endif
@@ -1,6 +1,12 @@
 #ifndef TIME_H
 #define TIME_H

+#ifdef timezone
+#pragma push_macro("timezone")  // Work around https://bugs.python.org/issue34657
+#undef timezone
+#define timezone timezone
+#endif
+
 #include <WinSock2.h>  // clients require timeval definition

 struct timeval;
@@ -12,4 +18,8 @@ extern "C"
    int
    gettimeofday(struct timeval *tv, struct timezone *tz);

+#ifdef timezone
+#pragma pop_macro("timezone")
+#endif
+
 #endif /* TIME_H */
@@ -268,7 +268,6 @@ genrule(
    ],
    cmd = """
        set -e
-        set -x
        WORK_DIR="$$(pwd)"
        # Copy generated files.
        GENERATED_DIR="$$WORK_DIR/streaming/python/generated"
@@ -166,7 +166,6 @@ genrule(
    ],
    outs = ["copy_pom_file.out"],
    cmd = """
-        set -x
        WORK_DIR="$$(pwd)"
        cp -f $(location //streaming/java:io_ray_ray_streaming-api_pom) "$$WORK_DIR/streaming/java/streaming-api/pom.xml"
        cp -f $(location //streaming/java:io_ray_ray_streaming-runtime_pom) "$$WORK_DIR/streaming/java/streaming-runtime/pom.xml"
@@ -184,7 +183,6 @@ genrule(
    ],
    outs = ["cp_java_generated.out"],
    cmd = """
-        set -x
        WORK_DIR="$$(pwd)"
        GENERATED_DIR="$$WORK_DIR/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/generated"
        rm -rf "$$GENERATED_DIR"
@@ -208,7 +206,6 @@ genrule(
    ],
    outs = ["gen_maven_deps.out"],
    cmd = """
-        set -x
        WORK_DIR="$$(pwd)"
        # Copy native dependencies.
        NATIVE_DEPS_DIR="$$WORK_DIR/streaming/java/streaming-runtime/native_dependencies/"
@@ -36,7 +36,7 @@ cd "$ROOT_DIR"/../../java
 echo "build ray maven deps"
 bazel build gen_maven_deps
 echo "maven install ray"
-mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN clean install -DskipTests
+mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN --no-transfer-progress clean install -DskipTests
 cd "$ROOT_DIR"
 echo "maven install ray streaming"
-mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN clean install -DskipTests
+mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN --no-transfer-progress clean install -DskipTests