Set up testing and wheels for Windows on GitHub Actions (#8131)

* Move some Java tests into ci.sh * Move C++ worker tests into ci.sh * Define run() * Prepare to move Python tests into ci.sh * Fix issues in install-dependencies.sh * Reload environment for GitHub Actions * Move wheels to ci.sh and fix related issues * Don't bypass failures in install-ray.sh anymore * Make CI a little quieter * Move linting into ci.sh * Add vitals test right after build * Fix os.uname() unavailability on Windows Co-authored-by: Mehrdad <noreply@github.com>
2026-06-27 19:32:11 +08:00 · 2020-04-29 21:19:02 -07:00
parent ae54e0dc0a
commit 254b1ec370
17 changed files with 302 additions and 145 deletions
@@ -18,7 +18,11 @@ jobs:
          - name: macos
            os: macos-10.15
    env:
+      GITHUB_PULL_REQUEST: ${{ github.event.number }}
      PYTHON: 3.6
+      TRAVIS_BUILD_DIR: ${{ github.workspace }}
+      TRAVIS_COMMIT: ${{ github.sha }}
+      #TRAVIS_PULL_REQUEST is defined in the scripts to account for "false"
    steps:
    - name: Pre-checkout system configuration
      # We add -l to process .bashrc, but need -e -o pipefail for consistency with GitHub Actions's default behavior.
@@ -40,9 +44,17 @@ jobs:
      env:
        BAZEL_CACHE_CREDENTIAL_B64: ${{ secrets.BAZEL_CACHE_CREDENTIAL_B64 }}
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        #LINUX_WHEELS: 1
+        LINT: 1
+        #MAC_WHEELS: 1
+        RAY_DEFAULT_BUILD: 1
+        WINDOWS_WHEELS: 1
      run: |
        . ./ci/travis/ci.sh init
+        . ./ci/travis/ci.sh lint
        . ./ci/travis/ci.sh build
+        . ./ci/travis/ci.sh test_python
+        . ./ci/travis/ci.sh test_wheels
    - name: Run Clang Include-What-You-Use
      continue-on-error: true
      if: runner.os == 'Linux'
@@ -37,7 +37,6 @@ matrix:
      install:
        - . ./ci/travis/ci.sh build
      script:
-        - . ./ci/travis/ci.sh preload
        - ./java/test.sh

    - os: linux
@@ -51,7 +50,6 @@ matrix:
      install:
        - . ./ci/travis/ci.sh build
      script:
-        - . ./ci/travis/ci.sh preload
        # Streaming cpp test.
        - if [ $RAY_CI_STREAMING_CPP_AFFECTED == "1" ]; then bazel test --config=ci --test_output=all //streaming:all && bash streaming/src/test/run_streaming_queue_test.sh; fi
        - if [ $RAY_CI_STREAMING_PYTHON_AFFECTED == "1" ]; then python -m pytest -v --durations=5 --timeout=300 streaming/python/tests/; fi
@@ -70,7 +68,6 @@ matrix:
      install:
        - . ./ci/travis/ci.sh build
      script:
-        - . ./ci/travis/ci.sh preload
        - ./ci/suppress_output bash src/ray/test/run_core_worker_tests.sh
        - ./ci/suppress_output bash streaming/src/test/run_streaming_queue_test.sh
        - ./java/test.sh
@@ -88,7 +85,6 @@ matrix:
      install:
        - . ./ci/travis/ci.sh build
      script:
-        - . ./ci/travis/ci.sh preload
        - ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/tests/...

    - os: linux
@@ -100,43 +96,7 @@ matrix:
      install:
        - . ./ci/travis/ci.sh build
      script:
-        - . ./ci/travis/ci.sh preload
-        - ./ci/travis/check-git-clang-format-output.sh
-        # Try generating Sphinx documentation. To do this, we need to install
-        # Ray first.
-        - cd doc
-        # readthedocs has an antiquated build env.
-        # This is a best effort to reproduce it locally to avoid doc build failures
-        # and hidden errors.
-        - pip install -r requirements-rtd.txt
-        - pip install -r requirements-doc.txt
-        - pip install yapf==0.23.0
-        - sphinx-build -q -W -E -T -b html source _build/html
-        - cd ..
-        # Run Python linting, ignore dict vs {} (C408), others are defaults
-        - flake8 --inline-quotes '"' --no-avoid-escape --exclude=python/ray/core/generated/,streaming/python/generated,doc/source/conf.py,python/ray/cloudpickle/,python/ray/thirdparty_files --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605
-        - ./ci/travis/format.sh --all
-        # Make sure that the README is formatted properly.
-        - cd python
-        - python setup.py check --restructuredtext --strict --metadata
-        - cd ..
-        # Run Bazel linter Buildifier.
-        - wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz
-        - tar -xf go1.14.2.linux-amd64.tar.gz
-        - mkdir $HOME/go_dir
-        - export GOROOT=`pwd`/go
-        - export GOPATH="$HOME/go_dir"
-        - export PATH="$GOROOT/bin:$GOPATH/bin:$PATH"
-        - go get github.com/bazelbuild/buildtools/buildifier
-        - ./ci/travis/bazel-format.sh
-        # Run TypeScript and HTML linting.
-        - pushd python/ray/dashboard/client
-        - source "$HOME/.nvm/nvm.sh"
-        - nvm use node
-        - node_modules/.bin/eslint --max-warnings 0 $(find src -name "*.ts" -or -name "*.tsx")
-        - node_modules/.bin/prettier --check $(find src -name "*.ts" -or -name "*.tsx")
-        - node_modules/.bin/prettier --check public/index.html
-        - popd
+        - . ./ci/travis/ci.sh lint

    # Build Linux wheels.
    - os: linux
@@ -149,9 +109,7 @@ matrix:
      install:
        - . ./ci/travis/ci.sh build
      script:
-        - . ./ci/travis/ci.sh preload
-        # Explicitly sleep 60 seconds for logs to go through
-        - ./ci/travis/test-wheels.sh || { cat /tmp/ray/session_latest/logs/* && sleep 60 && false; }
+        - . ./ci/travis/ci.sh test_wheels
      cache: false

    # Build MacOS wheels.
@@ -166,9 +124,7 @@ matrix:
      install:
        - . ./ci/travis/ci.sh build
      script:
-        - . ./ci/travis/ci.sh preload
-        # Explicitly sleep 60 seconds for logs to go through
-        - ./ci/travis/test-wheels.sh || { cat /tmp/ray/session_latest/logs/* && sleep 60 && false; }
+        - . ./ci/travis/ci.sh test_wheels

    # RLlib: Learning tests (from rllib/tuned_examples/regression_tests/*.yaml).
    - os: linux
@@ -185,7 +141,6 @@ matrix:
      install:
        - . ./ci/travis/ci.sh build
      script:
-        - . ./ci/travis/ci.sh preload
        - travis_wait 120 bazel test --config=ci --test_output=streamed --build_tests_only --test_tag_filters=learning_tests rllib/...

    # RLlib: Learning tests with tf=1.x (from rllib/tuned_examples/regression_tests/*.yaml).
@@ -204,7 +159,6 @@ matrix:
      install:
        - . ./ci/travis/ci.sh build
      script:
-        - . ./ci/travis/ci.sh preload
        - travis_wait 120 bazel test --config=ci --test_output=streamed --build_tests_only --test_tag_filters=learning_tests rllib/...

    # RLlib: Quick Agent train.py runs (compilation & running, no(!) learning).
@@ -223,7 +177,6 @@ matrix:
      install:
        - . ./ci/travis/ci.sh build
      script:
-        - . ./ci/travis/ci.sh preload
        - travis_wait 60 bazel test --config=ci --build_tests_only --test_tag_filters=quick_train rllib/...
        # Test everything that does not have any of the "main" labels:
        # "learning_tests|quick_train|examples|tests_dir".
@@ -244,7 +197,6 @@ matrix:
      install:
        - . ./ci/travis/ci.sh build
      script:
-        - . ./ci/travis/ci.sh preload
        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_A,examples_B rllib/...
        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_C rllib/...
        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_E,examples_L,examples_M,examples_N,examples_P rllib/...
@@ -265,7 +217,6 @@ matrix:
      install:
        - . ./ci/travis/ci.sh build
      script:
-        - . ./ci/travis/ci.sh preload
        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I rllib/...

    # RLlib: tests_dir: Everything in rllib/tests/ directory (J-Z).
@@ -283,7 +234,6 @@ matrix:
      install:
        - . ./ci/travis/ci.sh build
      script:
-        - . ./ci/travis/ci.sh preload
        - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_J,tests_dir_K,tests_dir_L,tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z rllib/...

    # Cpp worker test
@@ -296,11 +246,9 @@ matrix:
      install:
        - . ./ci/travis/ci.sh build
      script:
-        - . ./ci/travis/ci.sh preload
-        - bazel test --config=ci //cpp:all --build_tests_only --test_output=streamed
+        - . ./ci/travis/ci.sh test_cpp

 script:
-  - . ./ci/travis/ci.sh preload
  # raylet integration tests
  - ./ci/suppress_output bash src/ray/test/run_core_worker_tests.sh
  - ./ci/suppress_output bash src/ray/test/run_object_manager_tests.sh
@@ -312,10 +260,7 @@ script:
  - if [ $RAY_CI_SERVE_AFFECTED == "1" ]; then ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/serve/...; fi

  # ray operator tests
-  - cd ./deploy/ray-operator/
-  - CC=gcc ../../ci/suppress_output go build
-  - CC=gcc ../../ci/suppress_output go test ./...
-  - cd ../..
+  - (cd deploy/ray-operator && export CC=gcc && suppress_output go build && suppress_output go test ./...)

  # random python tests TODO(ekl): these should be moved to bazel
  - if [ $RAY_CI_PYTHON_AFFECTED == "1" ]; then python -m pytest -v --durations=5 --timeout=300 python/ray/experimental/test/async_test.py; fi
@@ -1530,7 +1530,7 @@ genrule(
            tmpdir="redis.tmp" &&
            path=$(location @com_github_antirez_redis//:file) &&
            cp -p -L -R -- "$${path%/*}" "$${tmpdir}" &&
-            make -s -C "$${tmpdir}" -j"$$(getconf _NPROCESSORS_ONLN || echo 1)" &&
+            make -s -C "$${tmpdir}" -j"$$(getconf _NPROCESSORS_ONLN || echo 1)" V=0 &&
            mv "$${tmpdir}"/src/redis-server $(location redis-server) &&
            chmod +x $(location redis-server) &&
            mv "$${tmpdir}"/src/redis-cli $(location redis-cli) &&
@@ -132,7 +132,7 @@ pushd $WORK_DIR
 git clone https://github.com/suquark/pickle5-backport
 pushd pickle5-backport
  git checkout 8ffe41ceba9d5e2ce8a98190f6b3d2f3325e5a72
-  CC=gcc "$PYTHON_EXECUTABLE" setup.py bdist_wheel
+  CC=gcc "$PYTHON_EXECUTABLE" setup.py --quiet bdist_wheel
  unzip -o dist/*.whl -d "$ROOT_DIR/python/ray/pickle5_files"
 popd
 popd
@@ -38,21 +38,210 @@ should_run_job() {
  return "${skip}"
 }

+# Idempotent environment loading
 reload_env() {
  # TODO: We should really just use a new login shell instead of doing this manually.
  # Otherwise we might source a script that isn't idempotent (e.g. one that blindly prepends to PATH).
-  { local set_x="${-//[^x]/}"; } 2> /dev/null  # save set -x to suppress noise
-  set +x
-  local to_add="$HOME/miniconda/bin" old_path=":${PATH}:"
-  if [ "${old_path}" = "${old_path##*:${to_add}:*}" ]; then
-    PATH="${to_add}:$PATH"
+
+  if [ -n "${GITHUB_PULL_REQUEST-}" ]; then
+    case "${GITHUB_PULL_REQUEST}" in
+      [1-9]*) TRAVIS_PULL_REQUEST="${GITHUB_PULL_REQUEST}";;
+      *) TRAVIS_PULL_REQUEST=false;;
+    esac
+    export TRAVIS_PULL_REQUEST
  fi
-  test -z "${set_x}" || set -x  # restore set -x
+
+  export PYTHON3_BIN_PATH=python
+  export GOROOT="${HOME}/go" GOPATH="${HOME}/go_dir"
  if [ "${OSTYPE}" = msys ]; then
-    PATH="${PATH// :/:}"  # HACK: Work around https://github.com/actions/virtual-environments/issues/635#issue-589318442
+    export USE_CLANG_CL=1
  fi
-  PYTHON3_BIN_PATH=python
-  export PATH PYTHON3_BIN_PATH
+
+  # NOTE: Modifying PATH invalidates Bazel's cache! Do not add to PATH unnecessarily.
+  PATH="${HOME}/miniconda/bin":"${PATH}"
+  if [ "${OSTYPE}" = msys ]; then
+    PATH="${HOME}/miniconda/bin/Scripts":"${PATH}"
+  fi
+
+  # Deduplicate PATH
+  PATH="$(set +x && printf "%s\n" "${PATH}" | tr ":" "\n" | awk '!a[$0]++' | tr "\n" ":")"
+  PATH="${PATH%:}"  # Remove trailing colon
+}
+
+need_wheels() {
+  local error_code=1
+  case "${OSTYPE}" in
+    linux*) if [ "${LINUX_WHEELS-}" = 1 ]; then error_code=0; fi;;
+    darwin*) if [ "${MAC_WHEELS-}" = 1 ]; then error_code=0; fi;;
+    msys*) if [ "${WINDOWS_WHEELS-}" = 1 ]; then error_code=0; fi;;
+  esac
+  return "${error_code}"
+}
+
+test_python() {
+  if [ "${OSTYPE}" = msys ]; then
+    # Windows -- most tests won't work yet; just do the ones we know work
+    PYTHONPATH=python python -m pytest --durations=5 --timeout=300 python/ray/tests/test_mini.py
+  fi
+}
+
+test_cpp() {
+  bazel test --config=ci //cpp:all --build_tests_only --test_output=streamed
+}
+
+test_wheels() {
+  local result=0 flush_logs=0
+
+  if need_wheels; then
+    "${WORKSPACE_DIR}"/ci/travis/test-wheels.sh || { result=$? && flush_logs=1; }
+  fi
+
+  if [ 0 -ne "${flush_logs}" ]; then
+    cat -- /tmp/ray/session_latest/logs/* || true
+    sleep 60  # Explicitly sleep 60 seconds for logs to go through
+  fi
+
+  return "${result}"
+}
+
+build_sphinx_docs() {
+  (
+    cd "${WORKSPACE_DIR}"/doc
+    if [ "${OSTYPE}" = msys ]; then
+      echo "WARNING: Documentation not built on Windows due to currently-unresolved issues"
+    else
+      sphinx-build -q -W -E -T -b html source _build/html
+    fi
+  )
+}
+
+install_cython_examples() {
+  "${ROOT_DIR}"/install-cython-examples.sh
+}
+
+install_go() {
+  eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=stable bash)"
+}
+
+install_ray() {
+  bazel build -k "//:*"   # Do a full build first to ensure everything passes
+  "${ROOT_DIR}"/install-ray.sh
+}
+
+build_wheels() {
+  case "${OSTYPE}" in
+    linux*)
+      # Mount bazel cache dir to the docker container.
+      # For the linux wheel build, we use a shared cache between all
+      # wheels, but not between different travis runs, because that
+      # caused timeouts in the past. See the "cache: false" line below.
+      local MOUNT_BAZEL_CACHE=(-v "${HOME}/ray-bazel-cache":/root/ray-bazel-cache -e TRAVIS=true -e TRAVIS_PULL_REQUEST="${TRAVIS_PULL_REQUEST:-false}" -e encrypted_1c30b31fe1ee_key="${encrypted_1c30b31fe1ee_key-}" -e encrypted_1c30b31fe1ee_iv="${encrypted_1c30b31fe1ee_iv-}")
+
+      # This command should be kept in sync with ray/python/README-building-wheels.md,
+      # except the "${MOUNT_BAZEL_CACHE[@]}" part.
+      suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh
+      ;;
+    darwin*)
+      # This command should be kept in sync with ray/python/README-building-wheels.md.
+      suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh
+      ;;
+    msys*)
+      if ! python -c "import ray" 2> /dev/null; then
+        # This installs e.g. the pickle5 dependencies, which are needed for correct wheel testing
+        install_ray
+      fi
+      (
+        cd "${WORKSPACE_DIR}"/python
+        python setup.py --quiet bdist_wheel
+      )
+      ;;
+  esac
+}
+
+lint_readme() {
+  (
+    cd "${WORKSPACE_DIR}"/python
+    python setup.py check --restructuredtext --strict --metadata
+  )
+}
+
+lint_python() {
+  # ignore dict vs {} (C408), others are defaults
+  command -V python
+  python -m flake8 --inline-quotes '"' --no-avoid-escape --exclude=python/ray/core/generated/,streaming/python/generated,doc/source/conf.py,python/ray/cloudpickle/,python/ray/thirdparty_files --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605
+  "${ROOT_DIR}"/format.sh --all
+}
+
+lint_bazel() {
+  # Run buildifier without affecting external environment variables
+  (
+    # TODO: Move installing Go & building buildifier to the dependency installation step?
+    if [ ! -d "${GOROOT}" ]; then
+      curl -s -L "https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz" | tar -C "${GOROOT%/*}" -xz
+    fi
+    mkdir -p -- "${GOPATH}"
+    export PATH="${GOPATH}/bin":"${GOROOT}/bin":"${PATH}"
+
+    # Build buildifier
+    go get github.com/bazelbuild/buildtools/buildifier
+
+    # Now run buildifier
+    "${ROOT_DIR}"/bazel-format.sh
+  )
+}
+
+lint_web() {
+  (
+    cd "${WORKSPACE_DIR}"/python/ray/dashboard/client
+    set +x # suppress set -x since it'll get very noisy here
+    . "${HOME}/.nvm/nvm.sh"
+    install_npm_project
+    nvm use node
+    node_modules/.bin/eslint --max-warnings 0 $(find src -name "*.ts" -or -name "*.tsx")
+    node_modules/.bin/prettier --check $(find src -name "*.ts" -or -name "*.tsx")
+    node_modules/.bin/prettier --check public/index.html
+  )
+}
+
+_lint() {
+  local platform=""
+  case "${OSTYPE}" in
+    linux*) platform=linux;;
+  esac
+
+  if [ -n "${TRAVIS_BRANCH-}" ]; then
+    "${ROOT_DIR}"/check-git-clang-format-output.sh
+  else
+    # TODO(mehrdadn): Implement this on GitHub Actions
+    echo "WARNING: Not running clang-format due to TRAVIS_BRANCH not being defined."
+  fi
+
+  # Run Python linting
+  lint_python
+
+  # Make sure that the README is formatted properly.
+  lint_readme
+
+  if [ "${platform}" = linux ]; then
+    # Run Bazel linter Buildifier.
+    lint_bazel
+
+    # Run TypeScript and HTML linting.
+    lint_web
+  fi
+}
+
+lint() {
+  # Checkout a clean copy of the repo to avoid seeing changes that have been made to the current one
+  (
+    WORKSPACE_DIR="$(TMPDIR="${WORKSPACE_DIR}/.." mktemp -d)"
+    ROOT_DIR="${WORKSPACE_DIR}"/ci/travis
+    git worktree add -q "${WORKSPACE_DIR}"
+    pushd "${WORKSPACE_DIR}"
+      . "${ROOT_DIR}"/ci.sh _lint
+    popd  # this is required so we can remove the worktree when we're done
+    git worktree remove --force "${WORKSPACE_DIR}"
+  )
 }

 preload() {
@@ -85,6 +274,13 @@ EOF1
  fi
 }

+configure_system() {
+  git config --global advice.detachedHead false
+  git config --global core.askpass ""
+  git config --global credential.helper ""
+  git config --global credential.modalprompt false
+}
+
 # Initializes the environment for the current job. Performs the following tasks:
 # - Calls 'exit 0' in this job step and all subsequent steps to quickly exit if provided a list of job names and
 #   none of them has been triggered.
@@ -98,53 +294,47 @@ EOF1
 init() {
  preload

-  if [ "${OSTYPE}" = msys ]; then
-    export USE_CLANG_CL=1
-  fi
+  configure_system

-  local wheels="${LINUX_WHEELS-}${MAC_WHEELS-}"
-  if [ -z "${wheels}" ]; then  # NOT building wheels
-    "${ROOT_DIR}"/install-bazel.sh
-  fi
+  "${ROOT_DIR}"/install-bazel.sh
  . "${ROOT_DIR}"/install-dependencies.sh
-  reload_env  # We just modified our environment; reload it so we can continue
+
+  cat <<EOF >> ~/.bashrc
+. "${BASH_SOURCE:-$0}"  # reload environment
+EOF
 }

 build() {
-  local wheels="${LINUX_WHEELS-}${MAC_WHEELS-}"
-  if [ -z "${wheels}" ]; then  # NOT building wheels
-    if [ "${LINT-}" != 1 ]; then  # NOT linting
-      bazel build -k "//:*"   # Do a full build first to ensure it passes
-      "${ROOT_DIR}"/install-ray.sh
+  if ! need_wheels; then
+    install_ray
+    if [ "${LINT-}" = 1 ]; then
+      # Try generating Sphinx documentation. To do this, we need to install Ray first.
+      build_sphinx_docs
    fi
  fi

  if [ "${RAY_CYTHON_EXAMPLES-}" = 1 ]; then
-    "${ROOT_DIR}"/install-cython-examples.sh
+    install_cython_examples
  fi

  if [ "${RAY_DEFAULT_BUILD-}" = 1 ]; then
-    eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=stable bash)"
+    install_go
  fi

-  if [ "${LINUX_WHEELS-}" = 1 ]; then
-    # Mount bazel cache dir to the docker container.
-    # For the linux wheel build, we use a shared cache between all
-    # wheels, but not between different travis runs, because that
-    # caused timeouts in the past. See the "cache: false" line below.
-    local MOUNT_BAZEL_CACHE=(-v "${HOME}/ray-bazel-cache":/root/ray-bazel-cache -e TRAVIS=true -e TRAVIS_PULL_REQUEST="${TRAVIS_PULL_REQUEST}" -e encrypted_1c30b31fe1ee_key="${encrypted_1c30b31fe1ee_key-}" -e encrypted_1c30b31fe1ee_iv="${encrypted_1c30b31fe1ee_iv-}")
-
-    # This command should be kept in sync with ray/python/README-building-wheels.md,
-    # except the "${MOUNT_BAZEL_CACHE[@]}" part.
-    suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" -ti rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh
-  fi
-
-  if [ "${MAC_WHEELS-}" = 1 ]; then
-    # This command should be kept in sync with ray/python/README-building-wheels.md.
-    suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh
+  if need_wheels; then
+    build_wheels
  fi
 }

-"$@"
+_main() {
+  if [ -n "${GITHUB_WORKFLOW-}" ]; then
+    # Necessary for GitHub Actions (which uses separate shells for different commands)
+    # Unnecessary for Travis (which uses one shell for different commands)
+    reload_env
+  fi
+  "$@"
+}
+
+_main "$@"

 { set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null  # Pop caller's shell options (quietly)
@@ -22,7 +22,9 @@ install_base() {
      pkg_install_helper build-essential curl unzip tmux gdb libunwind-dev python3-pip python3-setuptools
      if [ "${LINUX_WHEELS-}" = 1 ]; then
        pkg_install_helper docker
-        sudo usermod -a -G docker travis
+        if [ -n "${TRAVIS-}" ]; then
+          sudo usermod -a -G docker travis
+        fi
      fi
      if [ -n "${PYTHON-}" ]; then
        "${ROOT_DIR}/install-strace.sh" || true
@@ -32,6 +34,13 @@ install_base() {
 }

 install_miniconda() {
+  local miniconda_dir="${HOME}/miniconda"
+  if [ "${OSTYPE}" = msys ]; then
+    miniconda_dir="${miniconda_dir}/bin"  # HACK: Compensate for python.exe being in the installation root on Windows
+  fi
+  if [ -d "${miniconda_dir}" ]; then
+    return 0  # already installed
+  fi
  local miniconda_version="3-4.5.4" miniconda_platform="" exe_suffix=".sh"
  case "${OSTYPE}" in
    linux*) miniconda_platform=Linux;;
@@ -39,25 +48,18 @@ install_miniconda() {
    msys*) miniconda_platform=Windows; exe_suffix=".exe";;
  esac
  local miniconda_url="https://repo.continuum.io/miniconda/Miniconda${miniconda_version}-${miniconda_platform}-${HOSTTYPE}${exe_suffix}"
-  local miniconda_target="./${miniconda_url##*/}"
-  local miniconda_dir="$HOME/miniconda"
+  local miniconda_target="${HOME}/${miniconda_url##*/}"
  curl -s -L -o "${miniconda_target}" "${miniconda_url}"
  chmod +x "${miniconda_target}"
  case "${OSTYPE}" in
    msys*)
-      miniconda_dir="${miniconda_dir}/bin"  # HACK: Compensate for python.exe being in the installation root on Windows
      MSYS2_ARG_CONV_EXCL="*" "${miniconda_target}" /S /D="$(cygpath -w -- "${miniconda_dir}")"
      ;;
    *)
      "${miniconda_target}" -b -p "${miniconda_dir}" | grep --line-buffered -v "^\(installing: \|installation finished\.\)"
      ;;
  esac
-  { local set_x="${-//[^x]/}"; } 2> /dev/null  # save set -x to suppress noise
-  set +x
-  local source_line='PYTHON3_BIN_PATH=python; PATH="$HOME/miniconda/bin:$PATH"; export PYTHON3_BIN_PATH PATH;'
-  test -f ~/.bashrc && grep -x -q -F "${source_line}" -- ~/.bashrc || echo "${source_line}" >> ~/.bashrc
-  test -z "${set_x}" || set -x  # restore set -x
-  eval "${source_line}"
+  reload_env
  python -m pip install --upgrade --quiet pip
 }

@@ -121,6 +123,15 @@ run_npm() {
  npm ci
 }

+install_npm_project() {
+  if [ "${OSTYPE}" = msys ]; then
+    # Not Windows-compatible: https://github.com/npm/cli/issues/558#issuecomment-584673763
+    { echo "WARNING: Skipping running NPM due to module incompatibilities with Windows"; } 2> /dev/null
+  else
+    run_npm
+  fi
+}
+
 install_dependencies() {

  install_base
@@ -150,13 +161,15 @@ install_dependencies() {
      pip_packages+=(blist)  # https://github.com/DanielStutzbach/blist/issues/81#issue-391460716
    fi
    CC=gcc pip install "${pip_packages[@]}"
-  elif [ "${LINT-}" = 1 ]; then
+  fi
+
+  if [ "${LINT-}" = 1 ]; then
    install_miniconda
-    pip install flake8==3.7.7 flake8-comprehensions flake8-quotes==2.0.0  # Python linters
-    pushd "${WORKSPACE_DIR}/python/ray/dashboard/client"  # TypeScript & HTML linters
-      install_node
-      run_npm
-    popd
+    pip install flake8==3.7.7 flake8-comprehensions flake8-quotes==2.0.0 yapf==0.23.0  # Python linters
+    # readthedocs has an antiquated build env.
+    # This is a best effort to reproduce it locally to avoid doc build failures and hidden errors.
+    pip install -r "${WORKSPACE_DIR}"/doc/requirements-rtd.txt
+    pip install -r "${WORKSPACE_DIR}"/doc/requirements-doc.txt
  fi

  # Install modules needed in all jobs.
@@ -173,7 +186,7 @@ install_dependencies() {
    pip install msgpack>=0.6.2
  fi

-  if [ -n "${PYTHON-}" ] || [ "${MAC_WHEELS-}" = 1 ]; then
+  if [ -n "${PYTHON-}" ] || [ -n "${LINT-}" ] || [ "${MAC_WHEELS-}" = 1 ]; then
    install_node
  fi

@@ -24,11 +24,7 @@ install_ray() {
  (
    cd "${WORKSPACE_DIR}"/python
    build_dashboard_front_end
-    if [ "${OSTYPE}" = msys ]; then
-      "${WORKSPACE_DIR}"/ci/keep_alive pip install -v -e . || echo "WARNING: Ignoring Ray package build failure on Windows for now" 1>&2
-    else
-      "${WORKSPACE_DIR}"/ci/keep_alive pip install -v -e .
-    fi
+    "${WORKSPACE_DIR}"/ci/keep_alive pip install -e .
  )
 }

@@ -8,18 +8,13 @@ set -x

 ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)

-platform="unknown"
-unamestr="$(uname)"
-if [[ "$unamestr" == "Linux" ]]; then
-  echo "Platform is linux."
-  platform="linux"
-elif [[ "$unamestr" == "Darwin" ]]; then
-  echo "Platform is macosx."
-  platform="macosx"
-else
-  echo "Unrecognized platform."
-  exit 1
-fi
+platform=""
+case "${OSTYPE}" in
+  linux*) platform="linux";;
+  darwin*) platform="macosx";;
+  msys*) platform="windows";;
+  *) echo "Unrecognized platform."; exit 1;;
+esac

 TEST_DIR="$TRAVIS_BUILD_DIR/python/ray/tests"
 TEST_SCRIPTS=("$TEST_DIR/test_microbenchmarks.py" "$TEST_DIR/test_basic.py")
@@ -49,7 +44,7 @@ if [[ "$platform" == "linux" ]]; then
          "3.7.6"
          "3.8.2")
  wget --quiet "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" -O miniconda3.sh
-  bash miniconda3.sh -b -p "$HOME/miniconda3"
+  "${ROOT_DIR}"/../suppress_output bash miniconda3.sh -b -p "$HOME/miniconda3"
  export PATH="$HOME/miniconda3/bin:$PATH"

  for ((i=0; i<${#PY_MMS[@]}; ++i)); do
@@ -129,6 +124,8 @@ elif [[ "$platform" == "macosx" ]]; then
    fi

  done
+elif [ "${platform}" = windows ]; then
+  echo "WARNING: Wheel testing not yet implemented for Windows."
 else
  echo "Unrecognized environment."
  exit 3
@@ -1,4 +1,3 @@
-blist
 click
 colorama
 filelock
@@ -13,6 +13,7 @@ import json
 import logging
 import os
 import re
+import socket
 import threading
 import time
 import traceback
@@ -1224,7 +1225,7 @@ if __name__ == "__main__":
            args.redis_address, password=args.redis_password)
        traceback_str = ray.utils.format_error_message(traceback.format_exc())
        message = ("The dashboard on node {} failed with the following "
-                   "error:\n{}".format(os.uname()[1], traceback_str))
+                   "error:\n{}".format(socket.gethostname(), traceback_str))
        ray.utils.push_error_to_driver_through_redis(
            redis_client, ray_constants.DASHBOARD_DIED_ERROR, message)
        if isinstance(e, OSError) and e.errno == errno.ENOENT:
@@ -5,6 +5,7 @@ import json
 import logging
 import os
 import shutil
+import socket
 import time
 import traceback

@@ -301,7 +302,7 @@ if __name__ == "__main__":
            args.redis_address, password=args.redis_password)
        traceback_str = ray.utils.format_error_message(traceback.format_exc())
        message = ("The log monitor on node {} failed with the following "
-                   "error:\n{}".format(os.uname()[1], traceback_str))
+                   "error:\n{}".format(socket.gethostname(), traceback_str))
        ray.utils.push_error_to_driver_through_redis(
            redis_client, ray_constants.LOG_MONITOR_DIED_ERROR, message)
        raise e
@@ -252,7 +252,7 @@ if __name__ == "__main__":
            args.redis_address, password=args.redis_password)
        traceback_str = ray.utils.format_error_message(traceback.format_exc())
        message = ("The reporter on node {} failed with the following "
-                   "error:\n{}".format(os.uname()[1], traceback_str))
+                   "error:\n{}".format(socket.gethostname(), traceback_str))
        ray.utils.push_error_to_driver_through_redis(
            redis_client, ray_constants.REPORTER_DIED_ERROR, message)
        raise e
@@ -159,12 +159,15 @@ class build_ext(_build_ext.build_ext):
        source = filename
        destination = os.path.join(self.build_lib, filename)
        # Create the target directory if it doesn't already exist.
-        parent_directory = os.path.dirname(destination)
-        if not os.path.exists(parent_directory):
-            os.makedirs(parent_directory)
+        os.makedirs(os.path.dirname(destination), exist_ok=True)
        if not os.path.exists(destination):
            print("Copying {} to {}.".format(source, destination))
-            shutil.copy(source, destination, follow_symlinks=True)
+            if sys.platform == "win32":
+                # Does not preserve file mode (needed to avoid read-only bit)
+                shutil.copyfile(source, destination, follow_symlinks=True)
+            else:
+                # Preserves file mode (needed to copy executable bit)
+                shutil.copy(source, destination, follow_symlinks=True)


 class BinaryDistribution(Distribution):