[CI] Factor out more Travis code and update GitHub Actions (#8085)

.travis.yml

@@ -121,8 +121,8 @@ matrix:
         - python setup.py check --restructuredtext --strict --metadata
         - cd ..
         # Run Bazel linter Buildifier.
-        - wget -q https://dl.google.com/go/go1.12.linux-amd64.tar.gz
-        - tar -xf go1.12.linux-amd64.tar.gz
+        - wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz
+        - tar -xf go1.14.2.linux-amd64.tar.gz
         - mkdir $HOME/go_dir
         - export GOROOT=`pwd`/go
         - export GOPATH="$HOME/go_dir"
@@ -313,8 +313,8 @@ script:
 
   # ray operator tests
   - cd ./deploy/ray-operator/
-  - ../../ci/suppress_output go build
-  - ../../ci/suppress_output go test ./...
+  - CC=gcc ../../ci/suppress_output go build
+  - CC=gcc ../../ci/suppress_output go test ./...
   - cd ../..
 
   # random python tests TODO(ekl): these should be moved to bazel

build.sh

@@ -31,6 +31,8 @@ if [[ "$unamestr" == "Linux" ]]; then
   PARALLEL=1
 elif [[ "$unamestr" == "Darwin" ]]; then
   PARALLEL=$(sysctl -n hw.ncpu)
+elif [[ "${OSTYPE}" == "msys" ]]; then
+  PARALLEL="${NUMBER_OF_PROCESSORS-1}"
 else
   echo "Unrecognized platform."
   exit 1
@@ -106,8 +108,15 @@ echo "Using Python executable $PYTHON_EXECUTABLE."
 
 # Find the bazel executable. The script ci/travis/install-bazel.sh doesn't
 # always put the bazel executable on the PATH.
-BAZEL_EXECUTABLE=$(PATH="$PATH:$HOME/.bazel/bin" which bazel)
-echo "Using Bazel executable $BAZEL_EXECUTABLE."
+if [ -z "${BAZEL_EXECUTABLE-}" ]; then
+  BAZEL_EXECUTABLE=$(PATH="$PATH:$HOME/.bazel/bin" which bazel)
+fi
+if [ -f "${BAZEL_EXECUTABLE}" ]; then
+  echo "Using Bazel executable $BAZEL_EXECUTABLE."
+else
+  echo "Bazel not found: BAZEL_EXECUTABLE=\"${BAZEL_EXECUTABLE}\""
+  exit 1
+fi
 
 # Now we build everything.
 BUILD_DIR="$ROOT_DIR/build/"
@@ -130,7 +139,7 @@ popd
 
 
 if [ -z "$SKIP_THIRDPARTY_INSTALL" ]; then
-    "$PYTHON_EXECUTABLE" -m pip install -q psutil setproctitle \
+    CC=gcc "$PYTHON_EXECUTABLE" -m pip install -q psutil setproctitle \
             --target="$ROOT_DIR/python/ray/thirdparty_files"
 fi
 

ci/travis/check_import_order.py

@@ -9,6 +9,7 @@ some/file/path.py:23 import psutil without explicitly import ray before it.
 """
 
 import argparse
+import io
 import re
 import sys
 from pathlib import Path
@@ -23,7 +24,7 @@ def check_import(file):
         "import setproctitle": -1
     }
 
-    with open(file) as f:
+    with io.open(file, "r", encoding="utf-8") as f:
         for i, line in enumerate(f):
             for check in check_to_lines.keys():
                 # This regex will match the following case

ci/travis/ci.sh

@@ -4,11 +4,19 @@
 
 unset -f cd  # Travis defines this on Mac for RVM, but it floods the trace log and isn't relevant for us
 
-set -eo pipefail && if [ -n "${OSTYPE##darwin*}" ]; then set -ux; fi  # some options interfere with Travis's RVM on Mac
+set -eo pipefail && if [ -z "${TRAVIS_PULL_REQUEST-}" ] || [ -n "${OSTYPE##darwin*}" ]; then set -ux; fi  # some options interfere with Travis's RVM on Mac
 
 ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)"
 WORKSPACE_DIR="${ROOT_DIR}/../.."
 
+suppress_output() {
+  "${WORKSPACE_DIR}"/ci/suppress_output "$@"
+}
+
+keep_alive() {
+  "${WORKSPACE_DIR}"/ci/keep_alive "$@"
+}
+
 # If provided the names of one or more environment variables, returns success if any of them is triggered.
 # Usage: should_run_job [VAR_NAME]...
 should_run_job() {
@@ -17,7 +25,7 @@ should_run_job() {
     local envvar active_triggers=()
     for envvar in "$@"; do
       if [ "${!envvar}" = 1 ]; then
-        active_triggers+="${envvar}=${!envvar}"  # success! we found at least one of the given triggers is occurring
+        active_triggers+=("${envvar}=${!envvar}")  # success! we found at least one of the given triggers is occurring
       fi
     done
     if [ 0 -eq "${#active_triggers[@]}" ]; then
@@ -53,16 +61,33 @@ preload() {
   local variable_definitions
   variable_definitions=($(python "${ROOT_DIR}"/determine_tests_to_run.py))
   if [ 0 -lt "${#variable_definitions[@]}" ]; then
-    export "${variable_definitions[@]}"
+    local expression
+    expression="$(printf "%q " "${variable_definitions[@]}")"
+    eval "${expression}"
+    printf "%s\n" "${expression}" >> ~/.bashrc
   fi
 
   if ! (set +x && should_run_job ${job_names//,/ }); then
+    if [ -n "${GITHUB_WORKFLOW-}" ]; then
+      # If this job is to be skipped, emit an 'exit' command into .bashrc to quickly exit all following steps.
+      # This isn't needed for Travis (since everything runs in a single shell), but it is needed for GitHub Actions.
+      cat <<EOF1 >> ~/.bashrc
+      cat <<EOF2 1>&2
+Exiting shell as no triggers were active for this job:
+  ${job_names//,/}
+The active triggers during job initialization were the following:
+  ${variable_definitions[*]}
+EOF2
+      exit 0
+EOF1
+    fi
     exit 0
   fi
 }
 
 # Initializes the environment for the current job. Performs the following tasks:
-# - Calls 'exit 0' to quickly exit if provided a list of job names and none of them has been triggered.
+# - Calls 'exit 0' in this job step and all subsequent steps to quickly exit if provided a list of job names and
+#   none of them has been triggered.
 # - Sets variables to indicate the job names that have been triggered.
 #   Note: Please avoid exporting these variables. Instead, source any callees that need to use them.
 #   This helps reduce implicit coupling of callees to their parents, as they will be unable to run when not sourced, (especially with set -u).
@@ -99,7 +124,7 @@ build() {
   fi
 
   if [ "${RAY_DEFAULT_BUILD-}" = 1 ]; then
-    eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=master bash)"
+    eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=stable bash)"
   fi
 
   if [ "${LINUX_WHEELS-}" = 1 ]; then
@@ -111,12 +136,12 @@ build() {
 
     # This command should be kept in sync with ray/python/README-building-wheels.md,
     # except the "${MOUNT_BAZEL_CACHE[@]}" part.
-    "${WORKSPACE_DIR}"/ci/suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" -ti rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh
+    suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" -ti rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh
   fi
 
   if [ "${MAC_WHEELS-}" = 1 ]; then
     # This command should be kept in sync with ray/python/README-building-wheels.md.
-    "${WORKSPACE_DIR}"/ci/suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh
+    suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh
   fi
 }
 

ci/travis/install-toolchains.sh

@@ -32,6 +32,17 @@ install_toolchains() {
     install /dev/stdin "${target}"
     7z x -bsp0 -bso0 "${target}" -o"${targetdir}"
     rm -f -- "${target}"
+    (
+      # Add Clang/LLVM binaries to somewhere that's in already PATH
+      # (don't change PATH itself, to avoid invalidating Bazel's cache or having to manage environment variables)
+      mkdir -p -- ~/bin
+      set +x
+      local path
+      for path in "${targetdir}\\bin"/*.exe; do
+        local name="${path##*/}"
+        printf "%s\n" "#!/usr/bin/env bash" "exec \"${path}\" \"\$@\"" | install /dev/stdin ~/bin/"${name%.*}"
+      done
+    )
   else
     sudo tar -x -J --strip-components=1 -C "${targetdir}"
     command -V clang 1>&2

python/setup.py

@@ -14,15 +14,21 @@ import setuptools.command.build_ext as _build_ext
 # before these files have been created, so we have to move the files
 # manually.
 
+exe_suffix = ".exe" if sys.platform == "win32" else ""
+
+# .pyd is the extension Python requires on Windows for shared libraries.
+# https://docs.python.org/3/faq/windows.html#is-a-pyd-file-the-same-as-a-dll
+pyd_suffix = ".pyd" if sys.platform == "win32" else ".so"
+
 # NOTE: The lists below must be kept in sync with ray/BUILD.bazel.
 ray_files = [
     "ray/core/src/ray/thirdparty/redis/src/redis-server",
     "ray/core/src/ray/gcs/redis_module/libray_redis_module.so",
-    "ray/core/src/plasma/plasma_store_server",
-    "ray/_raylet.so",
-    "ray/core/src/ray/raylet/raylet_monitor",
-    "ray/core/src/ray/gcs/gcs_server",
-    "ray/core/src/ray/raylet/raylet",
+    "ray/core/src/plasma/plasma_store_server" + exe_suffix,
+    "ray/_raylet" + pyd_suffix,
+    "ray/core/src/ray/raylet/raylet_monitor" + exe_suffix,
+    "ray/core/src/ray/gcs/gcs_server" + exe_suffix,
+    "ray/core/src/ray/raylet/raylet" + exe_suffix,
     "ray/streaming/_streaming.so",
 ]
 
@@ -103,6 +109,12 @@ class build_ext(_build_ext.build_ext):
         # that certain flags will not be passed along such as --user or sudo.
         # TODO(rkn): Fix this.
         command = ["../build.sh", "-p", sys.executable]
+        if sys.platform == "win32" and command[0].lower().endswith(".sh"):
+            # We can't run .sh files directly in Windows, so find a shell.
+            # Don't use "bash" instead of "sh", because that might run the Bash
+            # from WSL! (We want MSYS2's Bash, which is also sh by default.)
+            shell = os.getenv("BAZEL_SH", "sh")  # NOT "bash"! (see above)
+            command.insert(0, shell)
         if build_java:
             # Also build binaries for Java if the above env variable exists.
             command += ["-l", "python,java"]