Fix & improve GitHub Actions CI builds (#7784)

2026-06-27 17:49:47 +08:00 · 2020-03-30 16:29:54 -07:00
parent e356e97eb2
commit f86e623095
14 changed files with 149 additions and 207 deletions
@@ -1,79 +1,44 @@
 name: CI

-env:
-  DEBIAN_FRONTEND: noninteractive
-
 on: [push, pull_request]

 jobs:
-  iwyu:
-    name: ${{ matrix.name }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        name: [
-          ubuntu-clang-iwyu,
-        ]
-        include:
-          - name: ubuntu-clang-iwyu
-            os: ubuntu-latest
-            compiler: clang
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-      with:
-        fetch-depth: 1
-    - name: Setup Bazel
-      shell: bash
-      env:
-        BAZEL_CACHE_CREDENTIAL_B64: ${{ secrets.BAZEL_CACHE_CREDENTIAL_B64 }}
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      run: ./ci/travis/install-bazel.sh
-    - name: Setup Clang Include-What-You-Use
-      shell: bash
-      run: |
-        sudo apt-get install iwyu
-    - name: Perform build
-      continue-on-error: true
-      shell: bash
-      env:
-        CC: ${{ matrix.compiler }}
-      run: |
-        # TODO(mehrdadn): Replace this with the same build script as below, once we factor that out
-        . ./ci/travis/build-helper.sh prep_build_env
-        bazel --batch build \
-          --compilation_mode=fastbuild \
-          --config=iwyu \
-          --keep_going \
-          "//:*"
  build:
    name: ${{ matrix.name }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
-        # Github Actions requires a single row to be added to the build matrix.
-        # See https://help.github.com/en/articles/workflow-syntax-for-github-actions.
-        name: [
-          windows-clang-cl,
-        ]
+        # Useful info: https://help.github.com/en/articles/workflow-syntax-for-github-actions
        include:
-          - name: windows-clang-cl
-            os: windows-latest
+          - name: ubuntu-clang
+            os: ubuntu-16.04
+            compiler: clang
+          - name: windows-clang
+            os: windows-2019
            compiler: clang-cl
+          - name: macos-clang
+            os: macos-10.15
+            compiler: clang
    steps:
+    - name: Configure Git
+      # We add -l to process .bashrc, but need -e -o pipefail for consistency with GitHub Actions's default behavior.
+      shell: bash -e -o pipefail -l {0}
+      run: |
+        git config --global core.symlinks true && git config --global core.autocrlf false
+        if command -v dpkg > /dev/null; then sudo dpkg-reconfigure debconf -f noninteractive -p high; fi
    - name: Checkout repository
      uses: actions/checkout@v2
+      with:
+        fetch-depth: 1
    - name: Setup Bazel
-      shell: bash
+      shell: bash -e -o pipefail -l {0}
      env:
        BAZEL_CACHE_CREDENTIAL_B64: ${{ secrets.BAZEL_CACHE_CREDENTIAL_B64 }}
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      run: ./ci/travis/install-bazel.sh
    - name: Install C/C++ toolchains
-      if: matrix.compiler == 'clang' || matrix.compiler == 'clang-cl'
-      shell: bash
+      shell: bash -e -o pipefail -l {0}
      env:
        CC: ${{ matrix.compiler }}
      run: ./ci/travis/install-toolchains.sh "${CC%%-*}"
@@ -83,9 +48,18 @@ jobs:
      with:
        update: false
    - name: Perform build
-      shell: bash
+      shell: bash -e -o pipefail -l {0}
      run: |
-        . ./ci/travis/build-helper.sh prep_build_env
-        bazel --batch build \
+        bazel build \
          --keep_going \
-          "//:ray_pkg"  # TODO(mehrdadn): Should be "//:*", but we get a linking error with _streaming.so
+          "//:*"
+    - name: Run Clang Include-What-You-Use
+      continue-on-error: true
+      if: runner.os == 'Linux'
+      shell: bash -e -o pipefail -l {0}
+      run: |
+        sudo apt-get install -qq -o=Dpkg::Use-Pty=0 iwyu
+        bazel build \
+          --keep_going \
+          --config=iwyu \
+          "//:*"
@@ -7,7 +7,7 @@ load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library")
 load("@com_github_grpc_grpc//bazel:cython_library.bzl", "pyx_library")
 load("@rules_proto_grpc//python:defs.bzl", "python_grpc_compile")
 load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_cc_library")
-load("//bazel:ray.bzl", "COPTS", "PROPAGATED_WINDOWS_DEFINES")
+load("//bazel:ray.bzl", "COPTS")

 # === Begin of protobuf definitions ===

@@ -394,10 +394,6 @@ cc_library(
        "src/ray/raylet/*.h",
    ]),
    copts = COPTS,
-    defines = select({
-        "@bazel_tools//src/conditions:windows": PROPAGATED_WINDOWS_DEFINES,
-        "//conditions:default": [],
-    }),
    linkopts = select({
        "@bazel_tools//src/conditions:windows": [
        ],
@@ -800,6 +796,7 @@ cc_library(
        "@bazel_tools//src/conditions:windows": glob([
            "src/shims/windows/**/*.c",
            "src/shims/windows/**/*.cc",
+            "src/shims/windows/**/*.h",
        ]),
        "//conditions:default": [],
    }),
@@ -838,7 +835,6 @@ cc_library(
    ],
    visibility = ["//visibility:public"],
    deps = [
-        ":platform_shims",
        ":sha256",
        "@boost//:asio",
        "@com_github_google_glog//:glog",
@@ -1287,23 +1283,11 @@ py_library(
    visibility = ["__subpackages__"],
 )

-genrule(
-    name = "cp_streaming_lib",
-    srcs = ["python/ray/streaming/_streaming.so"],
-    outs = ["cp_streaming_lib.out"],
-    cmd = """
-        set -x &&
-        WORK_DIR=$$(pwd) &&
-        cp -f $(location python/ray/streaming/_streaming.so) "$$WORK_DIR/python/ray/streaming" &&
-        echo "$$WORK_DIR" > $@
-    """,
-    local = 1,
-)
-
 genrule(
    name = "ray_pkg",
    srcs = [
        "python/ray/_raylet.so",
+        "python/ray/streaming/_streaming.so",
        "//:python_sources",
        "//:all_py_proto",
        "//:redis-server",
@@ -1314,10 +1298,7 @@ genrule(
        "//:gcs_server",
        "@plasma//:plasma_store_server",
        "//streaming:copy_streaming_py_proto",
-    ] + select({
-        "@bazel_tools//src/conditions:windows": [],  # ignore build streaming for windows
-        "//conditions:default": [":cp_streaming_lib"],
-    }),
+    ],
    outs = ["ray_pkg.out"],
    cmd = """
        set -x &&
@@ -1334,6 +1315,7 @@ genrule(
            fi;
        fi &&
        cp -f "$${RAYLET}" "$${RAYDIR}/$${RAYLET_TARGET_FILENAME}" &&
+        cp -f $(location python/ray/streaming/_streaming.so) "$$WORK_DIR/python/ray/streaming" &&
        mkdir -p "$$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/" &&
        cp -f $(location //:redis-server) "$$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/" &&
        cp -f $(location //:redis-cli) "$$WORK_DIR/python/ray/core/src/ray/thirdparty/redis/src/" &&
@@ -2,15 +2,13 @@ load("@com_github_google_flatbuffers//:build_defs.bzl", "flatbuffer_library_publ
 load("@com_github_checkstyle_java//checkstyle:checkstyle.bzl", "checkstyle_test")
 load("@bazel_common//tools/maven:pom_file.bzl", "pom_file")

-# TODO(mehrdadn): (How to) support dynamic linking?
-PROPAGATED_WINDOWS_DEFINES = ["RAY_STATIC"]
-
 COPTS = ["-DRAY_USE_GLOG"] + select({
    "@bazel_tools//src/conditions:windows": [
-        "-DWIN32_LEAN_AND_MEAN=",  # Block the inclusion of WinSock.h, which is obsolete and causes errors
        "-Wno-builtin-macro-redefined",  # To get rid of warnings caused by deterministic build macros (e.g. #define __DATE__ "redacted")
        "-Wno-microsoft-unqualified-friend",  # This shouldn't normally be enabled, but otherwise we get: google/protobuf/map_field.h: warning: unqualified friend declaration referring to type outside of the nearest enclosing namespace is a Microsoft extension; add a nested name specifier (for: friend class DynamicMessage)
-    ] + ["-D" + define for define in PROPAGATED_WINDOWS_DEFINES],
+        # TODO(mehrdadn): (How to) support dynamic linking?
+        "-DRAY_STATIC",
+    ],
    "//conditions:default": [
    ],
 })
@@ -240,4 +240,7 @@ def ray_deps_setup():
        build_file = True,
        url = "https://github.com/msgpack/msgpack-c/archive/8085ab8721090a447cf98bb802d1406ad7afe420.tar.gz",
        sha256 = "83c37c9ad926bbee68d564d9f53c6cbb057c1f755c264043ddd87d89e36d15bb",
+        patches = [
+            "//thirdparty/patches:msgpack-windows-iovec.patch",
+        ],
    )
@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-
-prep_build_env() {
-  export PATH="${PATH}:${HOME}/bin"
-  if [ "${OSTYPE}" = "msys" ]; then
-    export USE_CLANG_CL=1
-    export MSYS2_ARG_CONV_EXCL="*"  # Don't let MSYS2 attempt to auto-translate arguments that look like paths
-    local latest_python_bin=""
-    for latest_python_bin in /proc/registry/HKEY_LOCAL_MACHINE/Software/Python/PythonCore/*/InstallPath/@; do
-      if [ -f "${latest_python_bin}" ]; then
-        latest_python_bin="$(tr -d '\0' < "${latest_python_bin}")"
-        latest_python_bin="${latest_python_bin}\\"
-      else
-        latest_python_bin=""
-      fi
-    done
-    latest_python_bin="${latest_python_bin}python.exe"
-    if [ -f "${latest_python_bin}" ]; then
-      export PYTHON2_BIN_PATH="${latest_python_bin}" PYTHON3_BIN_PATH="${latest_python_bin}"
-    fi
-  fi
-}
-
-if [ 0 -lt "$#" ]; then
-  "$@"
-fi
@@ -26,6 +26,22 @@ case "${OSTYPE}" in
    exit 1
 esac

+# Sanity check: Verify we have symlinks where we expect them, or Bazel can produce weird "missing input file" errors.
+# This is most likely to occur on Windows, where symlinks are sometimes disabled by default.
+{ git ls-files -s || true; } | {
+  missing_symlinks=()
+  while read -r mode digest sn path; do
+    if [ "${mode}" = 120000 ]; then
+      test -L "${path}" || missing_symlinks+=("${paths}")
+    fi
+  done
+  if [ ! 0 -eq "${#missing_symlinks[@]}" ]; then
+    echo "error: expected symlink: ${missing_symlinks[@]}" 1>&2
+    echo "For a correct build, please run 'git config --local core.symlinks true' and re-run git checkout." 1>&2
+    false
+  fi
+}
+
 if [ "${OSTYPE}" = "msys" ]; then
  target="${MINGW_DIR-/usr}/bin/bazel.exe"
  mkdir -p "${target%/*}"
@@ -38,6 +54,17 @@ else
  rm -f "${target}"
 fi

+add_missing_lines() {
+  local file="$1"
+  shift
+  local line
+  for line in "$@"; do
+    grep -q -F -x -- "${line}" "${file}" || printf "%s\n" "${line}" >> "${file}"
+  done
+}
+
+add_missing_lines "${HOME}/.bashrc" 'export PATH="${HOME}/bin:${PATH}"'
+
 if [ "${TRAVIS-}" = true ]; then
  # Use bazel disk cache if this script is running in Travis.
  mkdir -p "${HOME}/ray-bazel-cache"
@@ -47,6 +74,26 @@ build --show_timestamps  # Travis doesn't have an option to show timestamps, but
 EOF
 fi
 if [ -n "${GITHUB_WORKFLOW-}" ]; then
+  cat <<"EOF" >> "${HOME}/.profile"
+# Set up environment variables the CI user needs on login to run Bazel on each platform.
+if [ "${OSTYPE}" = "msys" ]; then
+  export USE_CLANG_CL=1
+  export MSYS2_ARG_CONV_EXCL="*"  # Don't let MSYS2 attempt to auto-translate arguments that look like paths
+  latest_python_bin=""  # Detect the system Python from the registry
+  for latest_python_bin in /proc/registry/HKEY_LOCAL_MACHINE/Software/Python/PythonCore/*/InstallPath/@; do
+    if [ -f "${latest_python_bin}" ]; then
+      read -r latest_python_bin < "${latest_python_bin}"
+      latest_python_bin="${latest_python_bin}\\"
+    else
+      latest_python_bin=""
+    fi
+  done
+  latest_python_bin="${latest_python_bin}python.exe"
+  if [ -f "${latest_python_bin}" ]; then
+    export PYTHON2_BIN_PATH="${latest_python_bin}" PYTHON3_BIN_PATH="${latest_python_bin}"
+  fi
+fi
+EOF
  cat <<EOF >> "${HOME}/.bazelrc"
 --output_base=".bazel-out"  # On GitHub Actions, staying on the same volume seems to be faster
 EOF
@@ -54,6 +101,7 @@ fi
 if [ "${TRAVIS-}" = true ] || [ -n "${GITHUB_WORKFLOW-}" ]; then
  cat <<EOF >> "${HOME}/.bazelrc"
 # CI output doesn't scroll, so don't use curses
+build --color=yes
 build --curses=no
 build --progress_report_interval=60
 # Use ray google cloud cache
@@ -2,38 +2,42 @@

 set -euxo pipefail

-LLVM_VERSION_WINDOWS="9.0.0"
+LLVM_VERSION="9.0.0"

 install_clang() {
-  if [ "${OSTYPE}" = "msys" ]; then
-    export MSYS2_ARG_CONV_EXCL="*"  # Don't let MSYS2 attempt to auto-translate arguments that look like paths
-    # Ideally we should be able to use the Chocolatey package manager:
-    #   choco install --no-progress llvm
-    # However, it frequently gives HTTP 503 errors, so we just download and install manually.
-    local target_dir="${PROGRAMFILES}\LLVM"
-    if ! command -v clang "${target_dir}/clang" > /dev/null; then
-      local urldir="https://releases.llvm.org"
-      local arch=64
-      if [ "${HOSTTYPE}" = "${HOSTTYPE%64}" ]; then arch=32; fi
-      local version="${LLVM_VERSION_WINDOWS}"
-      local target="./LLVM-${version}-win${arch}.exe"
-      if [ ! -f "${target}" ]; then
-        mkdir -p -- "${target%/*}"
-        curl -s -L -R -o "${target}" "http://releases.llvm.org/${version}/${target##*/}"
-        chmod +x "${target}"
-      fi
-      if [ "${TRAVIS-}" = true ] || [ -n "${GITHUB_WORKFLOW-}" ]; then
-        7z x "${target}" -o"${target_dir}"  # 7-zip is faster than the self-extracting installer; good for CI
+  local osversion="" url="" urlbase="https://releases.llvm.org" targetdir="/usr/local"
+  case "${OSTYPE}" in
+    msys)
+      export MSYS2_ARG_CONV_EXCL="*"  # Don't let MSYS2 attempt to auto-translate arguments that look like paths
+      osversion=win
+      if [ ! "${HOSTTYPE}" = "${HOSTTYPE%64}" ]; then
+        osversion="${osversion}64"
      else
-        "${target}" /S  # for normal users we should install properly
-        rm -f -- "${target}"
+        osversion="${osversion}32"
      fi
-    fi
-  elif 1>&- command -v pacman; then
-    sudo pacman -S --needed --noconfirm --noprogressbar clang
-  elif 1>&- command -v apt-get; then
-    sudo apt-get -q -y install clang
+      url="${urlbase}/${LLVM_VERSION}/LLVM-${LLVM_VERSION}-${osversion}.exe"
+      targetdir="${PROGRAMFILES}\LLVM"
+      ;;
+    linux-gnu)
+      osversion="${OSTYPE}-$(sed -n -e '/^PRETTY_NAME/ { s/^[^=]*="\(.*\)"/\1/g; s/ /-/; s/\([0-9]*\.[0-9]*\)\.[0-9]*/\1/; s/ .*//; p }' /etc/os-release | tr '[:upper:]' '[:lower:]')"
+      ;;
+    darwin*)
+      osversion="darwin-apple"
+      ;;
+  esac
+  if [ -z "${url}" ]; then
+    url="${urlbase}/${LLVM_VERSION}/clang+llvm-${LLVM_VERSION}-${HOSTTYPE}-${osversion}.tar.xz"
  fi
+  curl -s -L -R "${url}" | if [ "${OSTYPE}" = "msys" ]; then
+    local target="./${url##*/}"
+    install /dev/stdin "${target}"
+    7z x "${target}" -o"${targetdir}"
+    rm -f -- "${target}"
+  else
+    sudo tar -x -J --strip-components=1 -C "${targetdir}"
+    command -V clang 1>&2
+  fi
+  "${targetdir}"/bin/clang --version 1>&2
 }

 install_"$@"
@@ -16,8 +16,6 @@ syntax = "proto3";

 package ray.rpc;

-import "src/ray/protobuf/common.proto";
-
 message GetProfilingStatsRequest {
  // PID of the worker process.
  uint32 pid = 1;
@@ -14,8 +14,6 @@

 #include "ray/raylet/worker_pool.h"

-#include <sys/wait.h>
-
 #include <algorithm>
 #include <boost/date_time/posix_time/posix_time.hpp>

@@ -51,6 +51,9 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <string.h>

 #ifdef _WIN32
+#ifndef _WINSOCKAPI_
+#include <WinSock2.h>
+#endif
 #include <Windows.h>
 #include <io.h>
 #include <ws2tcpip.h> /* socklen_t, et al (MSVC20xx) */
@@ -1,27 +0,0 @@
-#include <sys/wait.h>
-
-#ifndef WIN32_LEAN_AND_MEAN
-#define WIN32_LEAN_AND_MEAN 1
-#endif
-#include <Windows.h>
-
-pid_t waitpid(pid_t pid, int *status, int options) {
-  int result;
-  if (pid <= 0) {
-    result = -1;
-    errno = ECHILD;
-  } else if (HANDLE process = OpenProcess(SYNCHRONIZE, FALSE, pid)) {
-    DWORD timeout = status && *status == WNOHANG ? 0 : INFINITE;
-    if (WaitForSingleObject(process, timeout) != WAIT_FAILED) {
-      result = 0;
-    } else {
-      result = -1;
-      errno = ECHILD;
-    }
-    CloseHandle(process);
-  } else {
-    result = -1;
-    errno = ECHILD;
-  }
-  return result;
-}
@@ -1,15 +0,0 @@
-#ifndef WAIT_H
-#define WAIT_H
-
-#include <unistd.h>  // pid_t
-
-#define WNOHANG 1
-
-__declspec(
-    deprecated("Waiting on a process by ID has an inherent race condition"
-               " on Windows and is discouraged. "
-               "Please use a wrapper that keeps the process handle alive"
-               " and waits on it directly as needed."
-               "")) pid_t waitpid(pid_t pid, int *status, int options);
-
-#endif /* WAIT_H */
@@ -71,25 +71,3 @@ unsigned sleep(unsigned seconds) {
  Sleep(seconds * 1000);
  return 0;
 }
-
-int kill(pid_t pid, int sig) {
-  int result;
-  if (HANDLE process = OpenProcess(PROCESS_TERMINATE, FALSE, pid)) {
-    if (sig == SIGKILL) {
-      if (TerminateProcess(process, ERROR_PROCESS_ABORTED)) {
-        result = 0;
-      } else {
-        result = -1;
-        errno = EPERM;
-      }
-    } else {
-      result = -1;
-      errno = EINVAL;
-    }
-    CloseHandle(process);
-  } else {
-    result = -1;
-    errno = ESRCH;
-  }
-  return result;
-}
@@ -0,0 +1,24 @@
+diff --git include/msgpack/v1/vrefbuffer.hpp include/msgpack/v1/vrefbuffer.hpp
+--- include/msgpack/v1/vrefbuffer.hpp
+++ include/msgpack/v1/vrefbuffer.hpp
+@@ -28,4 +28,12 @@
+-struct iovec {
+-    void  *iov_base;
+-    size_t iov_len;
+-};
+#ifndef _WS2DEF_
+#include <Winsock2.h>
+#endif
+#ifndef iovec
+#define iovec _WSABUF
+#endif
+#ifndef iov_base
+#define iov_base buf
+#endif
+#ifndef iov_len
+#define iov_len len
+#endif
+@@ -171,1 +179,1 @@
+-                const_cast<const void *>((m_tail - 1)->iov_base)
+                (m_tail - 1)->iov_base
+--