From 0a54407961bc2184845fa3f18949e6e8e041a034 Mon Sep 17 00:00:00 2001 From: mehrdadn Date: Tue, 21 Apr 2020 09:53:08 -0700 Subject: [PATCH] [CI] Factor out more Travis code and update GitHub Actions (#8085) --- .travis.yml | 8 +++---- build.sh | 15 ++++++++++--- ci/travis/check_import_order.py | 3 ++- ci/travis/ci.sh | 39 +++++++++++++++++++++++++++------ ci/travis/install-toolchains.sh | 11 ++++++++++ python/setup.py | 22 ++++++++++++++----- 6 files changed, 78 insertions(+), 20 deletions(-) diff --git a/.travis.yml b/.travis.yml index c9231f718..e4720b054 100644 --- a/.travis.yml +++ b/.travis.yml @@ -121,8 +121,8 @@ matrix: - python setup.py check --restructuredtext --strict --metadata - cd .. # Run Bazel linter Buildifier. - - wget -q https://dl.google.com/go/go1.12.linux-amd64.tar.gz - - tar -xf go1.12.linux-amd64.tar.gz + - wget -q https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz + - tar -xf go1.14.2.linux-amd64.tar.gz - mkdir $HOME/go_dir - export GOROOT=`pwd`/go - export GOPATH="$HOME/go_dir" @@ -313,8 +313,8 @@ script: # ray operator tests - cd ./deploy/ray-operator/ - - ../../ci/suppress_output go build - - ../../ci/suppress_output go test ./... + - CC=gcc ../../ci/suppress_output go build + - CC=gcc ../../ci/suppress_output go test ./... - cd ../.. # random python tests TODO(ekl): these should be moved to bazel diff --git a/build.sh b/build.sh index fbe85bbf3..d2462ef06 100755 --- a/build.sh +++ b/build.sh @@ -31,6 +31,8 @@ if [[ "$unamestr" == "Linux" ]]; then PARALLEL=1 elif [[ "$unamestr" == "Darwin" ]]; then PARALLEL=$(sysctl -n hw.ncpu) +elif [[ "${OSTYPE}" == "msys" ]]; then + PARALLEL="${NUMBER_OF_PROCESSORS-1}" else echo "Unrecognized platform." exit 1 @@ -106,8 +108,15 @@ echo "Using Python executable $PYTHON_EXECUTABLE." # Find the bazel executable. The script ci/travis/install-bazel.sh doesn't # always put the bazel executable on the PATH. -BAZEL_EXECUTABLE=$(PATH="$PATH:$HOME/.bazel/bin" which bazel) -echo "Using Bazel executable $BAZEL_EXECUTABLE." +if [ -z "${BAZEL_EXECUTABLE-}" ]; then + BAZEL_EXECUTABLE=$(PATH="$PATH:$HOME/.bazel/bin" which bazel) +fi +if [ -f "${BAZEL_EXECUTABLE}" ]; then + echo "Using Bazel executable $BAZEL_EXECUTABLE." +else + echo "Bazel not found: BAZEL_EXECUTABLE=\"${BAZEL_EXECUTABLE}\"" + exit 1 +fi # Now we build everything. BUILD_DIR="$ROOT_DIR/build/" @@ -130,7 +139,7 @@ popd if [ -z "$SKIP_THIRDPARTY_INSTALL" ]; then - "$PYTHON_EXECUTABLE" -m pip install -q psutil setproctitle \ + CC=gcc "$PYTHON_EXECUTABLE" -m pip install -q psutil setproctitle \ --target="$ROOT_DIR/python/ray/thirdparty_files" fi diff --git a/ci/travis/check_import_order.py b/ci/travis/check_import_order.py index 863670163..93809af82 100644 --- a/ci/travis/check_import_order.py +++ b/ci/travis/check_import_order.py @@ -9,6 +9,7 @@ some/file/path.py:23 import psutil without explicitly import ray before it. """ import argparse +import io import re import sys from pathlib import Path @@ -23,7 +24,7 @@ def check_import(file): "import setproctitle": -1 } - with open(file) as f: + with io.open(file, "r", encoding="utf-8") as f: for i, line in enumerate(f): for check in check_to_lines.keys(): # This regex will match the following case diff --git a/ci/travis/ci.sh b/ci/travis/ci.sh index ada7c5675..7065ac47f 100755 --- a/ci/travis/ci.sh +++ b/ci/travis/ci.sh @@ -4,11 +4,19 @@ unset -f cd # Travis defines this on Mac for RVM, but it floods the trace log and isn't relevant for us -set -eo pipefail && if [ -n "${OSTYPE##darwin*}" ]; then set -ux; fi # some options interfere with Travis's RVM on Mac +set -eo pipefail && if [ -z "${TRAVIS_PULL_REQUEST-}" ] || [ -n "${OSTYPE##darwin*}" ]; then set -ux; fi # some options interfere with Travis's RVM on Mac ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)" WORKSPACE_DIR="${ROOT_DIR}/../.." +suppress_output() { + "${WORKSPACE_DIR}"/ci/suppress_output "$@" +} + +keep_alive() { + "${WORKSPACE_DIR}"/ci/keep_alive "$@" +} + # If provided the names of one or more environment variables, returns success if any of them is triggered. # Usage: should_run_job [VAR_NAME]... should_run_job() { @@ -17,7 +25,7 @@ should_run_job() { local envvar active_triggers=() for envvar in "$@"; do if [ "${!envvar}" = 1 ]; then - active_triggers+="${envvar}=${!envvar}" # success! we found at least one of the given triggers is occurring + active_triggers+=("${envvar}=${!envvar}") # success! we found at least one of the given triggers is occurring fi done if [ 0 -eq "${#active_triggers[@]}" ]; then @@ -53,16 +61,33 @@ preload() { local variable_definitions variable_definitions=($(python "${ROOT_DIR}"/determine_tests_to_run.py)) if [ 0 -lt "${#variable_definitions[@]}" ]; then - export "${variable_definitions[@]}" + local expression + expression="$(printf "%q " "${variable_definitions[@]}")" + eval "${expression}" + printf "%s\n" "${expression}" >> ~/.bashrc fi if ! (set +x && should_run_job ${job_names//,/ }); then + if [ -n "${GITHUB_WORKFLOW-}" ]; then + # If this job is to be skipped, emit an 'exit' command into .bashrc to quickly exit all following steps. + # This isn't needed for Travis (since everything runs in a single shell), but it is needed for GitHub Actions. + cat <> ~/.bashrc + cat <&2 +Exiting shell as no triggers were active for this job: + ${job_names//,/} +The active triggers during job initialization were the following: + ${variable_definitions[*]} +EOF2 + exit 0 +EOF1 + fi exit 0 fi } # Initializes the environment for the current job. Performs the following tasks: -# - Calls 'exit 0' to quickly exit if provided a list of job names and none of them has been triggered. +# - Calls 'exit 0' in this job step and all subsequent steps to quickly exit if provided a list of job names and +# none of them has been triggered. # - Sets variables to indicate the job names that have been triggered. # Note: Please avoid exporting these variables. Instead, source any callees that need to use them. # This helps reduce implicit coupling of callees to their parents, as they will be unable to run when not sourced, (especially with set -u). @@ -99,7 +124,7 @@ build() { fi if [ "${RAY_DEFAULT_BUILD-}" = 1 ]; then - eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=master bash)" + eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=stable bash)" fi if [ "${LINUX_WHEELS-}" = 1 ]; then @@ -111,12 +136,12 @@ build() { # This command should be kept in sync with ray/python/README-building-wheels.md, # except the "${MOUNT_BAZEL_CACHE[@]}" part. - "${WORKSPACE_DIR}"/ci/suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" -ti rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh + suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" -ti rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh fi if [ "${MAC_WHEELS-}" = 1 ]; then # This command should be kept in sync with ray/python/README-building-wheels.md. - "${WORKSPACE_DIR}"/ci/suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh + suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh fi } diff --git a/ci/travis/install-toolchains.sh b/ci/travis/install-toolchains.sh index 9d71ccd2e..9d33e7729 100755 --- a/ci/travis/install-toolchains.sh +++ b/ci/travis/install-toolchains.sh @@ -32,6 +32,17 @@ install_toolchains() { install /dev/stdin "${target}" 7z x -bsp0 -bso0 "${target}" -o"${targetdir}" rm -f -- "${target}" + ( + # Add Clang/LLVM binaries to somewhere that's in already PATH + # (don't change PATH itself, to avoid invalidating Bazel's cache or having to manage environment variables) + mkdir -p -- ~/bin + set +x + local path + for path in "${targetdir}\\bin"/*.exe; do + local name="${path##*/}" + printf "%s\n" "#!/usr/bin/env bash" "exec \"${path}\" \"\$@\"" | install /dev/stdin ~/bin/"${name%.*}" + done + ) else sudo tar -x -J --strip-components=1 -C "${targetdir}" command -V clang 1>&2 diff --git a/python/setup.py b/python/setup.py index 02b28517e..8e5e49538 100644 --- a/python/setup.py +++ b/python/setup.py @@ -14,15 +14,21 @@ import setuptools.command.build_ext as _build_ext # before these files have been created, so we have to move the files # manually. +exe_suffix = ".exe" if sys.platform == "win32" else "" + +# .pyd is the extension Python requires on Windows for shared libraries. +# https://docs.python.org/3/faq/windows.html#is-a-pyd-file-the-same-as-a-dll +pyd_suffix = ".pyd" if sys.platform == "win32" else ".so" + # NOTE: The lists below must be kept in sync with ray/BUILD.bazel. ray_files = [ "ray/core/src/ray/thirdparty/redis/src/redis-server", "ray/core/src/ray/gcs/redis_module/libray_redis_module.so", - "ray/core/src/plasma/plasma_store_server", - "ray/_raylet.so", - "ray/core/src/ray/raylet/raylet_monitor", - "ray/core/src/ray/gcs/gcs_server", - "ray/core/src/ray/raylet/raylet", + "ray/core/src/plasma/plasma_store_server" + exe_suffix, + "ray/_raylet" + pyd_suffix, + "ray/core/src/ray/raylet/raylet_monitor" + exe_suffix, + "ray/core/src/ray/gcs/gcs_server" + exe_suffix, + "ray/core/src/ray/raylet/raylet" + exe_suffix, "ray/streaming/_streaming.so", ] @@ -103,6 +109,12 @@ class build_ext(_build_ext.build_ext): # that certain flags will not be passed along such as --user or sudo. # TODO(rkn): Fix this. command = ["../build.sh", "-p", sys.executable] + if sys.platform == "win32" and command[0].lower().endswith(".sh"): + # We can't run .sh files directly in Windows, so find a shell. + # Don't use "bash" instead of "sh", because that might run the Bash + # from WSL! (We want MSYS2's Bash, which is also sh by default.) + shell = os.getenv("BAZEL_SH", "sh") # NOT "bash"! (see above) + command.insert(0, shell) if build_java: # Also build binaries for Java if the above env variable exists. command += ["-l", "python,java"]