From cd0037064c393a070e9b4c22b10ee52416c93b9d Mon Sep 17 00:00:00 2001 From: mehrdadn Date: Tue, 12 May 2020 22:06:04 -0700 Subject: [PATCH] Windows wheels for multiple Python versions (#8369) * Upload wheels to latest directory as well on GitHub Actions * Fix bug in install-dependencies.sh * Move out bazel build //:* from install_ray, since it isn't really necessary for that purpose * Build wheels for different versions of Python on Windows * Compile Windows in opt mode Co-authored-by: Mehrdad --- .bazelrc | 4 +- build.sh | 12 +++++- ci/travis/bazel-preclean.sh | 44 ++++++++++++++++++++ ci/travis/ci.sh | 68 +++++++++++++++++++++++++------ ci/travis/install-dependencies.sh | 2 +- 5 files changed, 112 insertions(+), 18 deletions(-) create mode 100755 ci/travis/bazel-preclean.sh diff --git a/.bazelrc b/.bazelrc index 526f78af5..127a4aebd 100644 --- a/.bazelrc +++ b/.bazelrc @@ -5,9 +5,7 @@ build --enable_platform_specific_config # On all platforms, provide: PYTHON3_BIN_PATH=python ############################################################################### build --action_env=PATH -build:linux --compilation_mode=opt -build:macos --compilation_mode=opt -build:windows --compilation_mode=fastbuild +build --compilation_mode=opt # This workaround is needed to prevent Bazel from compiling the same file twice (once PIC and once not). build:linux --force_pic build:macos --force_pic diff --git a/build.sh b/build.sh index d4d8433c9..8c075f641 100755 --- a/build.sh +++ b/build.sh @@ -127,8 +127,15 @@ fi pushd "$BUILD_DIR" -WORK_DIR=`mktemp -d` -pushd $WORK_DIR +need_pickle5_backport=0 +PYTHON_REVISION="$("$PYTHON_EXECUTABLE" -s -c "import sys; print('{}.{}.{}'.format(*sys.version_info))")" +case "${PYTHON_REVISION}.0" in +3\.[5-7].*|3.8.[0-1].*) need_pickle5_backport=1;; +esac + +if [ 0 -ne "${need_pickle5_backport}" ]; then +WORK_DIR="$(mktemp -d)" +pushd "${WORK_DIR}" git clone https://github.com/suquark/pickle5-backport pushd pickle5-backport git checkout 8ffe41ceba9d5e2ce8a98190f6b3d2f3325e5a72 @@ -136,6 +143,7 @@ pushd pickle5-backport unzip -q -o dist/*.whl -d "$ROOT_DIR/python/ray/pickle5_files" popd popd +fi if [ -z "$SKIP_THIRDPARTY_INSTALL" ]; then diff --git a/ci/travis/bazel-preclean.sh b/ci/travis/bazel-preclean.sh new file mode 100755 index 000000000..07a1a014d --- /dev/null +++ b/ci/travis/bazel-preclean.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Args: [Bazel-target] +# +# This script cleans up any genrule() outputs in the transitive dependencies of the provided target. +# +# This is useful for forcing genrule actions to re-run, because the _true_ outputs of those actions +# can include a larger set of files (e.g. files copied to the workspace) which Bazel is unable to +# detect changes to (or delete changes of). +# +# Usually, you would run this script along with 'git clean -f', to make sure Bazel re-copies outputs +# the next time a build occurs. + +( + set -euo pipefail + bazel aquery --color=no --show_progress=no --output=textproto \ + "mnemonic(\"Genrule\", deps(${1-//:*}))" | awk ' + { + body = 0; + } + /^^ / { + body = 1; + } + /^^\S.* {$/ { + section = $1; + delete arr; + } + body { + if (section == "artifacts") { + p = $2; + if ($1 == "exec_path:") { + p = substr(p, 2, length(p) - 2); # strip quotes + } + arr[$1] = p; + } + } + /^^}/ { + artifacts[arr["id:"]] = arr["exec_path:"]; # save the ID -> artifact mapping + } + /^^ *output_ids:/ { + print(artifacts[$2]); # print the output artifact + } + ' | tr "\n" "\0" | xargs -0 -r -- rm -f -- +) diff --git a/ci/travis/ci.sh b/ci/travis/ci.sh index e75be9585..8c901135c 100755 --- a/ci/travis/ci.sh +++ b/ci/travis/ci.sh @@ -17,6 +17,18 @@ keep_alive() { "${WORKSPACE_DIR}"/ci/keep_alive "$@" } +# Calls the provided command with set -x temporarily suppressed +suppress_xtrace() { + { + local restore_shell_state="" + if [ -o xtrace ]; then set +x; restore_shell_state="set -x"; fi + } 2> /dev/null + local status=0 + "$@" || status=$? + ${restore_shell_state} + { return "${status}"; } 2> /dev/null +} + # If provided the names of one or more environment variables, returns 0 if any of them is triggered. # Usage: should_run_job [VAR_NAME]... should_run_job() { @@ -83,11 +95,14 @@ upload_wheels() { if [ -z "${branch}" ]; then branch="${TRAVIS_BRANCH-}"; fi if [ -z "${branch}" ]; then echo "Unable to detect branch name" 1>&2; return 1; fi local local_dir="python/dist" - local remote_dir="${branch}/${commit}" if [ -d "${local_dir}" ]; then - if command -V aws; then - aws s3 sync --acl public-read --no-progress "${local_dir}" "s3://ray-wheels/${remote_dir}" - fi + ls -a -l -- "${local_dir}" + local remote_dir + for remote_dir in latest "${branch}/${commit}"; do + if command -V aws; then + aws s3 sync --acl public-read --no-progress "${local_dir}" "s3://ray-wheels/${remote_dir}" + fi + done fi } @@ -163,7 +178,7 @@ install_cython_examples() { install_go() { local gimme_url="https://raw.githubusercontent.com/travis-ci/gimme/master/gimme" - eval "$(curl -f -s -L "${gimme_url}" | GIMME_GO_VERSION=1.14.2 bash)" + suppress_xtrace eval "$(curl -f -s -L "${gimme_url}" | GIMME_GO_VERSION=1.14.2 bash)" if [ -z "${GOPATH-}" ]; then GOPATH="${GOPATH:-${HOME}/go_dir}" @@ -172,7 +187,6 @@ install_go() { } install_ray() { - bazel build -k "//:*" # Do a full build first to ensure everything passes ( cd "${WORKSPACE_DIR}"/python build_dashboard_front_end @@ -206,13 +220,42 @@ build_wheels() { suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh ;; msys*) - if ! python -c "import ray" 2> /dev/null; then - # This installs e.g. the pickle5 dependencies, which are needed for correct wheel testing - install_ray - fi ( - cd "${WORKSPACE_DIR}"/python - python setup.py --quiet bdist_wheel + local backup_conda="${CONDA_PREFIX}.bak" ray_uninstall_status=0 + test ! -d "${backup_conda}" + pip uninstall -y ray || ray_uninstall_status=1 + mv -n -T -- "${CONDA_PREFIX}" "${backup_conda}" # Back up conda + + local pyversion pyversions=() + for pyversion in 3.6 3.7 3.8; do + if [ "${pyversion}" = "${PYTHON-}" ]; then continue; fi # we'll build ${PYTHON} last + pyversions+=("${pyversion}") + done + + pyversions+=("${PYTHON-}") # build this last so any subsequent steps use the right version + local local_dir="python/dist" + for pyversion in "${pyversions[@]}"; do + if [ -z "${pyversion}" ]; then continue; fi + "${ROOT_DIR}"/bazel-preclean.sh + git clean -f -f -x -d -e "${local_dir}" -e python/ray/dashboard/client + git checkout -q -f -- . + cp -R -f -a -T -- "${backup_conda}" "${CONDA_PREFIX}" + local existing_version + existing_version="$(python -s -c "import sys; print('%s.%s' % sys.version_info[:2])")" + if [ "${pyversion}" != "${existing_version}" ]; then + suppress_xtrace conda install python="${pyversion}" + fi + install_ray + (cd "${WORKSPACE_DIR}"/python && python setup.py --quiet bdist_wheel) + pip uninstall -y ray + rm -r -f -- "${CONDA_PREFIX}" + done + + mv -n -T -- "${backup_conda}" "${CONDA_PREFIX}" + "${ROOT_DIR}"/bazel-preclean.sh + if [ 0 -eq "${ray_uninstall_status}" ]; then # If Ray was previously installed, restore it + install_ray + fi ) ;; esac @@ -360,6 +403,7 @@ init() { build() { if ! need_wheels; then + bazel build -k "//:*" # Do a full build first to ensure everything passes install_ray if [ "${LINT-}" = 1 ]; then # Try generating Sphinx documentation. To do this, we need to install Ray first. diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh index 17a848067..baeab4063 100755 --- a/ci/travis/install-dependencies.sh +++ b/ci/travis/install-dependencies.sh @@ -254,7 +254,7 @@ install_dependencies() { # Additional streaming dependencies. if [ "${RAY_CI_STREAMING_PYTHON_AFFECTED}" = 1 ]; then - pip install msgpack>=0.6.2 + pip install "msgpack>=0.6.2" fi if [ -n "${PYTHON-}" ] || [ -n "${LINT-}" ] || [ "${MAC_WHEELS-}" = 1 ]; then