mirror of
https://github.com/wassname/ray.git
synced 2026-06-27 18:06:25 +08:00
Windows wheels for multiple Python versions (#8369)
* Upload wheels to latest directory as well on GitHub Actions * Fix bug in install-dependencies.sh * Move out bazel build //:* from install_ray, since it isn't really necessary for that purpose * Build wheels for different versions of Python on Windows * Compile Windows in opt mode Co-authored-by: Mehrdad <noreply@github.com>
This commit is contained in:
mehrdadn
@@ -5,9 +5,7 @@ build --enable_platform_specific_config
|
||||
# On all platforms, provide: PYTHON3_BIN_PATH=python
|
||||
###############################################################################
|
||||
build --action_env=PATH
|
||||
build:linux --compilation_mode=opt
|
||||
build:macos --compilation_mode=opt
|
||||
build:windows --compilation_mode=fastbuild
|
||||
build --compilation_mode=opt
|
||||
# This workaround is needed to prevent Bazel from compiling the same file twice (once PIC and once not).
|
||||
build:linux --force_pic
|
||||
build:macos --force_pic
|
||||
|
||||
@@ -127,8 +127,15 @@ fi
|
||||
pushd "$BUILD_DIR"
|
||||
|
||||
|
||||
WORK_DIR=`mktemp -d`
|
||||
pushd $WORK_DIR
|
||||
need_pickle5_backport=0
|
||||
PYTHON_REVISION="$("$PYTHON_EXECUTABLE" -s -c "import sys; print('{}.{}.{}'.format(*sys.version_info))")"
|
||||
case "${PYTHON_REVISION}.0" in
|
||||
3\.[5-7].*|3.8.[0-1].*) need_pickle5_backport=1;;
|
||||
esac
|
||||
|
||||
if [ 0 -ne "${need_pickle5_backport}" ]; then
|
||||
WORK_DIR="$(mktemp -d)"
|
||||
pushd "${WORK_DIR}"
|
||||
git clone https://github.com/suquark/pickle5-backport
|
||||
pushd pickle5-backport
|
||||
git checkout 8ffe41ceba9d5e2ce8a98190f6b3d2f3325e5a72
|
||||
@@ -136,6 +143,7 @@ pushd pickle5-backport
|
||||
unzip -q -o dist/*.whl -d "$ROOT_DIR/python/ray/pickle5_files"
|
||||
popd
|
||||
popd
|
||||
fi
|
||||
|
||||
|
||||
if [ -z "$SKIP_THIRDPARTY_INSTALL" ]; then
|
||||
|
||||
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Args: [Bazel-target]
|
||||
#
|
||||
# This script cleans up any genrule() outputs in the transitive dependencies of the provided target.
|
||||
#
|
||||
# This is useful for forcing genrule actions to re-run, because the _true_ outputs of those actions
|
||||
# can include a larger set of files (e.g. files copied to the workspace) which Bazel is unable to
|
||||
# detect changes to (or delete changes of).
|
||||
#
|
||||
# Usually, you would run this script along with 'git clean -f', to make sure Bazel re-copies outputs
|
||||
# the next time a build occurs.
|
||||
|
||||
(
|
||||
set -euo pipefail
|
||||
bazel aquery --color=no --show_progress=no --output=textproto \
|
||||
"mnemonic(\"Genrule\", deps(${1-//:*}))" | awk '
|
||||
{
|
||||
body = 0;
|
||||
}
|
||||
/^^ / {
|
||||
body = 1;
|
||||
}
|
||||
/^^\S.* {$/ {
|
||||
section = $1;
|
||||
delete arr;
|
||||
}
|
||||
body {
|
||||
if (section == "artifacts") {
|
||||
p = $2;
|
||||
if ($1 == "exec_path:") {
|
||||
p = substr(p, 2, length(p) - 2); # strip quotes
|
||||
}
|
||||
arr[$1] = p;
|
||||
}
|
||||
}
|
||||
/^^}/ {
|
||||
artifacts[arr["id:"]] = arr["exec_path:"]; # save the ID -> artifact mapping
|
||||
}
|
||||
/^^ *output_ids:/ {
|
||||
print(artifacts[$2]); # print the output artifact
|
||||
}
|
||||
' | tr "\n" "\0" | xargs -0 -r -- rm -f --
|
||||
)
|
||||
+56
-12
@@ -17,6 +17,18 @@ keep_alive() {
|
||||
"${WORKSPACE_DIR}"/ci/keep_alive "$@"
|
||||
}
|
||||
|
||||
# Calls the provided command with set -x temporarily suppressed
|
||||
suppress_xtrace() {
|
||||
{
|
||||
local restore_shell_state=""
|
||||
if [ -o xtrace ]; then set +x; restore_shell_state="set -x"; fi
|
||||
} 2> /dev/null
|
||||
local status=0
|
||||
"$@" || status=$?
|
||||
${restore_shell_state}
|
||||
{ return "${status}"; } 2> /dev/null
|
||||
}
|
||||
|
||||
# If provided the names of one or more environment variables, returns 0 if any of them is triggered.
|
||||
# Usage: should_run_job [VAR_NAME]...
|
||||
should_run_job() {
|
||||
@@ -83,11 +95,14 @@ upload_wheels() {
|
||||
if [ -z "${branch}" ]; then branch="${TRAVIS_BRANCH-}"; fi
|
||||
if [ -z "${branch}" ]; then echo "Unable to detect branch name" 1>&2; return 1; fi
|
||||
local local_dir="python/dist"
|
||||
local remote_dir="${branch}/${commit}"
|
||||
if [ -d "${local_dir}" ]; then
|
||||
if command -V aws; then
|
||||
aws s3 sync --acl public-read --no-progress "${local_dir}" "s3://ray-wheels/${remote_dir}"
|
||||
fi
|
||||
ls -a -l -- "${local_dir}"
|
||||
local remote_dir
|
||||
for remote_dir in latest "${branch}/${commit}"; do
|
||||
if command -V aws; then
|
||||
aws s3 sync --acl public-read --no-progress "${local_dir}" "s3://ray-wheels/${remote_dir}"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -163,7 +178,7 @@ install_cython_examples() {
|
||||
|
||||
install_go() {
|
||||
local gimme_url="https://raw.githubusercontent.com/travis-ci/gimme/master/gimme"
|
||||
eval "$(curl -f -s -L "${gimme_url}" | GIMME_GO_VERSION=1.14.2 bash)"
|
||||
suppress_xtrace eval "$(curl -f -s -L "${gimme_url}" | GIMME_GO_VERSION=1.14.2 bash)"
|
||||
|
||||
if [ -z "${GOPATH-}" ]; then
|
||||
GOPATH="${GOPATH:-${HOME}/go_dir}"
|
||||
@@ -172,7 +187,6 @@ install_go() {
|
||||
}
|
||||
|
||||
install_ray() {
|
||||
bazel build -k "//:*" # Do a full build first to ensure everything passes
|
||||
(
|
||||
cd "${WORKSPACE_DIR}"/python
|
||||
build_dashboard_front_end
|
||||
@@ -206,13 +220,42 @@ build_wheels() {
|
||||
suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh
|
||||
;;
|
||||
msys*)
|
||||
if ! python -c "import ray" 2> /dev/null; then
|
||||
# This installs e.g. the pickle5 dependencies, which are needed for correct wheel testing
|
||||
install_ray
|
||||
fi
|
||||
(
|
||||
cd "${WORKSPACE_DIR}"/python
|
||||
python setup.py --quiet bdist_wheel
|
||||
local backup_conda="${CONDA_PREFIX}.bak" ray_uninstall_status=0
|
||||
test ! -d "${backup_conda}"
|
||||
pip uninstall -y ray || ray_uninstall_status=1
|
||||
mv -n -T -- "${CONDA_PREFIX}" "${backup_conda}" # Back up conda
|
||||
|
||||
local pyversion pyversions=()
|
||||
for pyversion in 3.6 3.7 3.8; do
|
||||
if [ "${pyversion}" = "${PYTHON-}" ]; then continue; fi # we'll build ${PYTHON} last
|
||||
pyversions+=("${pyversion}")
|
||||
done
|
||||
|
||||
pyversions+=("${PYTHON-}") # build this last so any subsequent steps use the right version
|
||||
local local_dir="python/dist"
|
||||
for pyversion in "${pyversions[@]}"; do
|
||||
if [ -z "${pyversion}" ]; then continue; fi
|
||||
"${ROOT_DIR}"/bazel-preclean.sh
|
||||
git clean -f -f -x -d -e "${local_dir}" -e python/ray/dashboard/client
|
||||
git checkout -q -f -- .
|
||||
cp -R -f -a -T -- "${backup_conda}" "${CONDA_PREFIX}"
|
||||
local existing_version
|
||||
existing_version="$(python -s -c "import sys; print('%s.%s' % sys.version_info[:2])")"
|
||||
if [ "${pyversion}" != "${existing_version}" ]; then
|
||||
suppress_xtrace conda install python="${pyversion}"
|
||||
fi
|
||||
install_ray
|
||||
(cd "${WORKSPACE_DIR}"/python && python setup.py --quiet bdist_wheel)
|
||||
pip uninstall -y ray
|
||||
rm -r -f -- "${CONDA_PREFIX}"
|
||||
done
|
||||
|
||||
mv -n -T -- "${backup_conda}" "${CONDA_PREFIX}"
|
||||
"${ROOT_DIR}"/bazel-preclean.sh
|
||||
if [ 0 -eq "${ray_uninstall_status}" ]; then # If Ray was previously installed, restore it
|
||||
install_ray
|
||||
fi
|
||||
)
|
||||
;;
|
||||
esac
|
||||
@@ -360,6 +403,7 @@ init() {
|
||||
|
||||
build() {
|
||||
if ! need_wheels; then
|
||||
bazel build -k "//:*" # Do a full build first to ensure everything passes
|
||||
install_ray
|
||||
if [ "${LINT-}" = 1 ]; then
|
||||
# Try generating Sphinx documentation. To do this, we need to install Ray first.
|
||||
|
||||
@@ -254,7 +254,7 @@ install_dependencies() {
|
||||
|
||||
# Additional streaming dependencies.
|
||||
if [ "${RAY_CI_STREAMING_PYTHON_AFFECTED}" = 1 ]; then
|
||||
pip install msgpack>=0.6.2
|
||||
pip install "msgpack>=0.6.2"
|
||||
fi
|
||||
|
||||
if [ -n "${PYTHON-}" ] || [ -n "${LINT-}" ] || [ "${MAC_WHEELS-}" = 1 ]; then
|
||||
|
||||
Reference in New Issue
Block a user