wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-27 20:22:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

Speedups for GitHub Actions (#9343)

Browse Source 
Co-authored-by: Mehrdad <noreply@github.com>
This commit is contained in:
mehrdadn
2020-07-14 14:51:51 -07:00
committed by GitHub
parent 003518619f
commit ca4f3b79db
3 changed files with 133 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/main.yml
+31 -3
View File
@@ -30,12 +30,15 @@ jobs:
python-version: 3.6
- name: ubuntu-jenkins-tune
os: ubuntu-latest
python-version: 3.6
jenkins-test: RUN_TUNE_TESTS
- name: ubuntu-jenkins-doc
os: ubuntu-latest
python-version: 3.6
jenkins-test: RUN_DOC_TESTS
- name: ubuntu-jenkins-sgd
os: ubuntu-latest
python-version: 3.6
jenkins-test: RUN_SGD_TESTS
env:
BAZEL_CONFIG: ${{ matrix.config }}
@@ -48,9 +51,9 @@ jobs:
shell: bash -e -o pipefail -l {0}
run: |
# Note: only configure settings here that are required _before_ the repository is cloned (such as Git settings).
if [ ! -f ~/.profile ] && [ ! -f ~/.bash_profile ]; then # missing on some machines, but needed for ~/.bashrc
echo 'if [ -n "${BASH_VERSION}" ] && [ -f ~/.bashrc ]; then . ~/.bashrc; fi' >> ~/.profile
fi
# Different machines behave very differently with Bash initialization -- make them more uniform
rm -f -- ~/.bash_profile ~/.profile ~/.bashrc
printf '\n%s\n' 'if [ -n "${BASH_VERSION}" ] && [ -f ~/.bashrc ]; then . ~/.bashrc; fi' >> ~/.profile
git config --global core.symlinks true && git config --global core.autocrlf false
if command -v dpkg > /dev/null; then sudo dpkg-reconfigure debconf -f noninteractive -p high; fi
- name: Checkout repository
@@ -60,11 +63,36 @@ jobs:
fetch-depth: 0
- name: Configure AWS Credentials
continue-on-error: true
if: github.repository == 'ray-project/ray' && github.event_name != 'pull_request'
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.GHA_AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.GHA_AWS_SECRET_ACCESS_KEY }}
aws-region: us-west-2
- name: Prepare caching
id: info
shell: bash
run: |
case "${OSTYPE}" in
linux*) pip_cache=~/.cache/pip;;
darwin*) pip_cache=~/Library/Caches/pip;;
msys) pip_cache="${LOCALAPPDATA}\pip\Cache";;
esac
cat <<-EOF
::set-output name=git_tree_hash::$(git cat-file -p HEAD | sed -n "s/^tree //p")
::set-output name=pip_cache::${pip_cache}
EOF
- name: Cache (pip)
uses: actions/cache@v1.1.2
continue-on-error: true
if: steps.info.outputs.pip_cache
with:
path: ${{ steps.info.outputs.pip_cache }}
key: |
pip-${{ runner.os }}-${{ steps.info.outputs.git_tree_hash }}
restore-keys: |
pip-${{ runner.os }}-
pip-
- name: Run Jenkins tests
shell: bash -e -o pipefail -l {0}
if: matrix.jenkins-test != ''
ci/travis/ci.sh
+13 -40
View File
@@ -224,11 +224,18 @@ install_go() {
fi
}
install_ray() {
(
bazel_ensure_buildable_on_windows() {
if [ "${OSTYPE}" = msys ]; then
# This performs as full of a build as possible, to ensure the repository always remains buildable on Windows.
# (Pip install will not perform a full build.)
# NOTE: Do not add build flags here. Use .bazelrc and --config instead.
bazel build -k "//:*" # Full build first, since pip install will build only a subset of targets
bazel build -k "//:*"
fi
}
install_ray() {
# TODO(mehrdadn): This function should be unified with the one in python/build-wheel-windows.sh.
(
cd "${WORKSPACE_DIR}"/python
build_dashboard_front_end
pip install -v -e .
@@ -261,43 +268,7 @@ build_wheels() {
suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh
;;
msys*)
(
local backup_conda="${CONDA_PREFIX}.bak" ray_uninstall_status=0
test ! -d "${backup_conda}"
pip uninstall -y ray || ray_uninstall_status=1
mv -n -T -- "${CONDA_PREFIX}" "${backup_conda}" # Back up conda
local pyversion pyversions=()
for pyversion in 3.6 3.7 3.8; do
if [ "${pyversion}" = "${PYTHON-}" ]; then continue; fi # we'll build ${PYTHON} last
pyversions+=("${pyversion}")
done
pyversions+=("${PYTHON-}") # build this last so any subsequent steps use the right version
local local_dir="python/dist"
for pyversion in "${pyversions[@]}"; do
if [ -z "${pyversion}" ]; then continue; fi
"${ROOT_DIR}"/bazel-preclean.sh
git clean -q -f -f -x -d -e "${local_dir}" -e python/ray/dashboard/client
git checkout -q -f -- .
cp -R -f -a -T -- "${backup_conda}" "${CONDA_PREFIX}"
local existing_version
existing_version="$(python -s -c "import sys; print('%s.%s' % sys.version_info[:2])")"
if [ "${pyversion}" != "${existing_version}" ]; then
suppress_output conda install python="${pyversion}"
fi
install_ray
(cd "${WORKSPACE_DIR}"/python && python setup.py --quiet bdist_wheel)
pip uninstall -y ray
rm -r -f -- "${CONDA_PREFIX}"
done
mv -n -T -- "${backup_conda}" "${CONDA_PREFIX}"
"${ROOT_DIR}"/bazel-preclean.sh
if [ 0 -eq "${ray_uninstall_status}" ]; then # If Ray was previously installed, restore it
install_ray
fi
)
suppress_output "${WORKSPACE_DIR}"/python/build-wheel-windows.sh
;;
esac
}
@@ -451,6 +422,8 @@ init() {
}
build() {
bazel_ensure_buildable_on_windows
if ! need_wheels; then
install_ray
if [ "${LINT-}" = 1 ]; then
python/build-wheel-windows.sh
Executable
+89
View File
@@ -0,0 +1,89 @@
#!/usr/bin/env bash
set -euxo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)"
WORKSPACE_DIR="${ROOT_DIR}/.."
PY_VERSIONS=(3.6 3.7 3.8)
PY_SCRIPT_SUBDIR=Scripts # 'bin' for UNIX, 'Scripts' for Windows
get_python_version() {
python -s -c "import sys; sys.stdout.write('%s.%s' % sys.version_info[:2])"
}
is_python_version() {
local expected result=0
expected="$1"
case "$(get_python_version).0." in
"${expected}".*) ;;
*) result=1;;
esac
case "$(pip --version | tr -d "\r")" in
*" (python ${expected})") ;;
*) result=1;;
esac
return "${result}"
}
install_ray() {
# TODO(mehrdadn): This function should be unified with the one in ci/travis/ci.sh.
(
pip install wheel
cd "${WORKSPACE_DIR}"/python
pip install -v -e .
)
}
uninstall_ray() {
pip uninstall -y ray
}
build_wheel_windows() {
local ray_uninstall_status=0
uninstall_ray || ray_uninstall_status=1
local pyversion pyversions=()
for pyversion in "${PY_VERSIONS[@]}"; do
if [ "${pyversion}" = "${PYTHON-}" ]; then continue; fi # we'll build ${PYTHON} last
pyversions+=("${pyversion}")
done
pyversions+=("${PYTHON-}") # build this last so any subsequent steps use the right version
local local_dir="python/dist"
for pyversion in "${pyversions[@]}"; do
if [ -z "${pyversion}" ]; then continue; fi
"${WORKSPACE_DIR}"/ci/travis/bazel-preclean.sh
git clean -q -f -f -x -d -e "${local_dir}" -e python/ray/dashboard/client
git checkout -q -f -- .
# Start a subshell to prevent PATH and cd from affecting our shell environment
(
if ! is_python_version "${pyversion}"; then
local pydirs=("${RUNNER_TOOL_CACHE}/Python/${pyversion}".*/x64)
local pydir="${pydirs[-1]}"
pydir="$(cygpath -u "${pydir}")" # Translate Windows path
test -d "${pydir}"
export PATH="${pydir}:${pydir}/${PY_SCRIPT_SUBDIR}:${PATH}"
fi
if ! is_python_version "${pyversion}"; then
echo "Expected pip for Python ${pyversion} but found Python $(get_python_version) with $(pip --version); exiting..." 1>&2
exit 1
fi
unset PYTHON2_BIN_PATH PYTHON3_BIN_PATH # make sure these aren't set by some chance
install_ray
cd "${WORKSPACE_DIR}"/python
python setup.py --quiet bdist_wheel
uninstall_ray
)
done
"${WORKSPACE_DIR}"/ci/travis/bazel-preclean.sh
if [ 0 -eq "${ray_uninstall_status}" ]; then # If Ray was previously installed, restore it
install_ray
fi
}
build_wheel_windows "$@"