wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-06-27 16:31:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

Various CI fixes and cleanup (#8289)

Browse Source
This commit is contained in:
mehrdadn
2020-05-05 10:47:49 -07:00
committed by GitHub
parent 97430b2d0f
commit 4bdef78e2e
19 changed files with 364 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.bazelrc
+1 -1
View File
@@ -2,7 +2,7 @@
build --enable_platform_specific_config
###############################################################################
# On Windows, provide: USE_CLANG_CL=1, CC=clang, BAZEL_LLVM, BAZEL_SH
# On all platforms, provide: PYTHON2_BIN_PATH, PYTHON3_BIN_PATH
# On all platforms, provide: PYTHON3_BIN_PATH=python
###############################################################################
build --action_env=PATH
build:linux --compilation_mode=opt
.github/workflows/main.yml
+6 -1
View File
@@ -2,6 +2,12 @@ name: CI
on: [push, pull_request]
env:
# Git GITHUB_... variables are useful for translating Travis environment variables
GITHUB_BASE_SHA: ${{ github.event.pull_request.base.sha }}
GITHUB_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
GITHUB_PULL_REQUEST: ${{ github.event.number }}
jobs:
build:
name: ${{ matrix.name }}
@@ -18,7 +24,6 @@ jobs:
- name: macos
os: macos-10.15
env:
GITHUB_PULL_REQUEST: ${{ github.event.number }}
PYTHON: 3.6
TRAVIS_BUILD_DIR: ${{ github.workspace }}
TRAVIS_COMMIT: ${{ github.sha }}
.travis.yml
+39 -35
View File
@@ -2,6 +2,9 @@ language: generic
# Use Ubuntu 16.04
dist: xenial
before_install:
- unset -f cd # Travis defines this on Mac for RVM, but it floods the trace log and isn't relevant for us
matrix:
include:
- os: linux
@@ -10,9 +13,9 @@ matrix:
- PYTHONWARNINGS=ignore
- RAY_DEFAULT_BUILD=1
- RAY_CYTHON_EXAMPLES=1
before_install:
- . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED
install:
- . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED
before_script:
- . ./ci/travis/ci.sh build
- os: osx
@@ -22,9 +25,9 @@ matrix:
- PYTHONWARNINGS=ignore
- RAY_DEFAULT_BUILD=1
- RAY_CYTHON_EXAMPLES=1
before_install:
- . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED
install:
- . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED
before_script:
- . ./ci/travis/ci.sh build
- os: linux
@@ -33,9 +36,9 @@ matrix:
- PYTHON=3.6 PYTHONWARNINGS=ignore
- RAY_INSTALL_JAVA=1
- RAY_GCS_ACTOR_SERVICE_ENABLED=true
before_install:
- . ./ci/travis/ci.sh init RAY_CI_JAVA_AFFECTED
install:
- . ./ci/travis/ci.sh init RAY_CI_JAVA_AFFECTED
before_script:
- . ./ci/travis/ci.sh build
script:
- ./java/test.sh
@@ -47,9 +50,9 @@ matrix:
- RAY_INSTALL_JAVA=1
- RAY_GCS_ACTOR_SERVICE_ENABLED=true
- PYTHON=3.6 PYTHONWARNINGS=ignore
before_install:
- . ./ci/travis/ci.sh init RAY_CI_STREAMING_PYTHON_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
install:
- . ./ci/travis/ci.sh init RAY_CI_STREAMING_PYTHON_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED
before_script:
- . ./ci/travis/ci.sh build
script:
# Streaming cpp test.
@@ -65,9 +68,9 @@ matrix:
- RAY_INSTALL_JAVA=1
- RAY_GCS_SERVICE_ENABLED=false
- RAY_CYTHON_EXAMPLES=1
before_install:
- . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED
install:
- . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED
before_script:
- . ./ci/travis/ci.sh build
script:
- ./ci/suppress_output bash src/ray/test/run_core_worker_tests.sh
@@ -82,9 +85,9 @@ matrix:
- RAY_INSTALL_JAVA=1
- RAY_GCS_SERVICE_ENABLED=false
- RAY_CYTHON_EXAMPLES=1
before_install:
- . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED
install:
- . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED
before_script:
- . ./ci/travis/ci.sh build
script:
- ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/tests/...
@@ -93,12 +96,13 @@ matrix:
env:
- LINT=1
- PYTHONWARNINGS=ignore
before_install:
- . ./ci/travis/ci.sh init
install:
- . ./ci/travis/ci.sh init
before_script:
- . ./ci/travis/ci.sh lint
- . ./ci/travis/ci.sh build
script:
- . ./ci/travis/ci.sh lint
- true # we still need this block to exist, otherwise it will fall back to the global one
# Build Linux wheels.
- os: linux
@@ -107,9 +111,9 @@ matrix:
- PYTHONWARNINGS=ignore
- RAY_INSTALL_JAVA=1
- RAY_GCS_ACTOR_SERVICE_ENABLED=true
before_install:
- . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED
install:
- . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED
before_script:
- . ./ci/travis/ci.sh build
script:
- . ./ci/travis/ci.sh test_wheels
@@ -123,9 +127,9 @@ matrix:
- PYTHONWARNINGS=ignore
- RAY_INSTALL_JAVA=1
- RAY_GCS_ACTOR_SERVICE_ENABLED=true
before_install:
- . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED
install:
- . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED
before_script:
- . ./ci/travis/ci.sh build
script:
- . ./ci/travis/ci.sh test_wheels
@@ -140,9 +144,9 @@ matrix:
- TORCH_VERSION=1.4
- PYTHON=3.6
- PYTHONWARNINGS=ignore
before_install:
- . ./ci/travis/ci.sh init RAY_CI_RLLIB_AFFECTED
install:
- . ./ci/travis/ci.sh init RAY_CI_RLLIB_AFFECTED
before_script:
- . ./ci/travis/ci.sh build
script:
- travis_wait 90 bazel test --config=ci --test_output=errors --build_tests_only --test_tag_filters=learning_tests_tf rllib/...
@@ -158,9 +162,9 @@ matrix:
- TORCH_VERSION=1.4
- PYTHON=3.6
- PYTHONWARNINGS=ignore
before_install:
- . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
install:
- . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
before_script:
- . ./ci/travis/ci.sh build
script:
- travis_wait 90 bazel test --config=ci --test_output=errors --build_tests_only --test_tag_filters=learning_tests_tf rllib/...
@@ -175,9 +179,9 @@ matrix:
- TORCH_VERSION=1.4
- PYTHON=3.6
- PYTHONWARNINGS=ignore
before_install:
- . ./ci/travis/ci.sh init RAY_CI_RLLIB_AFFECTED
install:
- . ./ci/travis/ci.sh init RAY_CI_RLLIB_AFFECTED
before_script:
- . ./ci/travis/ci.sh build
script:
- travis_wait 90 bazel test --config=ci --test_output=errors --build_tests_only --test_tag_filters=learning_tests_torch rllib/...
@@ -193,9 +197,9 @@ matrix:
- TFP_VERSION=0.8
- TORCH_VERSION=1.4
- PYTHONWARNINGS=ignore
before_install:
- . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
install:
- . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
before_script:
- . ./ci/travis/ci.sh build
script:
- travis_wait 60 bazel test --config=ci --build_tests_only --test_tag_filters=quick_train rllib/...
@@ -213,9 +217,9 @@ matrix:
- TFP_VERSION=0.8
- TORCH_VERSION=1.4
- PYTHONWARNINGS=ignore
before_install:
- . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
install:
- . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
before_script:
- . ./ci/travis/ci.sh build
script:
- ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_A,examples_B rllib/...
@@ -233,9 +237,9 @@ matrix:
- TFP_VERSION=0.8
- TORCH_VERSION=1.4
- PYTHONWARNINGS=ignore
before_install:
- . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
install:
- . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
before_script:
- . ./ci/travis/ci.sh build
script:
- ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I rllib/...
@@ -250,9 +254,9 @@ matrix:
- TFP_VERSION=0.8
- TORCH_VERSION=1.4
- PYTHONWARNINGS=ignore
before_install:
- . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
install:
- . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED
before_script:
- . ./ci/travis/ci.sh build
script:
- ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_J,tests_dir_K,tests_dir_L,tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z rllib/...
@@ -262,9 +266,9 @@ matrix:
env:
- TESTSUITE=cpp_worker
- PYTHON=3.6
before_install:
- . ./ci/travis/ci.sh init
install:
- . ./ci/travis/ci.sh init
before_script:
- . ./ci/travis/ci.sh build
script:
- . ./ci/travis/ci.sh test_cpp
BUILD.bazel
+3 -2
View File
@@ -1526,11 +1526,12 @@ genrule(
mv -f -- redis-cli.exe $(location redis-cli)
""",
"//conditions:default": """
set -x &&
tmpdir="redis.tmp" &&
path=$(location @com_github_antirez_redis//:file) &&
cp -p -L -R -- "$${path%/*}" "$${tmpdir}" &&
make -s -C "$${tmpdir}" -j"$$(getconf _NPROCESSORS_ONLN || echo 1)" V=0 &&
chmod +x "$${tmpdir}"/deps/jemalloc/configure &&
parallel="$$(getconf _NPROCESSORS_ONLN || echo 1)"
make -s -C "$${tmpdir}" -j"$${parallel}" V=0 CFLAGS="$${CFLAGS-} -DLUA_USE_MKSTEMP -Wno-pragmas -Wno-empty-body" &&
mv "$${tmpdir}"/src/redis-server $(location redis-server) &&
chmod +x $(location redis-server) &&
mv "$${tmpdir}"/src/redis-cli $(location redis-cli) &&
bazel/ray_deps_setup.bzl
+2 -1
View File
@@ -82,6 +82,7 @@ def ray_deps_setup():
"//thirdparty/patches:hiredis-windows-sigpipe.patch",
"//thirdparty/patches:hiredis-windows-sockets.patch",
"//thirdparty/patches:hiredis-windows-strerror.patch",
"//thirdparty/patches:redis-quiet.patch",
],
)
@@ -214,8 +215,8 @@ def ray_deps_setup():
url = "https://github.com/grpc/grpc/archive/4790ab6d97e634a1ede983be393f3bb3c132b2f7.tar.gz",
sha256 = "df83bd8a08975870b8b254c34afbecc94c51a55198e6e3a5aab61d62f40b7274",
patches = [
"//thirdparty/patches:grpc-command-quoting.patch",
"//thirdparty/patches:grpc-cython-copts.patch",
"//thirdparty/patches:grpc-python.patch",
],
)
build.sh
+1 -1
View File
@@ -133,7 +133,7 @@ git clone https://github.com/suquark/pickle5-backport
pushd pickle5-backport
git checkout 8ffe41ceba9d5e2ce8a98190f6b3d2f3325e5a72
CC=gcc "$PYTHON_EXECUTABLE" setup.py --quiet bdist_wheel
unzip -o dist/*.whl -d "$ROOT_DIR/python/ray/pickle5_files"
unzip -q -o dist/*.whl -d "$ROOT_DIR/python/ray/pickle5_files"
popd
popd
ci/README.md
+3 -2
View File
@@ -8,7 +8,7 @@ _Please double-check file/function/etc. names for changes, as this document may
All dependencies (e.g. `apt`, `pip`) should be installed in `install_dependencies()`, following the same pattern as
those that already exist.
Once a dependency is added/removed, please ensure that if `reload_env` (or similar) is updated if it exists, as CI
Once a dependency is added/removed, please ensure that shell environment variables are persisted appropriately, as CI
systems differ on when `~/.bashrc` et al. are reloaded, if at all. (And they are not necessarily idempotent.)
### Bazel, environment variables, and caching
@@ -39,10 +39,11 @@ Adding new scripts has a number of pitfalls that easily take hours (even days) t
The following practices can avoid such pitfalls while maintaining intuitive control flow:
- Put all environment-modifying functions in the _same_ shell script, so that their invocation behaves intuitively.
- Put all environment-modifying functions in the _top-level_ shell script, so that their invocation behaves intuitively.
(The sheer length of the script is a secondary concern and can be mitigated by keeping functions modular.)
- Avoid adding new scripts if possible. If it's necessary that you do so, call them instead of sourcing them.
Note that thies implies new scripts should not modify the environment, or the caller will not see such changes!
- Always add code inside a function, not at global scope. Use `local` for variables where it makes sense.
However, be careful and know the shell rules: for example, e.g. `local x=$(false)` succeeds even under `set -e`.
ci/travis/ci.sh
+96 -61
View File
@@ -1,10 +1,10 @@
#!/usr/bin/env bash
{ SHELLOPTS_STACK="${SHELLOPTS_STACK-}|$(set +o); set -$-"; } 2> /dev/null # Push caller's shell options (quietly)
# Push caller's shell options (quietly)
{ SHELLOPTS_STACK="${SHELLOPTS_STACK-}|$(set +o); set -$-"; } 2> /dev/null
unset -f cd # Travis defines this on Mac for RVM, but it floods the trace log and isn't relevant for us
set -eo pipefail && if [ -z "${TRAVIS_PULL_REQUEST-}" ] || [ -n "${OSTYPE##darwin*}" ]; then set -ux; fi # some options interfere with Travis's RVM on Mac
set -eo pipefail
if [ -z "${TRAVIS_PULL_REQUEST-}" ] || [ -n "${OSTYPE##darwin*}" ]; then set -ux; fi
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)"
WORKSPACE_DIR="${ROOT_DIR}/../.."
@@ -17,7 +17,7 @@ keep_alive() {
"${WORKSPACE_DIR}"/ci/keep_alive "$@"
}
# If provided the names of one or more environment variables, returns success if any of them is triggered.
# If provided the names of one or more environment variables, returns 0 if any of them is triggered.
# Usage: should_run_job [VAR_NAME]...
should_run_job() {
local skip=0
@@ -25,7 +25,8 @@ should_run_job() {
local envvar active_triggers=()
for envvar in "$@"; do
if [ "${!envvar}" = 1 ]; then
active_triggers+=("${envvar}=${!envvar}") # success! we found at least one of the given triggers is occurring
# success! we found at least one of the given triggers is occurring
active_triggers+=("${envvar}=${!envvar}")
fi
done
if [ 0 -eq "${#active_triggers[@]}" ]; then
@@ -41,8 +42,9 @@ should_run_job() {
# Idempotent environment loading
reload_env() {
# TODO: We should really just use a new login shell instead of doing this manually.
# Otherwise we might source a script that isn't idempotent (e.g. one that blindly prepends to PATH).
# Try to only modify CI-specific environment variables here (TRAVIS_... or GITHUB_...),
# e.g. for CI cross-compatibility.
# Normal environment variables should be set up at software installation time, not here.
if [ -n "${GITHUB_PULL_REQUEST-}" ]; then
case "${GITHUB_PULL_REQUEST}" in
@@ -52,21 +54,15 @@ reload_env() {
export TRAVIS_PULL_REQUEST
fi
export PYTHON3_BIN_PATH=python
export GOROOT="${HOME}/go" GOPATH="${HOME}/go_dir"
if [ "${OSTYPE}" = msys ]; then
export USE_CLANG_CL=1
if [ -z "${TRAVIS_BRANCH-}" ] && [ -n "${GITHUB_WORKFLOW-}" ]; then
# Define TRAVIS_BRANCH to make Travis scripts run on GitHub Actions.
TRAVIS_BRANCH="${GITHUB_BASE_REF:-${GITHUB_REF}}" # For pull requests, the base branch name
TRAVIS_BRANCH="${TRAVIS_BRANCH#refs/heads/}" # Remove refs/... prefix
# TODO(mehrdadn): Make TRAVIS_BRANCH be a named ref (e.g. 'master') like it's supposed to be.
# For now we use a hash because GitHub Actions doesn't clone refs the same way as Travis does.
TRAVIS_BRANCH="${GITHUB_HEAD_SHA:-${TRAVIS_BRANCH}}"
export TRAVIS_BRANCH
fi
# NOTE: Modifying PATH invalidates Bazel's cache! Do not add to PATH unnecessarily.
PATH="${HOME}/miniconda/bin":"${PATH}"
if [ "${OSTYPE}" = msys ]; then
PATH="${HOME}/miniconda/bin/Scripts":"${PATH}"
fi
# Deduplicate PATH
PATH="$(set +x && printf "%s\n" "${PATH}" | tr ":" "\n" | awk '!a[$0]++' | tr "\n" ":")"
PATH="${PATH%:}" # Remove trailing colon
}
need_wheels() {
@@ -82,9 +78,9 @@ need_wheels() {
upload_wheels() {
local branch="" commit
commit="$(git rev-parse --verify HEAD)"
if [ -z "${branch}" ]; then branch="${TRAVIS_BRANCH-}"; fi
if [ -z "${branch}" ]; then branch="${GITHUB_BASE_REF-}"; fi
if [ -z "${branch}" ]; then branch="${GITHUB_REF#refs/heads/}"; fi
if [ -z "${branch}" ]; then branch="${TRAVIS_BRANCH-}"; fi
if [ -z "${branch}" ]; then echo "Unable to detect branch name" 1>&2; return 1; fi
local local_dir="python/dist"
local remote_dir="${branch}/${commit}"
@@ -122,6 +118,30 @@ test_wheels() {
return "${result}"
}
install_npm_project() {
if [ "${OSTYPE}" = msys ]; then
# Not Windows-compatible: https://github.com/npm/cli/issues/558#issuecomment-584673763
{ echo "WARNING: Skipping NPM due to module incompatibilities with Windows"; } 2> /dev/null
else
npm ci -q
fi
}
build_dashboard_front_end() {
if [ "${OSTYPE}" = msys ]; then
{ echo "WARNING: Skipping dashboard due to NPM incompatibilities with Windows"; } 2> /dev/null
else
(
cd ray/dashboard/client
set +x # suppress set -x since it'll get very noisy here
. "${HOME}/.nvm/nvm.sh"
nvm use --silent node
install_npm_project
npm run -s build
)
fi
}
build_sphinx_docs() {
(
cd "${WORKSPACE_DIR}"/doc
@@ -134,16 +154,30 @@ build_sphinx_docs() {
}
install_cython_examples() {
"${ROOT_DIR}"/install-cython-examples.sh
(
cd "${WORKSPACE_DIR}"/doc/examples/cython
pip install scipy
python setup.py install --user
)
}
install_go() {
eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=stable bash)"
local gimme_url="https://raw.githubusercontent.com/travis-ci/gimme/master/gimme"
eval "$(curl -f -s -L "${gimme_url}" | GIMME_GO_VERSION=1.14.2 bash)"
if [ -z "${GOPATH-}" ]; then
GOPATH="${GOPATH:-${HOME}/go_dir}"
export GOPATH
fi
}
install_ray() {
bazel build -k "//:*" # Do a full build first to ensure everything passes
"${ROOT_DIR}"/install-ray.sh
(
cd "${WORKSPACE_DIR}"/python
build_dashboard_front_end
keep_alive pip install -e .
)
}
build_wheels() {
@@ -153,11 +187,19 @@ build_wheels() {
# For the linux wheel build, we use a shared cache between all
# wheels, but not between different travis runs, because that
# caused timeouts in the past. See the "cache: false" line below.
local MOUNT_BAZEL_CACHE=(-v "${HOME}/ray-bazel-cache":/root/ray-bazel-cache -e TRAVIS=true -e TRAVIS_PULL_REQUEST="${TRAVIS_PULL_REQUEST:-false}" -e encrypted_1c30b31fe1ee_key="${encrypted_1c30b31fe1ee_key-}" -e encrypted_1c30b31fe1ee_iv="${encrypted_1c30b31fe1ee_iv-}")
local MOUNT_BAZEL_CACHE=(
-v "${HOME}/ray-bazel-cache":/root/ray-bazel-cache
-e TRAVIS=true
-e TRAVIS_PULL_REQUEST="${TRAVIS_PULL_REQUEST:-false}"
-e encrypted_1c30b31fe1ee_key="${encrypted_1c30b31fe1ee_key-}"
-e encrypted_1c30b31fe1ee_iv="${encrypted_1c30b31fe1ee_iv-}"
)
# This command should be kept in sync with ray/python/README-building-wheels.md,
# except the "${MOUNT_BAZEL_CACHE[@]}" part.
suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh
suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" \
-e TRAVIS_COMMIT="${TRAVIS_COMMIT}" \
rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh
;;
darwin*)
# This command should be kept in sync with ray/python/README-building-wheels.md.
@@ -186,17 +228,15 @@ lint_readme() {
lint_python() {
# ignore dict vs {} (C408), others are defaults
command -V python
python -m flake8 --inline-quotes '"' --no-avoid-escape --exclude=python/ray/core/generated/,streaming/python/generated,doc/source/conf.py,python/ray/cloudpickle/,python/ray/thirdparty_files --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605
python -m flake8 --inline-quotes '"' --no-avoid-escape \
--exclude=python/ray/core/generated/,streaming/python/generated,doc/source/conf.py,python/ray/cloudpickle/,python/ray/thirdparty_files \
--ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605
"${ROOT_DIR}"/format.sh --all
}
lint_bazel() {
# Run buildifier without affecting external environment variables
(
# TODO: Move installing Go & building buildifier to the dependency installation step?
if [ ! -d "${GOROOT}" ]; then
curl -s -L "https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz" | tar -C "${GOROOT%/*}" -xz
fi
mkdir -p -- "${GOPATH}"
export PATH="${GOPATH}/bin":"${GOROOT}/bin":"${PATH}"
@@ -214,7 +254,7 @@ lint_web() {
set +x # suppress set -x since it'll get very noisy here
. "${HOME}/.nvm/nvm.sh"
install_npm_project
nvm use node
nvm use --silent node
node_modules/.bin/eslint --max-warnings 0 $(find src -name "*.ts" -or -name "*.tsx")
node_modules/.bin/prettier --check $(find src -name "*.ts" -or -name "*.tsx")
node_modules/.bin/prettier --check public/index.html
@@ -227,12 +267,7 @@ _lint() {
linux*) platform=linux;;
esac
if [ -n "${TRAVIS_BRANCH-}" ]; then
"${ROOT_DIR}"/check-git-clang-format-output.sh
else
# TODO(mehrdadn): Implement this on GitHub Actions
echo "WARNING: Not running clang-format due to TRAVIS_BRANCH not being defined."
fi
"${ROOT_DIR}"/check-git-clang-format-output.sh
# Run Python linting
lint_python
@@ -250,6 +285,7 @@ _lint() {
}
lint() {
install_go
# Checkout a clean copy of the repo to avoid seeing changes that have been made to the current one
(
WORKSPACE_DIR="$(TMPDIR="${WORKSPACE_DIR}/.." mktemp -d)"
@@ -262,23 +298,26 @@ lint() {
)
}
_preload() {
_check_job_triggers() {
local job_names
job_names="$1"
local variable_definitions
variable_definitions=($(python "${ROOT_DIR}"/determine_tests_to_run.py))
if [ 0 -lt "${#variable_definitions[@]}" ]; then
local expression
expression="$(printf "%q " "${variable_definitions[@]}")"
eval "${expression}"
printf "%s\n" "${expression}" >> ~/.bashrc
local expression restore_shell_state=""
if [ -o xtrace ]; then set +x; restore_shell_state="set -x;"; fi # Disable set -x (noisy here)
{
expression="$(printf "%q " "${variable_definitions[@]}")"
printf "%s\n" "${expression}" >> ~/.bashrc
}
eval "${restore_shell_state}" "${expression}" # Restore set -x, then evaluate expression
fi
if ! (set +x && should_run_job ${job_names//,/ }); then
if [ -n "${GITHUB_WORKFLOW-}" ]; then
# If this job is to be skipped, emit an 'exit' command into .bashrc to quickly exit all following steps.
# This isn't needed for Travis (since everything runs in a single shell), but it is needed for GitHub Actions.
# If this job is to be skipped, emit 'exit' into .bashrc to quickly exit all following steps.
# This isn't needed on Travis (since everything runs in one shell), but is on GitHub Actions.
cat <<EOF1 >> ~/.bashrc
cat <<EOF2 1>&2
Exiting shell as no triggers were active for this job:
@@ -301,26 +340,22 @@ configure_system() {
}
# Initializes the environment for the current job. Performs the following tasks:
# - Calls 'exit 0' in this job step and all subsequent steps to quickly exit if provided a list of job names and
# none of them has been triggered.
# - Calls 'exit 0' in this job step and all subsequent steps to quickly exit if provided a list of
# job names and none of them has been triggered.
# - Sets variables to indicate the job names that have been triggered.
# Note: Please avoid exporting these variables. Instead, source any callees that need to use them.
# This helps reduce implicit coupling of callees to their parents, as they will be unable to run when not sourced, (especially with set -u).
# This helps reduce implicit coupling of callees to their parents, as they will be unable to run
# when not sourced, (especially with set -u).
# - Installs dependencies for the current job.
# - Exports any environment variables necessary to run the build.
# Usage: init [JOB_NAMES]
# - JOB_NAMES (optional): Comma-separated list of job names to trigger on.
init() {
_preload "${1-}"
_check_job_triggers "${1-}"
configure_system
"${ROOT_DIR}"/install-bazel.sh
. "${ROOT_DIR}"/install-dependencies.sh
cat <<EOF >> ~/.bashrc
. "${BASH_SOURCE:-$0}" # reload environment
EOF
. "${ROOT_DIR}"/install-dependencies.sh # Script is sourced to propagate up environment changes
}
build() {
@@ -336,7 +371,7 @@ build() {
install_cython_examples
fi
if [ "${RAY_DEFAULT_BUILD-}" = 1 ]; then
if [ "${RAY_DEFAULT_BUILD-}" = 1 ] || [ "${LINT-}" = 1 ]; then
install_go
fi
@@ -347,8 +382,7 @@ build() {
_main() {
if [ -n "${GITHUB_WORKFLOW-}" ]; then
# Necessary for GitHub Actions (which uses separate shells for different commands)
# Unnecessary for Travis (which uses one shell for different commands)
exec 2>&1 # Merge stdout and stderr to prevent out-of-order buffering issues
reload_env
fi
"$@"
@@ -356,4 +390,5 @@ _main() {
_main "$@"
{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null # Pop caller's shell options (quietly)
# Pop caller's shell options (quietly)
{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null
ci/travis/install-bazel.sh
+5 -2
View File
@@ -45,10 +45,13 @@ esac
if [ "${OSTYPE}" = "msys" ]; then
target="${MINGW_DIR-/usr}/bin/bazel.exe"
mkdir -p "${target%/*}"
curl -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-${platform}-${achitecture}.exe"
curl -f -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-${platform}-${achitecture}.exe"
tee /etc/profile.d/bazel.sh > /dev/null <<EOF; . /etc/profile.d/bazel.sh
export USE_CLANG_CL=1 # Clang front-end for Visual C++
EOF
else
target="./install.sh"
curl -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-installer-${platform}-${achitecture}.sh"
curl -f -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-installer-${platform}-${achitecture}.sh"
chmod +x "${target}"
if [ "${TRAVIS-}" = true ] || [ -n "${GITHUB_WORKFLOW-}" ]; then
sudo "${target}" > /dev/null # system-wide install for CI
ci/travis/install-cython-examples.sh
-15
View File
@@ -1,15 +0,0 @@
#!/usr/bin/env bash
set -euxo pipefail
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
install_cython_examples() {
(
cd "${ROOT_DIR}/../../doc/examples/cython"
pip install scipy
python setup.py install --user
)
}
install_cython_examples "$@"
ci/travis/install-dependencies.sh
+115 -61
View File
@@ -1,6 +1,7 @@
#!/usr/bin/env bash
{ SHELLOPTS_STACK="${SHELLOPTS_STACK-}|$(set +o); set -$-"; } 2> /dev/null # Push caller's shell options (quietly)
# Push caller's shell options (quietly)
{ SHELLOPTS_STACK="${SHELLOPTS_STACK-}|$(set +o); set -$-"; } 2> /dev/null
set -euxo pipefail
@@ -9,17 +10,31 @@ WORKSPACE_DIR="${ROOT_DIR}/../.."
pkg_install_helper() {
case "${OSTYPE}" in
darwin*) brew install "$@";;
linux*) sudo apt-get install -qq -o=Dpkg::Use-Pty=0 "$@" | grep --line-buffered -v "^\(Preparing to unpack\|Unpacking\|Processing triggers for\) ";;
darwin*)
brew install "$@"
;;
linux*)
sudo apt-get install -qq -o=Dpkg::Use-Pty=0 "$@" | {
grep --line-buffered -v "^\(Preparing to unpack\|Unpacking\|Processing triggers for\) "
}
;;
*) false;;
esac
}
install_bazel() {
"${ROOT_DIR}"/install-bazel.sh
if [ -f /etc/profile.d/bazel.sh ]; then
. /etc/profile.d/bazel.sh
fi
}
install_base() {
case "${OSTYPE}" in
linux*)
sudo apt-get update -qq
pkg_install_helper build-essential curl unzip tmux gdb libunwind-dev python3-pip python3-setuptools
pkg_install_helper build-essential curl unzip libunwind-dev python3-pip python3-setuptools \
tmux gdb
if [ "${LINUX_WHEELS-}" = 1 ]; then
pkg_install_helper docker
if [ -n "${TRAVIS-}" ]; then
@@ -34,54 +49,102 @@ install_base() {
}
install_miniconda() {
local miniconda_dir="${HOME}/miniconda"
if [ "${OSTYPE}" = msys ]; then
miniconda_dir="${miniconda_dir}/bin" # HACK: Compensate for python.exe being in the installation root on Windows
local conda="${CONDA_EXE-}" # Try to get the activated conda executable
if [ -z "${conda}" ]; then # If no conda is found, try to find it in PATH
conda="$(command -v conda || true)"
fi
if [ -d "${miniconda_dir}" ]; then
return 0 # already installed
if [ ! -x "${conda}" ]; then # If no conda is found, install it
local miniconda_dir # Keep directories user-independent, to help with Bazel caching
case "${OSTYPE}" in
linux*) miniconda_dir="/opt/miniconda";;
darwin*) miniconda_dir="/usr/local/opt/miniconda";;
msys) miniconda_dir="${ALLUSERSPROFILE}\Miniconda3";; # Avoid spaces; prefer the default path
esac
local miniconda_version="Miniconda3-py37_4.8.2" miniconda_platform="" exe_suffix=".sh"
case "${OSTYPE}" in
linux*) miniconda_platform=Linux;;
darwin*) miniconda_platform=MacOSX;;
msys*) miniconda_platform=Windows; exe_suffix=".exe";;
esac
local miniconda_url="https://repo.continuum.io/miniconda/${miniconda_version}-${miniconda_platform}-${HOSTTYPE}${exe_suffix}"
local miniconda_target="${HOME}/${miniconda_url##*/}"
curl -f -s -L -o "${miniconda_target}" "${miniconda_url}"
chmod +x "${miniconda_target}"
case "${OSTYPE}" in
msys*)
# We set /AddToPath=0 because
# (1) it doesn't take care of the current shell, and
# (2) it's consistent with -b in the UNIX installers.
MSYS2_ARG_CONV_EXCL="*" "${miniconda_target}" \
/RegisterPython=0 /AddToPath=0 /InstallationType=AllUsers /S /D="${miniconda_dir}"
conda="${miniconda_dir}\Scripts\conda.exe"
;;
*)
mkdir -p -- "${miniconda_dir}"
# We're forced to pass -b for non-interactive mode.
# Unfortunately it inhibits PATH modifications as a side effect.
"${miniconda_target}" -f -b -p "${miniconda_dir}" | grep --line-buffered -v \
'^\(reinstalling: \|installing: \|using -f (force) option\|installation finished\.\|$\)'
conda="${miniconda_dir}/bin/conda"
;;
esac
fi
local miniconda_version="3-4.5.4" miniconda_platform="" exe_suffix=".sh"
case "${OSTYPE}" in
linux*) miniconda_platform=Linux;;
darwin*) miniconda_platform=MacOSX;;
msys*) miniconda_platform=Windows; exe_suffix=".exe";;
esac
local miniconda_url="https://repo.continuum.io/miniconda/Miniconda${miniconda_version}-${miniconda_platform}-${HOSTTYPE}${exe_suffix}"
local miniconda_target="${HOME}/${miniconda_url##*/}"
curl -s -L -o "${miniconda_target}" "${miniconda_url}"
chmod +x "${miniconda_target}"
case "${OSTYPE}" in
msys*)
MSYS2_ARG_CONV_EXCL="*" "${miniconda_target}" /S /D="$(cygpath -w -- "${miniconda_dir}")"
;;
*)
"${miniconda_target}" -b -p "${miniconda_dir}" | grep --line-buffered -v "^\(installing: \|installation finished\.\)"
;;
esac
reload_env
if [ -n "${PYTHON-}" ]; then
conda install -q -y python="${PYTHON}"
if [ ! -x "${CONDA_PYTHON_EXE-}" ]; then # If conda isn't activated, activate it
local restore_shell_state=""
if [ -o xtrace ]; then set +x && restore_shell_state="set -x"; fi # Disable set -x (noisy here)
# TODO(mehrdadn): conda activation is buggy on MSYS2; it adds C:/... to PATH,
# which gets split on a colon. Is it necessary to work around this?
eval "$("${conda}" shell."${SHELL##*/}" hook)" # Activate conda
conda init "${SHELL##*/}" # Add to future shells
${restore_shell_state} # Restore set -x
fi
python -m pip install --upgrade --quiet pip
local python_version
python_version="$(python -s -c "import sys; print('%s.%s' % sys.version_info[:2])")"
if [ -n "${PYTHON-}" ] && [ "${PYTHON}" != "${python_version}" ]; then # Update Python version
(
set +x
echo "Updating Anaconda Python ${python_version} to ${PYTHON}..."
conda install -q -y python="${PYTHON}"
)
fi
command -V python
test -x "${CONDA_PYTHON_EXE}" # make sure conda is activated
}
install_linters() {
pip install flake8==3.7.7 flake8-comprehensions flake8-quotes==2.0.0 yapf==0.23.0
}
install_nvm() {
local NVM_HOME="${HOME}/.nvm"
if [ "${OSTYPE}" = msys ]; then
local version="1.1.7"
local ver="1.1.7"
if [ ! -f "${NVM_HOME}/nvm.sh" ]; then
mkdir -p -- "${NVM_HOME}"
export NVM_SYMLINK="${PROGRAMFILES}\nodejs"
(
cd "${NVM_HOME}"
local target="./nvm-${version}.zip"
curl -s -L -o "${target}" "https://github.com/coreybutler/nvm-windows/releases/download/${version}/nvm-noinstall.zip"
local target="./nvm-${ver}.zip"
curl -f -s -L -o "${target}" \
"https://github.com/coreybutler/nvm-windows/releases/download/${ver}/nvm-noinstall.zip"
unzip -q -- "${target}"
rm -f -- "${target}"
printf "%s\r\n" "root: $(cygpath -w -- "${NVM_HOME}")" "path: ${NVM_SYMLINK}" > settings.txt
)
printf "%s\n" "export NVM_HOME=\"$(cygpath -w -- "${NVM_HOME}")\"" 'nvm() { "${NVM_HOME}/nvm.exe" "$@"; }' > "${NVM_HOME}/nvm.sh"
printf "%s\n" \
"export NVM_HOME=\"$(cygpath -w -- "${NVM_HOME}")\"" \
'nvm() { "${NVM_HOME}/nvm.exe" "$@"; }' \
> "${NVM_HOME}/nvm.sh"
fi
else
test -f "${NVM_HOME}/nvm.sh" # double-check NVM is already available on other platforms
@@ -94,7 +157,7 @@ install_pip() {
python=python3
fi
if "${python}" -m pip --version || "${python}" -m ensurepip; then # If pip is present, configure it
if "${python}" -m pip --version || "${python}" -m ensurepip; then # Configure pip if present
"${python}" -m pip install --upgrade --quiet pip
# If we're in a CI environment, do some configuration
@@ -109,7 +172,7 @@ install_pip() {
install_node() {
if [ "${OSTYPE}" = msys ]; then
{ echo "WARNING: Skipping running Node.js due to module incompatibilities with Windows"; } 2> /dev/null
{ echo "WARNING: Skipping running Node.js due to incompatibilities with Windows"; } 2> /dev/null
else
# Install the latest version of Node.js in order to build the dashboard.
(
@@ -122,23 +185,11 @@ install_node() {
fi
}
run_npm() {
npm ci
}
install_npm_project() {
if [ "${OSTYPE}" = msys ]; then
# Not Windows-compatible: https://github.com/npm/cli/issues/558#issuecomment-584673763
{ echo "WARNING: Skipping running NPM due to module incompatibilities with Windows"; } 2> /dev/null
else
run_npm
fi
}
install_dependencies() {
install_bazel
install_base
if [ -n "${GITHUB_WORKFLOW-}" ]; then # Keep Travis's built-in compilers and only use this for GitHub Actions (for now)
if [ -n "${GITHUB_WORKFLOW-}" ]; then # Not for Travis (keep built-in compilers there)
"${ROOT_DIR}"/install-toolchains.sh
fi
install_nvm
@@ -149,16 +200,18 @@ install_dependencies() {
# PyTorch is installed first since we are using a "-f" directive to find the wheels.
# We want to install the CPU version only.
local torch_url="https://download.pytorch.org/whl/torch_stable.html"
case "${OSTYPE}" in
linux*) pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html;;
linux*) pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f "${torch_url}";;
darwin*) pip install torch torchvision;;
msys*) pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html;;
msys*) pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f "${torch_url}";;
esac
pip_packages=(scipy tensorflow=="${TF_VERSION:-2.0.0b1}" cython==0.29.0 gym opencv-python-headless pyyaml \
pandas==0.24.2 requests feather-format lxml openpyxl xlrd py-spy pytest pytest-timeout networkx tabulate aiohttp \
uvicorn dataclasses pygments werkzeug kubernetes flask grpcio pytest-sugar pytest-rerunfailures pytest-asyncio \
scikit-learn numba Pillow)
pip_packages=(scipy tensorflow=="${TF_VERSION:-2.0.0b1}" cython==0.29.0 gym \
opencv-python-headless pyyaml pandas==0.24.2 requests feather-format lxml openpyxl xlrd \
py-spy pytest pytest-timeout networkx tabulate aiohttp uvicorn dataclasses pygments werkzeug \
kubernetes flask grpcio pytest-sugar pytest-rerunfailures pytest-asyncio scikit-learn numba \
Pillow)
if [ "${OSTYPE}" != msys ]; then
# These packages aren't Windows-compatible
pip_packages+=(blist) # https://github.com/DanielStutzbach/blist/issues/81#issue-391460716
@@ -168,7 +221,7 @@ install_dependencies() {
if [ "${LINT-}" = 1 ]; then
install_miniconda
pip install flake8==3.7.7 flake8-comprehensions flake8-quotes==2.0.0 yapf==0.23.0 # Python linters
install_linters
# readthedocs has an antiquated build env.
# This is a best effort to reproduce it locally to avoid doc build failures and hidden errors.
pip install -r "${WORKSPACE_DIR}"/doc/requirements-rtd.txt
@@ -180,8 +233,8 @@ install_dependencies() {
# Additional RLlib dependencies.
if [ "${RLLIB_TESTING-}" = 1 ]; then
pip install tensorflow-probability=="${TFP_VERSION-0.8}" gast==0.2.2 torch=="${TORCH_VERSION-1.4}" torchvision \
atari_py gym[atari] lz4 smart_open
pip install tensorflow-probability=="${TFP_VERSION-0.8}" gast==0.2.2 \
torch=="${TORCH_VERSION-1.4}" torchvision atari_py gym[atari] lz4 smart_open
fi
# Additional streaming dependencies.
@@ -198,4 +251,5 @@ install_dependencies() {
install_dependencies "$@"
{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null # Pop caller's shell options (quietly)
# Pop caller's shell options (quietly)
{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null
ci/travis/install-ray.sh
-31
View File
@@ -1,31 +0,0 @@
#!/usr/bin/env bash
set -euxo pipefail
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
WORKSPACE_DIR="${ROOT_DIR}/../.."
build_dashboard_front_end() {
if [ "${OSTYPE}" = msys ]; then
{ echo "WARNING: Not building dashboard front-end due to NPM package incompatibilities with Windows"; } 2> /dev/null
else
(
cd ray/dashboard/client
set +x # suppress set -x since it'll get very noisy here
. "${HOME}/.nvm/nvm.sh"
nvm use --silent node
npm ci
npm run -s build
)
fi
}
install_ray() {
(
cd "${WORKSPACE_DIR}"/python
build_dashboard_front_end
"${WORKSPACE_DIR}"/ci/keep_alive pip install -e .
)
}
install_ray "$@"
ci/travis/install-strace.sh
+13 -9
View File
@@ -5,16 +5,20 @@ set -euxo pipefail
install_strace() {
case "${OSTYPE}" in
linux*)
if ! strace -qq -k -e trace=exit /bin/true 1> /dev/null 2> /dev/null; then
{ echo "This Linux distribution doesn't appear to support strace -k." "Attempting to build & install a recent version..."; } 2> /dev/null
git -c advice.detachedHead=false clone -q --depth=1 "https://github.com/strace/strace" -b v5.5 && (
cd strace &&
./bootstrap > /dev/null &&
CPPFLAGS="-w ${CPPFLAGS-}" ./configure --quiet --with-libunwind --enable-mpers=no &&
make -s -j"$(getconf _NPROCESSORS_ONLN || echo 1)" &&
if ! strace -qq -k -e trace=exit /bin/true 2> /dev/null; then
(
set +x
echo "This Linux distribution doesn't appear to support strace -k." \
"Attempting to build & install a recent version..." 1>&2
git -c advice.detachedHead=false clone -q --depth=1 -b v5.5 \
"https://github.com/strace/strace"
cd strace
./bootstrap
CPPFLAGS="-w ${CPPFLAGS-}" ./configure --quiet --with-libunwind --enable-mpers=no
make -s -j"$(getconf _NPROCESSORS_ONLN || echo 1)"
sudo make -s install
) > /dev/null
fi;;
)
fi > /dev/null;;
*) false;;
esac
}
ci/travis/install-toolchains.sh
+4 -14
View File
@@ -15,7 +15,6 @@ install_toolchains() {
osversion="${osversion}32"
fi
url="${urlbase}/${LLVM_VERSION}/LLVM-${LLVM_VERSION}-${osversion}.exe"
targetdir="${PROGRAMFILES}\LLVM"
;;
linux-gnu)
osversion="${OSTYPE}-$(sed -n -e '/^PRETTY_NAME/ { s/^[^=]*="\(.*\)"/\1/g; s/ /-/; s/\([0-9]*\.[0-9]*\)\.[0-9]*/\1/; s/ .*//; p }' /etc/os-release | tr '[:upper:]' '[:lower:]')"
@@ -27,25 +26,16 @@ install_toolchains() {
if [ -z "${url}" ]; then
url="${urlbase}/${LLVM_VERSION}/clang+llvm-${LLVM_VERSION}-${HOSTTYPE}-${osversion}.tar.xz"
fi
curl -s -L -R "${url}" | if [ "${OSTYPE}" = "msys" ]; then
curl -f -s -L -R "${url}" | if [ "${OSTYPE}" = "msys" ]; then
local target="./${url##*/}"
install /dev/stdin "${target}"
mkdir -p -- "${targetdir}"
7z x -bsp0 -bso0 "${target}" -o"${targetdir}"
MSYS2_ARG_CONV_EXCL="*" Reg Add "HKLM\SOFTWARE\LLVM\LLVM" /ve /t REG_SZ /f /reg:32 \
/d "$(cygpath -w -- "${targetdir}")" > /dev/null
rm -f -- "${target}"
(
# Add Clang/LLVM binaries to somewhere that's in already PATH
# (don't change PATH itself, to avoid invalidating Bazel's cache or having to manage environment variables)
mkdir -p -- ~/bin
set +x
local path
for path in "${targetdir}\\bin"/*.exe; do
local name="${path##*/}"
printf "%s\n" "#!/usr/bin/env bash" "exec \"${path}\" \"\$@\"" | install /dev/stdin ~/bin/"${name%.*}"
done
)
else
sudo tar -x -J --strip-components=1 -C "${targetdir}"
command -V clang 1>&2
fi
"${targetdir}"/bin/clang --version 1>&2
}
python/ray/tests/test_metrics.py
+3 -2
View File
@@ -111,8 +111,9 @@ def test_worker_stats(shutdown_only):
]
for process in processes:
# TODO(ekl) why does travis/mi end up in the process list
assert ("python" in process or "ray" in process
or "travis" in process)
assert ("python" in process or "conda" in process
or "travis" in process or "runner" in process
or "ray" in process)
break
# Test kill_actor.
streaming/java/BUILD.bazel
+1 -1
View File
@@ -234,7 +234,7 @@ genrule(
mkdir -p "$$GENERATED_DIR"
# Copy protobuf-generated files.
for f in $(locations //streaming/java:all_java_proto); do
unzip -o "$$f" -x META-INF/MANIFEST.MF -d "$$WORK_DIR/streaming/java/streaming-runtime/src/main/java"
unzip -q -o "$$f" -x META-INF/MANIFEST.MF -d "$$WORK_DIR/streaming/java/streaming-runtime/src/main/java"
done
date > $@
""",
thirdparty/patches/grpc-command-quoting.patch
Vendored
-7
View File
@@ -1,7 +0,0 @@
diff --git third_party/py/python_configure.bzl third_party/py/python_configure.bzl
--- third_party/py/python_configure.bzl
+++ third_party/py/python_configure.bzl
@@ -193,1 +193,1 @@ def _get_python_lib(repository_ctx, python_bin, lib_path_key):
- cmd = '%s - %s' % (python_bin, print_lib)
+ cmd = '"%s" - %s' % (python_bin, print_lib)
--
thirdparty/patches/grpc-python.patch
Vendored
+23
View File
@@ -0,0 +1,23 @@
diff --git third_party/py/python_configure.bzl third_party/py/python_configure.bzl
--- third_party/py/python_configure.bzl
+++ third_party/py/python_configure.bzl
@@ -163,1 +163,1 @@
- bash_bin_path = repository_ctx.which("bash")
+ bash_bin_path = repository_ctx.which("bash" if not _is_windows(repository_ctx) else "sh.exe")
@@ -193,1 +193,1 @@ def _get_python_lib(repository_ctx, python_bin, lib_path_key):
- cmd = '%s - %s' % (python_bin, print_lib)
+ cmd = '"%s" - %s' % (python_bin, print_lib)
@@ -275,10 +275,10 @@
_create_single_version_package(repository_ctx,
"_python2",
_PYTHON2_BIN_PATH,
- "python",
+ "python" if not _is_windows(repository_ctx) else "python.exe",
_PYTHON2_LIB_PATH)
_create_single_version_package(repository_ctx,
"_python3",
_PYTHON3_BIN_PATH,
- "python3",
+ "python3" if not _is_windows(repository_ctx) else "python.exe",
_PYTHON3_LIB_PATH)
--
thirdparty/patches/redis-quiet.patch
Vendored
+49
View File
@@ -0,0 +1,49 @@
diff --git deps/jemalloc/configure deps/jemalloc/configure
--- deps/jemalloc/configure
+++ deps/jemalloc/configure
@@ -7094 +7094 @@
-ARFLAGS='crus'
+ARFLAGS='crs'
diff --git deps/jemalloc/Makefile.in deps/jemalloc/Makefile.in
--- deps/jemalloc/Makefile.in
+++ deps/jemalloc/Makefile.in
@@ -370 +370 @@
- @if ! `cmp -s $< $@` ; then echo "cp $< $<"; cp $< $@ ; fi
+ @if ! `cmp -s $< $@` ; then cp $< $@ ; fi
diff --git deps/Makefile deps/Makefile
--- deps/Makefile
+++ deps/Makefile
@@ -45 +45 @@
- @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
+# @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
@@ -51 +51 @@
- @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
+# @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
@@ -61 +61 @@
-LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' $(CFLAGS)
+LUA_CFLAGS+= -O2 -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' $(CFLAGS)
@@ -67 +67 @@
-ARFLAGS=rcu
+ARFLAGS=rc
@@ -70 +70 @@
- @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
+# @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
@@ -75 +75 @@
-JEMALLOC_CFLAGS= -std=gnu99 -Wall -pipe -g3 -O3 -funroll-loops $(CFLAGS)
+JEMALLOC_CFLAGS= -std=gnu99 -pipe -g3 -O3 -funroll-loops $(CFLAGS)
@@ -79,2 +79,2 @@
- @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
+# @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR)
- cd jemalloc && ./configure --with-version=5.1.0-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ --enable-cc-silence CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)"
+ cd jemalloc && ./configure --with-version=5.1.0-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ --silent CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)"
diff --git src/Makefile src/Makefile
--- src/Makefile
+++ src/Makefile
@@ -176,3 +176,3 @@
- @echo ""
+# @echo ""
- @echo "Hint: It's a good idea to run 'make test' ;)"
+# @echo "Hint: It's a good idea to run 'make test' ;)"
- @echo ""
+# @echo ""
--