From 4bdef78e2e9cc905259754c982e405b8c1af4e29 Mon Sep 17 00:00:00 2001 From: mehrdadn Date: Tue, 5 May 2020 10:47:49 -0700 Subject: [PATCH] Various CI fixes and cleanup (#8289) --- .bazelrc | 2 +- .github/workflows/main.yml | 7 +- .travis.yml | 74 ++++---- BUILD.bazel | 5 +- bazel/ray_deps_setup.bzl | 3 +- build.sh | 2 +- ci/README.md | 5 +- ci/travis/ci.sh | 157 ++++++++++------ ci/travis/install-bazel.sh | 7 +- ci/travis/install-cython-examples.sh | 15 -- ci/travis/install-dependencies.sh | 176 ++++++++++++------ ci/travis/install-ray.sh | 31 --- ci/travis/install-strace.sh | 22 ++- ci/travis/install-toolchains.sh | 18 +- python/ray/tests/test_metrics.py | 5 +- streaming/java/BUILD.bazel | 2 +- thirdparty/patches/grpc-command-quoting.patch | 7 - thirdparty/patches/grpc-python.patch | 23 +++ thirdparty/patches/redis-quiet.patch | 49 +++++ 19 files changed, 364 insertions(+), 246 deletions(-) delete mode 100755 ci/travis/install-cython-examples.sh delete mode 100755 ci/travis/install-ray.sh delete mode 100644 thirdparty/patches/grpc-command-quoting.patch create mode 100644 thirdparty/patches/grpc-python.patch create mode 100644 thirdparty/patches/redis-quiet.patch diff --git a/.bazelrc b/.bazelrc index 5043e18cb..526f78af5 100644 --- a/.bazelrc +++ b/.bazelrc @@ -2,7 +2,7 @@ build --enable_platform_specific_config ############################################################################### # On Windows, provide: USE_CLANG_CL=1, CC=clang, BAZEL_LLVM, BAZEL_SH -# On all platforms, provide: PYTHON2_BIN_PATH, PYTHON3_BIN_PATH +# On all platforms, provide: PYTHON3_BIN_PATH=python ############################################################################### build --action_env=PATH build:linux --compilation_mode=opt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4a6273230..4fd481144 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -2,6 +2,12 @@ name: CI on: [push, pull_request] +env: + # Git GITHUB_... variables are useful for translating Travis environment variables + GITHUB_BASE_SHA: ${{ github.event.pull_request.base.sha }} + GITHUB_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + GITHUB_PULL_REQUEST: ${{ github.event.number }} + jobs: build: name: ${{ matrix.name }} @@ -18,7 +24,6 @@ jobs: - name: macos os: macos-10.15 env: - GITHUB_PULL_REQUEST: ${{ github.event.number }} PYTHON: 3.6 TRAVIS_BUILD_DIR: ${{ github.workspace }} TRAVIS_COMMIT: ${{ github.sha }} diff --git a/.travis.yml b/.travis.yml index 3a5bfd6e4..11a4e409e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,9 @@ language: generic # Use Ubuntu 16.04 dist: xenial +before_install: + - unset -f cd # Travis defines this on Mac for RVM, but it floods the trace log and isn't relevant for us + matrix: include: - os: linux @@ -10,9 +13,9 @@ matrix: - PYTHONWARNINGS=ignore - RAY_DEFAULT_BUILD=1 - RAY_CYTHON_EXAMPLES=1 - before_install: - - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED install: + - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED + before_script: - . ./ci/travis/ci.sh build - os: osx @@ -22,9 +25,9 @@ matrix: - PYTHONWARNINGS=ignore - RAY_DEFAULT_BUILD=1 - RAY_CYTHON_EXAMPLES=1 - before_install: - - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED install: + - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED + before_script: - . ./ci/travis/ci.sh build - os: linux @@ -33,9 +36,9 @@ matrix: - PYTHON=3.6 PYTHONWARNINGS=ignore - RAY_INSTALL_JAVA=1 - RAY_GCS_ACTOR_SERVICE_ENABLED=true - before_install: - - . ./ci/travis/ci.sh init RAY_CI_JAVA_AFFECTED install: + - . ./ci/travis/ci.sh init RAY_CI_JAVA_AFFECTED + before_script: - . ./ci/travis/ci.sh build script: - ./java/test.sh @@ -47,9 +50,9 @@ matrix: - RAY_INSTALL_JAVA=1 - RAY_GCS_ACTOR_SERVICE_ENABLED=true - PYTHON=3.6 PYTHONWARNINGS=ignore - before_install: - - . ./ci/travis/ci.sh init RAY_CI_STREAMING_PYTHON_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED install: + - . ./ci/travis/ci.sh init RAY_CI_STREAMING_PYTHON_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED + before_script: - . ./ci/travis/ci.sh build script: # Streaming cpp test. @@ -65,9 +68,9 @@ matrix: - RAY_INSTALL_JAVA=1 - RAY_GCS_SERVICE_ENABLED=false - RAY_CYTHON_EXAMPLES=1 - before_install: - - . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED install: + - . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED + before_script: - . ./ci/travis/ci.sh build script: - ./ci/suppress_output bash src/ray/test/run_core_worker_tests.sh @@ -82,9 +85,9 @@ matrix: - RAY_INSTALL_JAVA=1 - RAY_GCS_SERVICE_ENABLED=false - RAY_CYTHON_EXAMPLES=1 - before_install: - - . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED install: + - . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED + before_script: - . ./ci/travis/ci.sh build script: - ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/tests/... @@ -93,12 +96,13 @@ matrix: env: - LINT=1 - PYTHONWARNINGS=ignore - before_install: - - . ./ci/travis/ci.sh init install: + - . ./ci/travis/ci.sh init + before_script: + - . ./ci/travis/ci.sh lint - . ./ci/travis/ci.sh build script: - - . ./ci/travis/ci.sh lint + - true # we still need this block to exist, otherwise it will fall back to the global one # Build Linux wheels. - os: linux @@ -107,9 +111,9 @@ matrix: - PYTHONWARNINGS=ignore - RAY_INSTALL_JAVA=1 - RAY_GCS_ACTOR_SERVICE_ENABLED=true - before_install: - - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED install: + - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED + before_script: - . ./ci/travis/ci.sh build script: - . ./ci/travis/ci.sh test_wheels @@ -123,9 +127,9 @@ matrix: - PYTHONWARNINGS=ignore - RAY_INSTALL_JAVA=1 - RAY_GCS_ACTOR_SERVICE_ENABLED=true - before_install: - - . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED install: + - . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED + before_script: - . ./ci/travis/ci.sh build script: - . ./ci/travis/ci.sh test_wheels @@ -140,9 +144,9 @@ matrix: - TORCH_VERSION=1.4 - PYTHON=3.6 - PYTHONWARNINGS=ignore - before_install: - - . ./ci/travis/ci.sh init RAY_CI_RLLIB_AFFECTED install: + - . ./ci/travis/ci.sh init RAY_CI_RLLIB_AFFECTED + before_script: - . ./ci/travis/ci.sh build script: - travis_wait 90 bazel test --config=ci --test_output=errors --build_tests_only --test_tag_filters=learning_tests_tf rllib/... @@ -158,9 +162,9 @@ matrix: - TORCH_VERSION=1.4 - PYTHON=3.6 - PYTHONWARNINGS=ignore - before_install: - - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED install: + - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED + before_script: - . ./ci/travis/ci.sh build script: - travis_wait 90 bazel test --config=ci --test_output=errors --build_tests_only --test_tag_filters=learning_tests_tf rllib/... @@ -175,9 +179,9 @@ matrix: - TORCH_VERSION=1.4 - PYTHON=3.6 - PYTHONWARNINGS=ignore - before_install: - - . ./ci/travis/ci.sh init RAY_CI_RLLIB_AFFECTED install: + - . ./ci/travis/ci.sh init RAY_CI_RLLIB_AFFECTED + before_script: - . ./ci/travis/ci.sh build script: - travis_wait 90 bazel test --config=ci --test_output=errors --build_tests_only --test_tag_filters=learning_tests_torch rllib/... @@ -193,9 +197,9 @@ matrix: - TFP_VERSION=0.8 - TORCH_VERSION=1.4 - PYTHONWARNINGS=ignore - before_install: - - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED install: + - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED + before_script: - . ./ci/travis/ci.sh build script: - travis_wait 60 bazel test --config=ci --build_tests_only --test_tag_filters=quick_train rllib/... @@ -213,9 +217,9 @@ matrix: - TFP_VERSION=0.8 - TORCH_VERSION=1.4 - PYTHONWARNINGS=ignore - before_install: - - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED install: + - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED + before_script: - . ./ci/travis/ci.sh build script: - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_A,examples_B rllib/... @@ -233,9 +237,9 @@ matrix: - TFP_VERSION=0.8 - TORCH_VERSION=1.4 - PYTHONWARNINGS=ignore - before_install: - - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED install: + - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED + before_script: - . ./ci/travis/ci.sh build script: - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I rllib/... @@ -250,9 +254,9 @@ matrix: - TFP_VERSION=0.8 - TORCH_VERSION=1.4 - PYTHONWARNINGS=ignore - before_install: - - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED install: + - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED + before_script: - . ./ci/travis/ci.sh build script: - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_J,tests_dir_K,tests_dir_L,tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z rllib/... @@ -262,9 +266,9 @@ matrix: env: - TESTSUITE=cpp_worker - PYTHON=3.6 - before_install: - - . ./ci/travis/ci.sh init install: + - . ./ci/travis/ci.sh init + before_script: - . ./ci/travis/ci.sh build script: - . ./ci/travis/ci.sh test_cpp diff --git a/BUILD.bazel b/BUILD.bazel index dd5a9b866..def4e7c31 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1526,11 +1526,12 @@ genrule( mv -f -- redis-cli.exe $(location redis-cli) """, "//conditions:default": """ - set -x && tmpdir="redis.tmp" && path=$(location @com_github_antirez_redis//:file) && cp -p -L -R -- "$${path%/*}" "$${tmpdir}" && - make -s -C "$${tmpdir}" -j"$$(getconf _NPROCESSORS_ONLN || echo 1)" V=0 && + chmod +x "$${tmpdir}"/deps/jemalloc/configure && + parallel="$$(getconf _NPROCESSORS_ONLN || echo 1)" + make -s -C "$${tmpdir}" -j"$${parallel}" V=0 CFLAGS="$${CFLAGS-} -DLUA_USE_MKSTEMP -Wno-pragmas -Wno-empty-body" && mv "$${tmpdir}"/src/redis-server $(location redis-server) && chmod +x $(location redis-server) && mv "$${tmpdir}"/src/redis-cli $(location redis-cli) && diff --git a/bazel/ray_deps_setup.bzl b/bazel/ray_deps_setup.bzl index 9e3f272f7..8e277da2b 100644 --- a/bazel/ray_deps_setup.bzl +++ b/bazel/ray_deps_setup.bzl @@ -82,6 +82,7 @@ def ray_deps_setup(): "//thirdparty/patches:hiredis-windows-sigpipe.patch", "//thirdparty/patches:hiredis-windows-sockets.patch", "//thirdparty/patches:hiredis-windows-strerror.patch", + "//thirdparty/patches:redis-quiet.patch", ], ) @@ -214,8 +215,8 @@ def ray_deps_setup(): url = "https://github.com/grpc/grpc/archive/4790ab6d97e634a1ede983be393f3bb3c132b2f7.tar.gz", sha256 = "df83bd8a08975870b8b254c34afbecc94c51a55198e6e3a5aab61d62f40b7274", patches = [ - "//thirdparty/patches:grpc-command-quoting.patch", "//thirdparty/patches:grpc-cython-copts.patch", + "//thirdparty/patches:grpc-python.patch", ], ) diff --git a/build.sh b/build.sh index 19aa893b6..d4d8433c9 100755 --- a/build.sh +++ b/build.sh @@ -133,7 +133,7 @@ git clone https://github.com/suquark/pickle5-backport pushd pickle5-backport git checkout 8ffe41ceba9d5e2ce8a98190f6b3d2f3325e5a72 CC=gcc "$PYTHON_EXECUTABLE" setup.py --quiet bdist_wheel - unzip -o dist/*.whl -d "$ROOT_DIR/python/ray/pickle5_files" + unzip -q -o dist/*.whl -d "$ROOT_DIR/python/ray/pickle5_files" popd popd diff --git a/ci/README.md b/ci/README.md index 0684e88a0..4dae833fe 100644 --- a/ci/README.md +++ b/ci/README.md @@ -8,7 +8,7 @@ _Please double-check file/function/etc. names for changes, as this document may All dependencies (e.g. `apt`, `pip`) should be installed in `install_dependencies()`, following the same pattern as those that already exist. -Once a dependency is added/removed, please ensure that if `reload_env` (or similar) is updated if it exists, as CI +Once a dependency is added/removed, please ensure that shell environment variables are persisted appropriately, as CI systems differ on when `~/.bashrc` et al. are reloaded, if at all. (And they are not necessarily idempotent.) ### Bazel, environment variables, and caching @@ -39,10 +39,11 @@ Adding new scripts has a number of pitfalls that easily take hours (even days) t The following practices can avoid such pitfalls while maintaining intuitive control flow: -- Put all environment-modifying functions in the _same_ shell script, so that their invocation behaves intuitively. +- Put all environment-modifying functions in the _top-level_ shell script, so that their invocation behaves intuitively. (The sheer length of the script is a secondary concern and can be mitigated by keeping functions modular.) - Avoid adding new scripts if possible. If it's necessary that you do so, call them instead of sourcing them. + Note that thies implies new scripts should not modify the environment, or the caller will not see such changes! - Always add code inside a function, not at global scope. Use `local` for variables where it makes sense. However, be careful and know the shell rules: for example, e.g. `local x=$(false)` succeeds even under `set -e`. diff --git a/ci/travis/ci.sh b/ci/travis/ci.sh index 17cd041d4..e75be9585 100755 --- a/ci/travis/ci.sh +++ b/ci/travis/ci.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash -{ SHELLOPTS_STACK="${SHELLOPTS_STACK-}|$(set +o); set -$-"; } 2> /dev/null # Push caller's shell options (quietly) +# Push caller's shell options (quietly) +{ SHELLOPTS_STACK="${SHELLOPTS_STACK-}|$(set +o); set -$-"; } 2> /dev/null -unset -f cd # Travis defines this on Mac for RVM, but it floods the trace log and isn't relevant for us - -set -eo pipefail && if [ -z "${TRAVIS_PULL_REQUEST-}" ] || [ -n "${OSTYPE##darwin*}" ]; then set -ux; fi # some options interfere with Travis's RVM on Mac +set -eo pipefail +if [ -z "${TRAVIS_PULL_REQUEST-}" ] || [ -n "${OSTYPE##darwin*}" ]; then set -ux; fi ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)" WORKSPACE_DIR="${ROOT_DIR}/../.." @@ -17,7 +17,7 @@ keep_alive() { "${WORKSPACE_DIR}"/ci/keep_alive "$@" } -# If provided the names of one or more environment variables, returns success if any of them is triggered. +# If provided the names of one or more environment variables, returns 0 if any of them is triggered. # Usage: should_run_job [VAR_NAME]... should_run_job() { local skip=0 @@ -25,7 +25,8 @@ should_run_job() { local envvar active_triggers=() for envvar in "$@"; do if [ "${!envvar}" = 1 ]; then - active_triggers+=("${envvar}=${!envvar}") # success! we found at least one of the given triggers is occurring + # success! we found at least one of the given triggers is occurring + active_triggers+=("${envvar}=${!envvar}") fi done if [ 0 -eq "${#active_triggers[@]}" ]; then @@ -41,8 +42,9 @@ should_run_job() { # Idempotent environment loading reload_env() { - # TODO: We should really just use a new login shell instead of doing this manually. - # Otherwise we might source a script that isn't idempotent (e.g. one that blindly prepends to PATH). + # Try to only modify CI-specific environment variables here (TRAVIS_... or GITHUB_...), + # e.g. for CI cross-compatibility. + # Normal environment variables should be set up at software installation time, not here. if [ -n "${GITHUB_PULL_REQUEST-}" ]; then case "${GITHUB_PULL_REQUEST}" in @@ -52,21 +54,15 @@ reload_env() { export TRAVIS_PULL_REQUEST fi - export PYTHON3_BIN_PATH=python - export GOROOT="${HOME}/go" GOPATH="${HOME}/go_dir" - if [ "${OSTYPE}" = msys ]; then - export USE_CLANG_CL=1 + if [ -z "${TRAVIS_BRANCH-}" ] && [ -n "${GITHUB_WORKFLOW-}" ]; then + # Define TRAVIS_BRANCH to make Travis scripts run on GitHub Actions. + TRAVIS_BRANCH="${GITHUB_BASE_REF:-${GITHUB_REF}}" # For pull requests, the base branch name + TRAVIS_BRANCH="${TRAVIS_BRANCH#refs/heads/}" # Remove refs/... prefix + # TODO(mehrdadn): Make TRAVIS_BRANCH be a named ref (e.g. 'master') like it's supposed to be. + # For now we use a hash because GitHub Actions doesn't clone refs the same way as Travis does. + TRAVIS_BRANCH="${GITHUB_HEAD_SHA:-${TRAVIS_BRANCH}}" + export TRAVIS_BRANCH fi - - # NOTE: Modifying PATH invalidates Bazel's cache! Do not add to PATH unnecessarily. - PATH="${HOME}/miniconda/bin":"${PATH}" - if [ "${OSTYPE}" = msys ]; then - PATH="${HOME}/miniconda/bin/Scripts":"${PATH}" - fi - - # Deduplicate PATH - PATH="$(set +x && printf "%s\n" "${PATH}" | tr ":" "\n" | awk '!a[$0]++' | tr "\n" ":")" - PATH="${PATH%:}" # Remove trailing colon } need_wheels() { @@ -82,9 +78,9 @@ need_wheels() { upload_wheels() { local branch="" commit commit="$(git rev-parse --verify HEAD)" - if [ -z "${branch}" ]; then branch="${TRAVIS_BRANCH-}"; fi if [ -z "${branch}" ]; then branch="${GITHUB_BASE_REF-}"; fi if [ -z "${branch}" ]; then branch="${GITHUB_REF#refs/heads/}"; fi + if [ -z "${branch}" ]; then branch="${TRAVIS_BRANCH-}"; fi if [ -z "${branch}" ]; then echo "Unable to detect branch name" 1>&2; return 1; fi local local_dir="python/dist" local remote_dir="${branch}/${commit}" @@ -122,6 +118,30 @@ test_wheels() { return "${result}" } +install_npm_project() { + if [ "${OSTYPE}" = msys ]; then + # Not Windows-compatible: https://github.com/npm/cli/issues/558#issuecomment-584673763 + { echo "WARNING: Skipping NPM due to module incompatibilities with Windows"; } 2> /dev/null + else + npm ci -q + fi +} + +build_dashboard_front_end() { + if [ "${OSTYPE}" = msys ]; then + { echo "WARNING: Skipping dashboard due to NPM incompatibilities with Windows"; } 2> /dev/null + else + ( + cd ray/dashboard/client + set +x # suppress set -x since it'll get very noisy here + . "${HOME}/.nvm/nvm.sh" + nvm use --silent node + install_npm_project + npm run -s build + ) + fi +} + build_sphinx_docs() { ( cd "${WORKSPACE_DIR}"/doc @@ -134,16 +154,30 @@ build_sphinx_docs() { } install_cython_examples() { - "${ROOT_DIR}"/install-cython-examples.sh + ( + cd "${WORKSPACE_DIR}"/doc/examples/cython + pip install scipy + python setup.py install --user + ) } install_go() { - eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=stable bash)" + local gimme_url="https://raw.githubusercontent.com/travis-ci/gimme/master/gimme" + eval "$(curl -f -s -L "${gimme_url}" | GIMME_GO_VERSION=1.14.2 bash)" + + if [ -z "${GOPATH-}" ]; then + GOPATH="${GOPATH:-${HOME}/go_dir}" + export GOPATH + fi } install_ray() { bazel build -k "//:*" # Do a full build first to ensure everything passes - "${ROOT_DIR}"/install-ray.sh + ( + cd "${WORKSPACE_DIR}"/python + build_dashboard_front_end + keep_alive pip install -e . + ) } build_wheels() { @@ -153,11 +187,19 @@ build_wheels() { # For the linux wheel build, we use a shared cache between all # wheels, but not between different travis runs, because that # caused timeouts in the past. See the "cache: false" line below. - local MOUNT_BAZEL_CACHE=(-v "${HOME}/ray-bazel-cache":/root/ray-bazel-cache -e TRAVIS=true -e TRAVIS_PULL_REQUEST="${TRAVIS_PULL_REQUEST:-false}" -e encrypted_1c30b31fe1ee_key="${encrypted_1c30b31fe1ee_key-}" -e encrypted_1c30b31fe1ee_iv="${encrypted_1c30b31fe1ee_iv-}") + local MOUNT_BAZEL_CACHE=( + -v "${HOME}/ray-bazel-cache":/root/ray-bazel-cache + -e TRAVIS=true + -e TRAVIS_PULL_REQUEST="${TRAVIS_PULL_REQUEST:-false}" + -e encrypted_1c30b31fe1ee_key="${encrypted_1c30b31fe1ee_key-}" + -e encrypted_1c30b31fe1ee_iv="${encrypted_1c30b31fe1ee_iv-}" + ) # This command should be kept in sync with ray/python/README-building-wheels.md, # except the "${MOUNT_BAZEL_CACHE[@]}" part. - suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh + suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" \ + -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" \ + rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh ;; darwin*) # This command should be kept in sync with ray/python/README-building-wheels.md. @@ -186,17 +228,15 @@ lint_readme() { lint_python() { # ignore dict vs {} (C408), others are defaults command -V python - python -m flake8 --inline-quotes '"' --no-avoid-escape --exclude=python/ray/core/generated/,streaming/python/generated,doc/source/conf.py,python/ray/cloudpickle/,python/ray/thirdparty_files --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605 + python -m flake8 --inline-quotes '"' --no-avoid-escape \ + --exclude=python/ray/core/generated/,streaming/python/generated,doc/source/conf.py,python/ray/cloudpickle/,python/ray/thirdparty_files \ + --ignore=C408,E121,E123,E126,E226,E24,E704,W503,W504,W605 "${ROOT_DIR}"/format.sh --all } lint_bazel() { # Run buildifier without affecting external environment variables ( - # TODO: Move installing Go & building buildifier to the dependency installation step? - if [ ! -d "${GOROOT}" ]; then - curl -s -L "https://dl.google.com/go/go1.14.2.linux-amd64.tar.gz" | tar -C "${GOROOT%/*}" -xz - fi mkdir -p -- "${GOPATH}" export PATH="${GOPATH}/bin":"${GOROOT}/bin":"${PATH}" @@ -214,7 +254,7 @@ lint_web() { set +x # suppress set -x since it'll get very noisy here . "${HOME}/.nvm/nvm.sh" install_npm_project - nvm use node + nvm use --silent node node_modules/.bin/eslint --max-warnings 0 $(find src -name "*.ts" -or -name "*.tsx") node_modules/.bin/prettier --check $(find src -name "*.ts" -or -name "*.tsx") node_modules/.bin/prettier --check public/index.html @@ -227,12 +267,7 @@ _lint() { linux*) platform=linux;; esac - if [ -n "${TRAVIS_BRANCH-}" ]; then - "${ROOT_DIR}"/check-git-clang-format-output.sh - else - # TODO(mehrdadn): Implement this on GitHub Actions - echo "WARNING: Not running clang-format due to TRAVIS_BRANCH not being defined." - fi + "${ROOT_DIR}"/check-git-clang-format-output.sh # Run Python linting lint_python @@ -250,6 +285,7 @@ _lint() { } lint() { + install_go # Checkout a clean copy of the repo to avoid seeing changes that have been made to the current one ( WORKSPACE_DIR="$(TMPDIR="${WORKSPACE_DIR}/.." mktemp -d)" @@ -262,23 +298,26 @@ lint() { ) } -_preload() { +_check_job_triggers() { local job_names job_names="$1" local variable_definitions variable_definitions=($(python "${ROOT_DIR}"/determine_tests_to_run.py)) if [ 0 -lt "${#variable_definitions[@]}" ]; then - local expression - expression="$(printf "%q " "${variable_definitions[@]}")" - eval "${expression}" - printf "%s\n" "${expression}" >> ~/.bashrc + local expression restore_shell_state="" + if [ -o xtrace ]; then set +x; restore_shell_state="set -x;"; fi # Disable set -x (noisy here) + { + expression="$(printf "%q " "${variable_definitions[@]}")" + printf "%s\n" "${expression}" >> ~/.bashrc + } + eval "${restore_shell_state}" "${expression}" # Restore set -x, then evaluate expression fi if ! (set +x && should_run_job ${job_names//,/ }); then if [ -n "${GITHUB_WORKFLOW-}" ]; then - # If this job is to be skipped, emit an 'exit' command into .bashrc to quickly exit all following steps. - # This isn't needed for Travis (since everything runs in a single shell), but it is needed for GitHub Actions. + # If this job is to be skipped, emit 'exit' into .bashrc to quickly exit all following steps. + # This isn't needed on Travis (since everything runs in one shell), but is on GitHub Actions. cat <> ~/.bashrc cat <&2 Exiting shell as no triggers were active for this job: @@ -301,26 +340,22 @@ configure_system() { } # Initializes the environment for the current job. Performs the following tasks: -# - Calls 'exit 0' in this job step and all subsequent steps to quickly exit if provided a list of job names and -# none of them has been triggered. +# - Calls 'exit 0' in this job step and all subsequent steps to quickly exit if provided a list of +# job names and none of them has been triggered. # - Sets variables to indicate the job names that have been triggered. # Note: Please avoid exporting these variables. Instead, source any callees that need to use them. -# This helps reduce implicit coupling of callees to their parents, as they will be unable to run when not sourced, (especially with set -u). +# This helps reduce implicit coupling of callees to their parents, as they will be unable to run +# when not sourced, (especially with set -u). # - Installs dependencies for the current job. # - Exports any environment variables necessary to run the build. # Usage: init [JOB_NAMES] # - JOB_NAMES (optional): Comma-separated list of job names to trigger on. init() { - _preload "${1-}" + _check_job_triggers "${1-}" configure_system - "${ROOT_DIR}"/install-bazel.sh - . "${ROOT_DIR}"/install-dependencies.sh - - cat <> ~/.bashrc -. "${BASH_SOURCE:-$0}" # reload environment -EOF + . "${ROOT_DIR}"/install-dependencies.sh # Script is sourced to propagate up environment changes } build() { @@ -336,7 +371,7 @@ build() { install_cython_examples fi - if [ "${RAY_DEFAULT_BUILD-}" = 1 ]; then + if [ "${RAY_DEFAULT_BUILD-}" = 1 ] || [ "${LINT-}" = 1 ]; then install_go fi @@ -347,8 +382,7 @@ build() { _main() { if [ -n "${GITHUB_WORKFLOW-}" ]; then - # Necessary for GitHub Actions (which uses separate shells for different commands) - # Unnecessary for Travis (which uses one shell for different commands) + exec 2>&1 # Merge stdout and stderr to prevent out-of-order buffering issues reload_env fi "$@" @@ -356,4 +390,5 @@ _main() { _main "$@" -{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null # Pop caller's shell options (quietly) +# Pop caller's shell options (quietly) +{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null diff --git a/ci/travis/install-bazel.sh b/ci/travis/install-bazel.sh index 54face5d4..5a044a393 100755 --- a/ci/travis/install-bazel.sh +++ b/ci/travis/install-bazel.sh @@ -45,10 +45,13 @@ esac if [ "${OSTYPE}" = "msys" ]; then target="${MINGW_DIR-/usr}/bin/bazel.exe" mkdir -p "${target%/*}" - curl -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-${platform}-${achitecture}.exe" + curl -f -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-${platform}-${achitecture}.exe" + tee /etc/profile.d/bazel.sh > /dev/null < /dev/null # system-wide install for CI diff --git a/ci/travis/install-cython-examples.sh b/ci/travis/install-cython-examples.sh deleted file mode 100755 index 3a57afe97..000000000 --- a/ci/travis/install-cython-examples.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) - -install_cython_examples() { - ( - cd "${ROOT_DIR}/../../doc/examples/cython" - pip install scipy - python setup.py install --user - ) -} - -install_cython_examples "$@" diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh index 16f473f7a..d3151b41d 100755 --- a/ci/travis/install-dependencies.sh +++ b/ci/travis/install-dependencies.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash -{ SHELLOPTS_STACK="${SHELLOPTS_STACK-}|$(set +o); set -$-"; } 2> /dev/null # Push caller's shell options (quietly) +# Push caller's shell options (quietly) +{ SHELLOPTS_STACK="${SHELLOPTS_STACK-}|$(set +o); set -$-"; } 2> /dev/null set -euxo pipefail @@ -9,17 +10,31 @@ WORKSPACE_DIR="${ROOT_DIR}/../.." pkg_install_helper() { case "${OSTYPE}" in - darwin*) brew install "$@";; - linux*) sudo apt-get install -qq -o=Dpkg::Use-Pty=0 "$@" | grep --line-buffered -v "^\(Preparing to unpack\|Unpacking\|Processing triggers for\) ";; + darwin*) + brew install "$@" + ;; + linux*) + sudo apt-get install -qq -o=Dpkg::Use-Pty=0 "$@" | { + grep --line-buffered -v "^\(Preparing to unpack\|Unpacking\|Processing triggers for\) " + } + ;; *) false;; esac } +install_bazel() { + "${ROOT_DIR}"/install-bazel.sh + if [ -f /etc/profile.d/bazel.sh ]; then + . /etc/profile.d/bazel.sh + fi +} + install_base() { case "${OSTYPE}" in linux*) sudo apt-get update -qq - pkg_install_helper build-essential curl unzip tmux gdb libunwind-dev python3-pip python3-setuptools + pkg_install_helper build-essential curl unzip libunwind-dev python3-pip python3-setuptools \ + tmux gdb if [ "${LINUX_WHEELS-}" = 1 ]; then pkg_install_helper docker if [ -n "${TRAVIS-}" ]; then @@ -34,54 +49,102 @@ install_base() { } install_miniconda() { - local miniconda_dir="${HOME}/miniconda" - if [ "${OSTYPE}" = msys ]; then - miniconda_dir="${miniconda_dir}/bin" # HACK: Compensate for python.exe being in the installation root on Windows + local conda="${CONDA_EXE-}" # Try to get the activated conda executable + + if [ -z "${conda}" ]; then # If no conda is found, try to find it in PATH + conda="$(command -v conda || true)" fi - if [ -d "${miniconda_dir}" ]; then - return 0 # already installed + + if [ ! -x "${conda}" ]; then # If no conda is found, install it + local miniconda_dir # Keep directories user-independent, to help with Bazel caching + case "${OSTYPE}" in + linux*) miniconda_dir="/opt/miniconda";; + darwin*) miniconda_dir="/usr/local/opt/miniconda";; + msys) miniconda_dir="${ALLUSERSPROFILE}\Miniconda3";; # Avoid spaces; prefer the default path + esac + + local miniconda_version="Miniconda3-py37_4.8.2" miniconda_platform="" exe_suffix=".sh" + case "${OSTYPE}" in + linux*) miniconda_platform=Linux;; + darwin*) miniconda_platform=MacOSX;; + msys*) miniconda_platform=Windows; exe_suffix=".exe";; + esac + + local miniconda_url="https://repo.continuum.io/miniconda/${miniconda_version}-${miniconda_platform}-${HOSTTYPE}${exe_suffix}" + local miniconda_target="${HOME}/${miniconda_url##*/}" + curl -f -s -L -o "${miniconda_target}" "${miniconda_url}" + chmod +x "${miniconda_target}" + + case "${OSTYPE}" in + msys*) + # We set /AddToPath=0 because + # (1) it doesn't take care of the current shell, and + # (2) it's consistent with -b in the UNIX installers. + MSYS2_ARG_CONV_EXCL="*" "${miniconda_target}" \ + /RegisterPython=0 /AddToPath=0 /InstallationType=AllUsers /S /D="${miniconda_dir}" + conda="${miniconda_dir}\Scripts\conda.exe" + ;; + *) + mkdir -p -- "${miniconda_dir}" + # We're forced to pass -b for non-interactive mode. + # Unfortunately it inhibits PATH modifications as a side effect. + "${miniconda_target}" -f -b -p "${miniconda_dir}" | grep --line-buffered -v \ + '^\(reinstalling: \|installing: \|using -f (force) option\|installation finished\.\|$\)' + conda="${miniconda_dir}/bin/conda" + ;; + esac fi - local miniconda_version="3-4.5.4" miniconda_platform="" exe_suffix=".sh" - case "${OSTYPE}" in - linux*) miniconda_platform=Linux;; - darwin*) miniconda_platform=MacOSX;; - msys*) miniconda_platform=Windows; exe_suffix=".exe";; - esac - local miniconda_url="https://repo.continuum.io/miniconda/Miniconda${miniconda_version}-${miniconda_platform}-${HOSTTYPE}${exe_suffix}" - local miniconda_target="${HOME}/${miniconda_url##*/}" - curl -s -L -o "${miniconda_target}" "${miniconda_url}" - chmod +x "${miniconda_target}" - case "${OSTYPE}" in - msys*) - MSYS2_ARG_CONV_EXCL="*" "${miniconda_target}" /S /D="$(cygpath -w -- "${miniconda_dir}")" - ;; - *) - "${miniconda_target}" -b -p "${miniconda_dir}" | grep --line-buffered -v "^\(installing: \|installation finished\.\)" - ;; - esac - reload_env - if [ -n "${PYTHON-}" ]; then - conda install -q -y python="${PYTHON}" + + if [ ! -x "${CONDA_PYTHON_EXE-}" ]; then # If conda isn't activated, activate it + local restore_shell_state="" + if [ -o xtrace ]; then set +x && restore_shell_state="set -x"; fi # Disable set -x (noisy here) + + # TODO(mehrdadn): conda activation is buggy on MSYS2; it adds C:/... to PATH, + # which gets split on a colon. Is it necessary to work around this? + eval "$("${conda}" shell."${SHELL##*/}" hook)" # Activate conda + conda init "${SHELL##*/}" # Add to future shells + + ${restore_shell_state} # Restore set -x fi - python -m pip install --upgrade --quiet pip + + local python_version + python_version="$(python -s -c "import sys; print('%s.%s' % sys.version_info[:2])")" + if [ -n "${PYTHON-}" ] && [ "${PYTHON}" != "${python_version}" ]; then # Update Python version + ( + set +x + echo "Updating Anaconda Python ${python_version} to ${PYTHON}..." + conda install -q -y python="${PYTHON}" + ) + fi + + command -V python + test -x "${CONDA_PYTHON_EXE}" # make sure conda is activated +} + +install_linters() { + pip install flake8==3.7.7 flake8-comprehensions flake8-quotes==2.0.0 yapf==0.23.0 } install_nvm() { local NVM_HOME="${HOME}/.nvm" if [ "${OSTYPE}" = msys ]; then - local version="1.1.7" + local ver="1.1.7" if [ ! -f "${NVM_HOME}/nvm.sh" ]; then mkdir -p -- "${NVM_HOME}" export NVM_SYMLINK="${PROGRAMFILES}\nodejs" ( cd "${NVM_HOME}" - local target="./nvm-${version}.zip" - curl -s -L -o "${target}" "https://github.com/coreybutler/nvm-windows/releases/download/${version}/nvm-noinstall.zip" + local target="./nvm-${ver}.zip" + curl -f -s -L -o "${target}" \ + "https://github.com/coreybutler/nvm-windows/releases/download/${ver}/nvm-noinstall.zip" unzip -q -- "${target}" rm -f -- "${target}" printf "%s\r\n" "root: $(cygpath -w -- "${NVM_HOME}")" "path: ${NVM_SYMLINK}" > settings.txt ) - printf "%s\n" "export NVM_HOME=\"$(cygpath -w -- "${NVM_HOME}")\"" 'nvm() { "${NVM_HOME}/nvm.exe" "$@"; }' > "${NVM_HOME}/nvm.sh" + printf "%s\n" \ + "export NVM_HOME=\"$(cygpath -w -- "${NVM_HOME}")\"" \ + 'nvm() { "${NVM_HOME}/nvm.exe" "$@"; }' \ + > "${NVM_HOME}/nvm.sh" fi else test -f "${NVM_HOME}/nvm.sh" # double-check NVM is already available on other platforms @@ -94,7 +157,7 @@ install_pip() { python=python3 fi - if "${python}" -m pip --version || "${python}" -m ensurepip; then # If pip is present, configure it + if "${python}" -m pip --version || "${python}" -m ensurepip; then # Configure pip if present "${python}" -m pip install --upgrade --quiet pip # If we're in a CI environment, do some configuration @@ -109,7 +172,7 @@ install_pip() { install_node() { if [ "${OSTYPE}" = msys ]; then - { echo "WARNING: Skipping running Node.js due to module incompatibilities with Windows"; } 2> /dev/null + { echo "WARNING: Skipping running Node.js due to incompatibilities with Windows"; } 2> /dev/null else # Install the latest version of Node.js in order to build the dashboard. ( @@ -122,23 +185,11 @@ install_node() { fi } -run_npm() { - npm ci -} - -install_npm_project() { - if [ "${OSTYPE}" = msys ]; then - # Not Windows-compatible: https://github.com/npm/cli/issues/558#issuecomment-584673763 - { echo "WARNING: Skipping running NPM due to module incompatibilities with Windows"; } 2> /dev/null - else - run_npm - fi -} - install_dependencies() { + install_bazel install_base - if [ -n "${GITHUB_WORKFLOW-}" ]; then # Keep Travis's built-in compilers and only use this for GitHub Actions (for now) + if [ -n "${GITHUB_WORKFLOW-}" ]; then # Not for Travis (keep built-in compilers there) "${ROOT_DIR}"/install-toolchains.sh fi install_nvm @@ -149,16 +200,18 @@ install_dependencies() { # PyTorch is installed first since we are using a "-f" directive to find the wheels. # We want to install the CPU version only. + local torch_url="https://download.pytorch.org/whl/torch_stable.html" case "${OSTYPE}" in - linux*) pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html;; + linux*) pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f "${torch_url}";; darwin*) pip install torch torchvision;; - msys*) pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html;; + msys*) pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f "${torch_url}";; esac - pip_packages=(scipy tensorflow=="${TF_VERSION:-2.0.0b1}" cython==0.29.0 gym opencv-python-headless pyyaml \ - pandas==0.24.2 requests feather-format lxml openpyxl xlrd py-spy pytest pytest-timeout networkx tabulate aiohttp \ - uvicorn dataclasses pygments werkzeug kubernetes flask grpcio pytest-sugar pytest-rerunfailures pytest-asyncio \ - scikit-learn numba Pillow) + pip_packages=(scipy tensorflow=="${TF_VERSION:-2.0.0b1}" cython==0.29.0 gym \ + opencv-python-headless pyyaml pandas==0.24.2 requests feather-format lxml openpyxl xlrd \ + py-spy pytest pytest-timeout networkx tabulate aiohttp uvicorn dataclasses pygments werkzeug \ + kubernetes flask grpcio pytest-sugar pytest-rerunfailures pytest-asyncio scikit-learn numba \ + Pillow) if [ "${OSTYPE}" != msys ]; then # These packages aren't Windows-compatible pip_packages+=(blist) # https://github.com/DanielStutzbach/blist/issues/81#issue-391460716 @@ -168,7 +221,7 @@ install_dependencies() { if [ "${LINT-}" = 1 ]; then install_miniconda - pip install flake8==3.7.7 flake8-comprehensions flake8-quotes==2.0.0 yapf==0.23.0 # Python linters + install_linters # readthedocs has an antiquated build env. # This is a best effort to reproduce it locally to avoid doc build failures and hidden errors. pip install -r "${WORKSPACE_DIR}"/doc/requirements-rtd.txt @@ -180,8 +233,8 @@ install_dependencies() { # Additional RLlib dependencies. if [ "${RLLIB_TESTING-}" = 1 ]; then - pip install tensorflow-probability=="${TFP_VERSION-0.8}" gast==0.2.2 torch=="${TORCH_VERSION-1.4}" torchvision \ - atari_py gym[atari] lz4 smart_open + pip install tensorflow-probability=="${TFP_VERSION-0.8}" gast==0.2.2 \ + torch=="${TORCH_VERSION-1.4}" torchvision atari_py gym[atari] lz4 smart_open fi # Additional streaming dependencies. @@ -198,4 +251,5 @@ install_dependencies() { install_dependencies "$@" -{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null # Pop caller's shell options (quietly) +# Pop caller's shell options (quietly) +{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null diff --git a/ci/travis/install-ray.sh b/ci/travis/install-ray.sh deleted file mode 100755 index a68354d54..000000000 --- a/ci/travis/install-ray.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -set -euxo pipefail - -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -WORKSPACE_DIR="${ROOT_DIR}/../.." - -build_dashboard_front_end() { - if [ "${OSTYPE}" = msys ]; then - { echo "WARNING: Not building dashboard front-end due to NPM package incompatibilities with Windows"; } 2> /dev/null - else - ( - cd ray/dashboard/client - set +x # suppress set -x since it'll get very noisy here - . "${HOME}/.nvm/nvm.sh" - nvm use --silent node - npm ci - npm run -s build - ) - fi -} - -install_ray() { - ( - cd "${WORKSPACE_DIR}"/python - build_dashboard_front_end - "${WORKSPACE_DIR}"/ci/keep_alive pip install -e . - ) -} - -install_ray "$@" diff --git a/ci/travis/install-strace.sh b/ci/travis/install-strace.sh index 91aca5980..af17d9f01 100755 --- a/ci/travis/install-strace.sh +++ b/ci/travis/install-strace.sh @@ -5,16 +5,20 @@ set -euxo pipefail install_strace() { case "${OSTYPE}" in linux*) - if ! strace -qq -k -e trace=exit /bin/true 1> /dev/null 2> /dev/null; then - { echo "This Linux distribution doesn't appear to support strace -k." "Attempting to build & install a recent version..."; } 2> /dev/null - git -c advice.detachedHead=false clone -q --depth=1 "https://github.com/strace/strace" -b v5.5 && ( - cd strace && - ./bootstrap > /dev/null && - CPPFLAGS="-w ${CPPFLAGS-}" ./configure --quiet --with-libunwind --enable-mpers=no && - make -s -j"$(getconf _NPROCESSORS_ONLN || echo 1)" && + if ! strace -qq -k -e trace=exit /bin/true 2> /dev/null; then + ( + set +x + echo "This Linux distribution doesn't appear to support strace -k." \ + "Attempting to build & install a recent version..." 1>&2 + git -c advice.detachedHead=false clone -q --depth=1 -b v5.5 \ + "https://github.com/strace/strace" + cd strace + ./bootstrap + CPPFLAGS="-w ${CPPFLAGS-}" ./configure --quiet --with-libunwind --enable-mpers=no + make -s -j"$(getconf _NPROCESSORS_ONLN || echo 1)" sudo make -s install - ) > /dev/null - fi;; + ) + fi > /dev/null;; *) false;; esac } diff --git a/ci/travis/install-toolchains.sh b/ci/travis/install-toolchains.sh index 9d33e7729..44b4b964e 100755 --- a/ci/travis/install-toolchains.sh +++ b/ci/travis/install-toolchains.sh @@ -15,7 +15,6 @@ install_toolchains() { osversion="${osversion}32" fi url="${urlbase}/${LLVM_VERSION}/LLVM-${LLVM_VERSION}-${osversion}.exe" - targetdir="${PROGRAMFILES}\LLVM" ;; linux-gnu) osversion="${OSTYPE}-$(sed -n -e '/^PRETTY_NAME/ { s/^[^=]*="\(.*\)"/\1/g; s/ /-/; s/\([0-9]*\.[0-9]*\)\.[0-9]*/\1/; s/ .*//; p }' /etc/os-release | tr '[:upper:]' '[:lower:]')" @@ -27,25 +26,16 @@ install_toolchains() { if [ -z "${url}" ]; then url="${urlbase}/${LLVM_VERSION}/clang+llvm-${LLVM_VERSION}-${HOSTTYPE}-${osversion}.tar.xz" fi - curl -s -L -R "${url}" | if [ "${OSTYPE}" = "msys" ]; then + curl -f -s -L -R "${url}" | if [ "${OSTYPE}" = "msys" ]; then local target="./${url##*/}" install /dev/stdin "${target}" + mkdir -p -- "${targetdir}" 7z x -bsp0 -bso0 "${target}" -o"${targetdir}" + MSYS2_ARG_CONV_EXCL="*" Reg Add "HKLM\SOFTWARE\LLVM\LLVM" /ve /t REG_SZ /f /reg:32 \ + /d "$(cygpath -w -- "${targetdir}")" > /dev/null rm -f -- "${target}" - ( - # Add Clang/LLVM binaries to somewhere that's in already PATH - # (don't change PATH itself, to avoid invalidating Bazel's cache or having to manage environment variables) - mkdir -p -- ~/bin - set +x - local path - for path in "${targetdir}\\bin"/*.exe; do - local name="${path##*/}" - printf "%s\n" "#!/usr/bin/env bash" "exec \"${path}\" \"\$@\"" | install /dev/stdin ~/bin/"${name%.*}" - done - ) else sudo tar -x -J --strip-components=1 -C "${targetdir}" - command -V clang 1>&2 fi "${targetdir}"/bin/clang --version 1>&2 } diff --git a/python/ray/tests/test_metrics.py b/python/ray/tests/test_metrics.py index fc9f4e7f3..8f2c79846 100644 --- a/python/ray/tests/test_metrics.py +++ b/python/ray/tests/test_metrics.py @@ -111,8 +111,9 @@ def test_worker_stats(shutdown_only): ] for process in processes: # TODO(ekl) why does travis/mi end up in the process list - assert ("python" in process or "ray" in process - or "travis" in process) + assert ("python" in process or "conda" in process + or "travis" in process or "runner" in process + or "ray" in process) break # Test kill_actor. diff --git a/streaming/java/BUILD.bazel b/streaming/java/BUILD.bazel index 8d66ebd91..09c8e08c3 100644 --- a/streaming/java/BUILD.bazel +++ b/streaming/java/BUILD.bazel @@ -234,7 +234,7 @@ genrule( mkdir -p "$$GENERATED_DIR" # Copy protobuf-generated files. for f in $(locations //streaming/java:all_java_proto); do - unzip -o "$$f" -x META-INF/MANIFEST.MF -d "$$WORK_DIR/streaming/java/streaming-runtime/src/main/java" + unzip -q -o "$$f" -x META-INF/MANIFEST.MF -d "$$WORK_DIR/streaming/java/streaming-runtime/src/main/java" done date > $@ """, diff --git a/thirdparty/patches/grpc-command-quoting.patch b/thirdparty/patches/grpc-command-quoting.patch deleted file mode 100644 index affb6c0e4..000000000 --- a/thirdparty/patches/grpc-command-quoting.patch +++ /dev/null @@ -1,7 +0,0 @@ -diff --git third_party/py/python_configure.bzl third_party/py/python_configure.bzl ---- third_party/py/python_configure.bzl -+++ third_party/py/python_configure.bzl -@@ -193,1 +193,1 @@ def _get_python_lib(repository_ctx, python_bin, lib_path_key): -- cmd = '%s - %s' % (python_bin, print_lib) -+ cmd = '"%s" - %s' % (python_bin, print_lib) --- diff --git a/thirdparty/patches/grpc-python.patch b/thirdparty/patches/grpc-python.patch new file mode 100644 index 000000000..b06a13bc6 --- /dev/null +++ b/thirdparty/patches/grpc-python.patch @@ -0,0 +1,23 @@ +diff --git third_party/py/python_configure.bzl third_party/py/python_configure.bzl +--- third_party/py/python_configure.bzl ++++ third_party/py/python_configure.bzl +@@ -163,1 +163,1 @@ +- bash_bin_path = repository_ctx.which("bash") ++ bash_bin_path = repository_ctx.which("bash" if not _is_windows(repository_ctx) else "sh.exe") +@@ -193,1 +193,1 @@ def _get_python_lib(repository_ctx, python_bin, lib_path_key): +- cmd = '%s - %s' % (python_bin, print_lib) ++ cmd = '"%s" - %s' % (python_bin, print_lib) +@@ -275,10 +275,10 @@ + _create_single_version_package(repository_ctx, + "_python2", + _PYTHON2_BIN_PATH, +- "python", ++ "python" if not _is_windows(repository_ctx) else "python.exe", + _PYTHON2_LIB_PATH) + _create_single_version_package(repository_ctx, + "_python3", + _PYTHON3_BIN_PATH, +- "python3", ++ "python3" if not _is_windows(repository_ctx) else "python.exe", + _PYTHON3_LIB_PATH) +-- diff --git a/thirdparty/patches/redis-quiet.patch b/thirdparty/patches/redis-quiet.patch new file mode 100644 index 000000000..51f696ab4 --- /dev/null +++ b/thirdparty/patches/redis-quiet.patch @@ -0,0 +1,49 @@ +diff --git deps/jemalloc/configure deps/jemalloc/configure +--- deps/jemalloc/configure ++++ deps/jemalloc/configure +@@ -7094 +7094 @@ +-ARFLAGS='crus' ++ARFLAGS='crs' +diff --git deps/jemalloc/Makefile.in deps/jemalloc/Makefile.in +--- deps/jemalloc/Makefile.in ++++ deps/jemalloc/Makefile.in +@@ -370 +370 @@ +- @if ! `cmp -s $< $@` ; then echo "cp $< $<"; cp $< $@ ; fi ++ @if ! `cmp -s $< $@` ; then cp $< $@ ; fi +diff --git deps/Makefile deps/Makefile +--- deps/Makefile ++++ deps/Makefile +@@ -45 +45 @@ +- @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) ++# @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) +@@ -51 +51 @@ +- @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) ++# @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) +@@ -61 +61 @@ +-LUA_CFLAGS+= -O2 -Wall -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' $(CFLAGS) ++LUA_CFLAGS+= -O2 -DLUA_ANSI -DENABLE_CJSON_GLOBAL -DREDIS_STATIC='' $(CFLAGS) +@@ -67 +67 @@ +-ARFLAGS=rcu ++ARFLAGS=rc +@@ -70 +70 @@ +- @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) ++# @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) +@@ -75 +75 @@ +-JEMALLOC_CFLAGS= -std=gnu99 -Wall -pipe -g3 -O3 -funroll-loops $(CFLAGS) ++JEMALLOC_CFLAGS= -std=gnu99 -pipe -g3 -O3 -funroll-loops $(CFLAGS) +@@ -79,2 +79,2 @@ +- @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) ++# @printf '%b %b\n' $(MAKECOLOR)MAKE$(ENDCOLOR) $(BINCOLOR)$@$(ENDCOLOR) +- cd jemalloc && ./configure --with-version=5.1.0-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ --enable-cc-silence CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" ++ cd jemalloc && ./configure --with-version=5.1.0-0-g0 --with-lg-quantum=3 --with-jemalloc-prefix=je_ --silent CFLAGS="$(JEMALLOC_CFLAGS)" LDFLAGS="$(JEMALLOC_LDFLAGS)" +diff --git src/Makefile src/Makefile +--- src/Makefile ++++ src/Makefile +@@ -176,3 +176,3 @@ +- @echo "" ++# @echo "" +- @echo "Hint: It's a good idea to run 'make test' ;)" ++# @echo "Hint: It's a good idea to run 'make test' ;)" +- @echo "" ++# @echo "" +--