From ba00c29b670283c00f96bde26bf17694cef38ff9 Mon Sep 17 00:00:00 2001 From: mehrdadn Date: Wed, 15 Apr 2020 08:10:22 -0700 Subject: [PATCH] Factor out Travis 'install' sections for use with GitHub Actions (#7988) --- .github/workflows/jenkins.yml | 2 +- .github/workflows/main.yml | 40 ++-- .travis.yml | 180 ++++++------------ BUILD.bazel | 1 - build.sh | 2 +- ci/README.md | 51 +++++ ci/travis/ci.sh | 119 ++++++++++++ ci/travis/determine_tests_to_run.py | 57 +++--- ci/travis/install-bazel.sh | 23 +-- ci/travis/install-cython-examples.sh | 33 +--- ci/travis/install-dependencies.sh | 270 ++++++++++++++++----------- ci/travis/install-ray.sh | 46 +++-- ci/travis/install-strace.sh | 38 ++-- ci/travis/install-toolchains.sh | 9 +- java/BUILD.bazel | 2 - java/test.sh | 2 +- src/shims/windows/sys/time.cc | 10 + src/shims/windows/sys/time.h | 10 + streaming/BUILD.bazel | 1 - streaming/java/BUILD.bazel | 3 - streaming/java/test.sh | 4 +- 21 files changed, 519 insertions(+), 384 deletions(-) create mode 100644 ci/README.md create mode 100755 ci/travis/ci.sh diff --git a/.github/workflows/jenkins.yml b/.github/workflows/jenkins.yml index f66019131..565289698 100644 --- a/.github/workflows/jenkins.yml +++ b/.github/workflows/jenkins.yml @@ -17,7 +17,7 @@ jobs: - name: Checkout repository uses: actions/checkout@v1 with: - fetch-depth: 1 + fetch-depth: 0 - name: Run Jenkins Tests shell: bash -e -o pipefail -l {0} env: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9741544c1..a89633f22 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -11,48 +11,38 @@ jobs: matrix: # Useful info: https://help.github.com/en/articles/workflow-syntax-for-github-actions include: - - name: ubuntu-clang + - name: ubuntu os: ubuntu-16.04 - compiler: clang - - name: windows-clang + - name: windows os: windows-2019 - compiler: clang-cl - - name: macos-clang + - name: macos os: macos-10.15 - compiler: clang + env: + PYTHON: 3.6 steps: - - name: Configure Git + - name: Pre-checkout system configuration # We add -l to process .bashrc, but need -e -o pipefail for consistency with GitHub Actions's default behavior. shell: bash -e -o pipefail -l {0} run: | + # Note: only configure settings here that are required _before_ the repository is cloned (such as Git settings). + if [ ! -f ~/.profile ] && [ ! -f ~/.bash_profile ]; then # missing on some machines, but needed for ~/.bashrc + echo 'if [ -n "${BASH_VERSION}" ] && [ -f ~/.bashrc ]; then . ~/.bashrc; fi' >> ~/.profile + fi git config --global core.symlinks true && git config --global core.autocrlf false if command -v dpkg > /dev/null; then sudo dpkg-reconfigure debconf -f noninteractive -p high; fi - name: Checkout repository uses: actions/checkout@v2 with: - fetch-depth: 1 - - name: Setup Bazel + # we need full history to diff against the original for linting etc. + fetch-depth: 0 + - name: Run CI script shell: bash -e -o pipefail -l {0} env: BAZEL_CACHE_CREDENTIAL_B64: ${{ secrets.BAZEL_CACHE_CREDENTIAL_B64 }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: ./ci/travis/install-bazel.sh - - name: Install C/C++ toolchains - shell: bash -e -o pipefail -l {0} - env: - CC: ${{ matrix.compiler }} - run: ./ci/travis/install-toolchains.sh "${CC%%-*}" - - name: Install MSYS2 (only needed if some shell commands are missing) - if: false && runner.os == 'Windows' - uses: numworks/setup-msys2@v1 - with: - update: false - - name: Perform build - shell: bash -e -o pipefail -l {0} run: | - bazel build \ - --keep_going \ - "//:*" + . ./ci/travis/ci.sh init + . ./ci/travis/ci.sh build - name: Run Clang Include-What-You-Use continue-on-error: true if: runner.os == 'Linux' diff --git a/.travis.yml b/.travis.yml index 22aa9927b..a13f17620 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,27 +8,35 @@ matrix: env: - PYTHON=3.6 - PYTHONWARNINGS=ignore + - RAY_DEFAULT_BUILD=1 + - RAY_CYTHON_EXAMPLES=1 + before_install: + - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED + install: + - . ./ci/travis/ci.sh build - os: osx osx_image: xcode7 env: - PYTHON=3.6 - PYTHONWARNINGS=ignore + - RAY_DEFAULT_BUILD=1 + - RAY_CYTHON_EXAMPLES=1 + before_install: + - . ./ci/travis/ci.sh init RAY_CI_SERVE_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_PYTHON_AFFECTED + install: + - . ./ci/travis/ci.sh build - os: linux env: - JDK='Oracle JDK 8' - PYTHON=3.6 PYTHONWARNINGS=ignore - RAY_INSTALL_JAVA=1 + before_install: + - . ./ci/travis/ci.sh init RAY_CI_JAVA_AFFECTED install: - - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - - if [ $RAY_CI_JAVA_AFFECTED != "1" ]; then exit; fi - - ./ci/travis/install-bazel.sh - - ./ci/suppress_output ./ci/travis/install-dependencies.sh - - export PATH="$HOME/miniconda/bin:$PATH" - - ./ci/suppress_output ./ci/travis/install-ray.sh + - . ./ci/travis/ci.sh build script: - - if [ $RAY_CI_JAVA_AFFECTED != "1" ]; then exit; fi - ./java/test.sh - os: linux @@ -37,14 +45,10 @@ matrix: - JDK='Oracle JDK 8' - RAY_INSTALL_JAVA=1 - PYTHON=3.6 PYTHONWARNINGS=ignore + before_install: + - . ./ci/travis/ci.sh init RAY_CI_STREAMING_PYTHON_AFFECTED,RAY_CI_STREAMING_JAVA_AFFECTED install: - - python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py - - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - - if [[ $RAY_CI_STREAMING_PYTHON_AFFECTED != "1" && $RAY_CI_STREAMING_JAVA_AFFECTED != "1" ]]; then exit; fi - - ./ci/travis/install-bazel.sh - - ./ci/suppress_output ./ci/travis/install-dependencies.sh - - export PATH="$HOME/miniconda/bin:$PATH" - - ./ci/suppress_output ./ci/travis/install-ray.sh + - . ./ci/travis/ci.sh build script: # Streaming cpp test. - if [ $RAY_CI_STREAMING_CPP_AFFECTED == "1" ]; then bazel test --config=ci --test_output=all //streaming:all && bash streaming/src/test/run_streaming_queue_test.sh; fi @@ -58,17 +62,12 @@ matrix: - PYTHON=3.6 PYTHONWARNINGS=ignore - RAY_INSTALL_JAVA=1 - RAY_GCS_SERVICE_ENABLED=false + - RAY_CYTHON_EXAMPLES=1 + before_install: + - . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED install: - - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - - if [ $RAY_CI_ONLY_RLLIB_AFFECTED == "1" ]; then exit; fi - - ./ci/travis/install-bazel.sh - - ./ci/suppress_output ./ci/travis/install-dependencies.sh - - export PATH="$HOME/miniconda/bin:$PATH" - - ./ci/suppress_output ./ci/travis/install-ray.sh - - ./ci/suppress_output ./ci/travis/install-cython-examples.sh - - eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=master bash)" + - . ./ci/travis/ci.sh build script: - - if [ $RAY_CI_ONLY_RLLIB_AFFECTED == "1" ]; then exit; fi - ./ci/suppress_output bash src/ray/test/run_core_worker_tests.sh - ./ci/suppress_output bash streaming/src/test/run_streaming_queue_test.sh - ./java/test.sh @@ -80,18 +79,12 @@ matrix: - PYTHON=3.6 PYTHONWARNINGS=ignore - RAY_INSTALL_JAVA=1 - RAY_GCS_SERVICE_ENABLED=false + - RAY_CYTHON_EXAMPLES=1 + before_install: + - . ./ci/travis/ci.sh init RAY_CI_ONLY_RLLIB_AFFECTED install: - - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - - if [ $RAY_CI_ONLY_RLLIB_AFFECTED == "1" ]; then exit; fi - - ./ci/travis/install-bazel.sh - - ./ci/suppress_output ./ci/travis/install-dependencies.sh - - export PATH="$HOME/miniconda/bin:$PATH" - - ./ci/suppress_output ./ci/travis/install-ray.sh - - ./ci/suppress_output ./ci/travis/install-cython-examples.sh - - eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=master bash)" + - . ./ci/travis/ci.sh build script: - - if [ $RAY_CI_ONLY_RLLIB_AFFECTED == "1" ]; then exit; fi - - export PATH="$HOME/miniconda/bin:$PATH" - ./ci/keep_alive bazel test --config=ci --test_tag_filters=-jenkins_only python/ray/tests/... - os: linux @@ -99,14 +92,13 @@ matrix: - LINT=1 - PYTHONWARNINGS=ignore before_install: - - sudo apt-get update -qq - install: [] + - . ./ci/travis/ci.sh init + install: + - . ./ci/travis/ci.sh build script: - ./ci/travis/check-git-clang-format-output.sh # Try generating Sphinx documentation. To do this, we need to install # Ray first. - - ./ci/travis/install-dependencies.sh - - export PATH="$HOME/miniconda/bin:$PATH" - cd doc # readthedocs has an antiquated build env. # This is a best effort to reproduce it locally to avoid doc build failures @@ -147,24 +139,11 @@ matrix: - LINUX_WHEELS=1 - PYTHONWARNINGS=ignore - RAY_INSTALL_JAVA=1 + before_install: + - . ./ci/travis/ci.sh init RAY_CI_LINUX_WHEELS_AFFECTED install: - - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - - if [ $RAY_CI_LINUX_WHEELS_AFFECTED != "1" ]; then exit; fi - - ./ci/suppress_output ./ci/travis/install-dependencies.sh - - # Mount bazel cache dir to the docker container. - # For the linux wheel build, we use a shared cache between all - # wheels, but not between different travis runs, because that - # caused timeouts in the past. See the "cache: false" line below. - - export MOUNT_BAZEL_CACHE="-v $HOME/ray-bazel-cache:/root/ray-bazel-cache -e TRAVIS=true -e TRAVIS_PULL_REQUEST=$TRAVIS_PULL_REQUEST -e encrypted_1c30b31fe1ee_key=$encrypted_1c30b31fe1ee_key -e encrypted_1c30b31fe1ee_iv=$encrypted_1c30b31fe1ee_iv" - - # This command should be kept in sync with ray/python/README-building-wheels.md, - # except the `$MOUNT_BAZEL_CACHE` part. - - - ./ci/suppress_output docker run --rm -w /ray -v `pwd`:/ray $MOUNT_BAZEL_CACHE -e TRAVIS_COMMIT=$TRAVIS_COMMIT -ti rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh - + - . ./ci/travis/ci.sh build script: - - if [ $RAY_CI_LINUX_WHEELS_AFFECTED != "1" ]; then exit; fi # Explicitly sleep 60 seconds for logs to go through - ./ci/travis/test-wheels.sh || { cat /tmp/ray/session_latest/logs/* && sleep 60 && false; } cache: false @@ -176,16 +155,11 @@ matrix: - MAC_WHEELS=1 - PYTHONWARNINGS=ignore - RAY_INSTALL_JAVA=1 + before_install: + - . ./ci/travis/ci.sh init RAY_CI_MACOS_WHEELS_AFFECTED install: - - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - - if [ $RAY_CI_MACOS_WHEELS_AFFECTED != "1" ]; then exit; fi - - - ./ci/suppress_output ./ci/travis/install-dependencies.sh - # This command should be kept in sync with ray/python/README-building-wheels.md. - - ./ci/suppress_output ./python/build-wheel-macos.sh + - . ./ci/travis/ci.sh build script: - - if [ $RAY_CI_MACOS_WHEELS_AFFECTED != "1" ]; then exit; fi - # Explicitly sleep 60 seconds for logs to go through - ./ci/travis/test-wheels.sh || { cat /tmp/ray/session_latest/logs/* && sleep 60 && false; } @@ -199,15 +173,11 @@ matrix: - TORCH_VERSION=1.4 - PYTHON=3.6 - PYTHONWARNINGS=ignore + before_install: + - . ./ci/travis/ci.sh init RAY_CI_RLLIB_AFFECTED install: - - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - - if [ $RAY_CI_RLLIB_AFFECTED != "1" ]; then exit; fi - - ./ci/travis/install-bazel.sh - - ./ci/travis/install-dependencies.sh - - export PATH="$HOME/miniconda/bin:$PATH" - - ./ci/suppress_output ./ci/travis/install-ray.sh + - . ./ci/travis/ci.sh build script: - - if [ $RAY_CI_RLLIB_AFFECTED != "1" ]; then exit; fi - travis_wait 90 bazel test --config=ci --test_output=streamed --build_tests_only --test_tag_filters=learning_tests rllib/... # RLlib: Learning tests with tf=1.x (from rllib/tuned_examples/regression_tests/*.yaml). @@ -221,15 +191,11 @@ matrix: - TORCH_VERSION=1.4 - PYTHON=3.6 - PYTHONWARNINGS=ignore + before_install: + - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED install: - - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi - - ./ci/travis/install-bazel.sh - - ./ci/travis/install-dependencies.sh - - export PATH="$HOME/miniconda/bin:$PATH" - - ./ci/suppress_output ./ci/travis/install-ray.sh + - . ./ci/travis/ci.sh build script: - - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi - travis_wait 90 bazel test --config=ci --test_output=streamed --build_tests_only --test_tag_filters=learning_tests rllib/... # RLlib: Quick Agent train.py runs (compilation & running, no(!) learning). @@ -243,15 +209,11 @@ matrix: - TFP_VERSION=0.8 - TORCH_VERSION=1.4 - PYTHONWARNINGS=ignore + before_install: + - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED install: - - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi - - ./ci/travis/install-bazel.sh - - ./ci/travis/install-dependencies.sh - - export PATH="$HOME/miniconda/bin:$PATH" - - ./ci/suppress_output ./ci/travis/install-ray.sh + - . ./ci/travis/ci.sh build script: - - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi - travis_wait 60 bazel test --config=ci --build_tests_only --test_tag_filters=quick_train rllib/... # Test everything that does not have any of the "main" labels: # "learning_tests|quick_train|examples|tests_dir". @@ -267,15 +229,11 @@ matrix: - TFP_VERSION=0.8 - TORCH_VERSION=1.4 - PYTHONWARNINGS=ignore + before_install: + - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED install: - - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi - - ./ci/travis/install-bazel.sh - - ./ci/travis/install-dependencies.sh - - export PATH="$HOME/miniconda/bin:$PATH" - - ./ci/suppress_output ./ci/travis/install-ray.sh + - . ./ci/travis/ci.sh build script: - - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_A,examples_B rllib/... - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_C rllib/... - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=examples_E,examples_L,examples_M,examples_P rllib/... @@ -291,15 +249,11 @@ matrix: - TFP_VERSION=0.8 - TORCH_VERSION=1.4 - PYTHONWARNINGS=ignore + before_install: + - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED install: - - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi - - ./ci/travis/install-bazel.sh - - ./ci/travis/install-dependencies.sh - - export PATH="$HOME/miniconda/bin:$PATH" - - ./ci/suppress_output ./ci/travis/install-ray.sh + - . ./ci/travis/ci.sh build script: - - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_A,tests_dir_B,tests_dir_C,tests_dir_D,tests_dir_E,tests_dir_F,tests_dir_G,tests_dir_H,tests_dir_I rllib/... # RLlib: tests_dir: Everything in rllib/tests/ directory (J-Z). @@ -312,15 +266,11 @@ matrix: - TFP_VERSION=0.8 - TORCH_VERSION=1.4 - PYTHONWARNINGS=ignore + before_install: + - . ./ci/travis/ci.sh init RAY_CI_RLLIB_FULL_AFFECTED install: - - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi - - ./ci/travis/install-bazel.sh - - ./ci/travis/install-dependencies.sh - - export PATH="$HOME/miniconda/bin:$PATH" - - ./ci/suppress_output ./ci/travis/install-ray.sh + - . ./ci/travis/ci.sh build script: - - if [ $RAY_CI_RLLIB_FULL_AFFECTED != "1" ]; then exit; fi - ./ci/keep_alive bazel test --config=ci --build_tests_only --test_tag_filters=tests_dir_J,tests_dir_K,tests_dir_L,tests_dir_M,tests_dir_N,tests_dir_O,tests_dir_P,tests_dir_Q,tests_dir_R,tests_dir_S,tests_dir_T,tests_dir_U,tests_dir_V,tests_dir_W,tests_dir_X,tests_dir_Y,tests_dir_Z rllib/... # Cpp worker test @@ -328,32 +278,14 @@ matrix: env: - TESTSUITE=cpp_worker - PYTHON=3.6 + before_install: + - . ./ci/travis/ci.sh init install: - - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - - ./ci/travis/install-bazel.sh - - ./ci/suppress_output ./ci/travis/install-dependencies.sh - - export PATH="$HOME/miniconda/bin:$PATH" - - ./ci/suppress_output ./ci/travis/install-ray.sh + - . ./ci/travis/ci.sh build script: - bazel test --config=ci //cpp:all --build_tests_only --test_output=streamed -install: - - eval `python $TRAVIS_BUILD_DIR/ci/travis/determine_tests_to_run.py` - - if [ $RAY_CI_SERVE_AFFECTED != "1" ] && [ $RAY_CI_TUNE_AFFECTED != "1" ] && [ $RAY_CI_PYTHON_AFFECTED != "1" ]; then exit; fi - - - ./ci/travis/install-bazel.sh - - ./ci/suppress_output ./ci/travis/install-dependencies.sh - - export PATH="$HOME/miniconda/bin:$PATH" - - ./ci/suppress_output ./ci/travis/install-ray.sh - - ./ci/suppress_output ./ci/travis/install-cython-examples.sh - - - eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=master bash)" - script: - - if [ $RAY_CI_SERVE_AFFECTED != "1" ] && [ $RAY_CI_TUNE_AFFECTED != "1" ] && [ $RAY_CI_PYTHON_AFFECTED != "1" ]; then exit; fi - - - export PATH="$HOME/miniconda/bin:$PATH" - # raylet integration tests - ./ci/suppress_output bash src/ray/test/run_core_worker_tests.sh - ./ci/suppress_output bash src/ray/test/run_object_manager_tests.sh diff --git a/BUILD.bazel b/BUILD.bazel index ef5261562..2e64339bd 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1520,7 +1520,6 @@ genrule( ], outs = ["ray_pkg.out"], cmd = """ - set -x && WORK_DIR="$$(pwd)" && RAYDIR="$$WORK_DIR/python/ray" RAYLET="$(location python/ray/_raylet.so)" diff --git a/build.sh b/build.sh index 88f078785..fbe85bbf3 100755 --- a/build.sh +++ b/build.sh @@ -123,7 +123,7 @@ pushd $WORK_DIR git clone https://github.com/suquark/pickle5-backport pushd pickle5-backport git checkout 8ffe41ceba9d5e2ce8a98190f6b3d2f3325e5a72 - "$PYTHON_EXECUTABLE" setup.py bdist_wheel + CC=gcc "$PYTHON_EXECUTABLE" setup.py bdist_wheel unzip -o dist/*.whl -d "$ROOT_DIR/python/ray/pickle5_files" popd popd diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 000000000..0684e88a0 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,51 @@ +# CI process + +_This document is a work-in-progress._ +_Please double-check file/function/etc. names for changes, as this document may be out of sync._ + +### Dependencies + +All dependencies (e.g. `apt`, `pip`) should be installed in `install_dependencies()`, following the same pattern as +those that already exist. + +Once a dependency is added/removed, please ensure that if `reload_env` (or similar) is updated if it exists, as CI +systems differ on when `~/.bashrc` et al. are reloaded, if at all. (And they are not necessarily idempotent.) + +### Bazel, environment variables, and caching + +Any environment variables passed to Bazel actions (e.g. `PATH`) should be idempotent to hit the Bazel cache. + +If a different `PATH` gets passed to a Bazel action, Bazel will not hit the cache, and you might trigger a full rebuild +when you really expect an incremental (or no-op) build for an option (say `pip install -e .` after `bazel build //...`). + +### Invocation + +The CI system (such as Travis) must _source_ (_not_ execute) `ci/travis/ci.sh` and pass the action(s) to execute. +The script either handles the work or dispatches it to other script(s) as it deems appropriate. +This helps ensure any environment setup/teardown is handled appropriately. + +### Development best practices & pitfalls (read before adding a new script) + +Before adding new scripts, please read this section. + +First, please consider modifying an existing script instead (e.g. add your code as a separate function). +Adding new scripts has a number of pitfalls that easily take hours (even days) to track down and fix: + +- When _calling_ other scripts (as executables), environment variables (like `PATH`) _cannot_ propagate back up to the + caller. Often, the caller expects such variables to be updated. + +- When _sourcing_ other scripts, global state (`ROOT_DIR`, `main`, `set -e`, etc.) may be overwritten silently, causing + unexpected behavior. + +The following practices can avoid such pitfalls while maintaining intuitive control flow: + +- Put all environment-modifying functions in the _same_ shell script, so that their invocation behaves intuitively. + (The sheer length of the script is a secondary concern and can be mitigated by keeping functions modular.) + +- Avoid adding new scripts if possible. If it's necessary that you do so, call them instead of sourcing them. + +- Always add code inside a function, not at global scope. Use `local` for variables where it makes sense. + However, be careful and know the shell rules: for example, e.g. `local x=$(false)` succeeds even under `set -e`. + +Ultimately, it's best to _only_ add new scripts if they might need to be executed directly by _non-CI_ code, +as in that case, they should probably not use CI entrypoints (which assume exclusive control over the machine). diff --git a/ci/travis/ci.sh b/ci/travis/ci.sh new file mode 100755 index 000000000..85a1646f9 --- /dev/null +++ b/ci/travis/ci.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash + +{ SHELLOPTS_STACK="${SHELLOPTS_STACK-}|$(set +o); set -$-"; } 2> /dev/null # Push caller's shell options (quietly) + +unset -f cd # Travis defines this on Mac for RVM, but it floods the trace log and isn't relevant for us + +set -eo pipefail && if [ -n "${OSTYPE##darwin*}" ]; then set -ux; fi # some options interfere with Travis's RVM on Mac + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)" +WORKSPACE_DIR="${ROOT_DIR}/../.." + +# If provided the names of one or more environment variables, returns success if any of them is triggered. +# Usage: should_run_job [VAR_NAME]... +should_run_job() { + local skip=0 + if [ -n "${1-}" ]; then # were any triggers provided? (if not, then the job will always run) + local envvar active_triggers=() + for envvar in "$@"; do + if [ "${!envvar}" = 1 ]; then + active_triggers+="${envvar}=${!envvar}" # success! we found at least one of the given triggers is occurring + fi + done + if [ 0 -eq "${#active_triggers[@]}" ]; then + echo "Job is not triggered by any of $1; skipping job." + skip=1 + else + echo "Job is triggered by: ${#active_triggers[*]}" + fi + fi + return "${skip}" +} + +reload_env() { + # TODO: We should really just use a new login shell instead of doing this manually. + # Otherwise we might source a script that isn't idempotent (e.g. one that blindly prepends to PATH). + { local set_x="${-//[^x]/}"; } 2> /dev/null # save set -x to suppress noise + set +x + local to_add="$HOME/miniconda/bin" old_path=":${PATH}:" + if [ "${old_path}" = "${old_path##*:${to_add}:*}" ]; then + PATH="${to_add}:$PATH" + fi + test -z "${set_x}" || set -x # restore set -x + if [ "${OSTYPE}" = msys ]; then + PATH="${PATH// :/:}" # HACK: Work around https://github.com/actions/virtual-environments/issues/635#issue-589318442 + fi + PYTHON3_BIN_PATH=python + export PATH PYTHON3_BIN_PATH +} + +# Initializes the environment for the current job. Performs the following tasks: +# - Calls 'exit 0' to quickly exit if provided a list of job names and none of them has been triggered. +# - Sets variables to indicate the job names that have been triggered. +# Note: Please avoid exporting these variables. Instead, source any callees that need to use them. +# This helps reduce implicit coupling of callees to their parents, as they will be unable to run when not sourced, (especially with set -u). +# - Installs dependencies for the current job. +# - Exports any environment variables necessary to run the build. +# Usage: init [JOB_NAMES] +# - JOB_NAMES (optional): Comma-separated list of job names to trigger on. +init() { + local job_names="${1-}" + + local variable_definitions + variable_definitions=($(python "${ROOT_DIR}"/determine_tests_to_run.py)) + { declare "${variable_definitions[@]}"; } > /dev/null 2> /dev/null + + if ! (set +x && should_run_job ${job_names//,/ }); then + exit 0 + fi + + if [ "${OSTYPE}" = msys ]; then + export USE_CLANG_CL=1 + fi + + local wheels="${LINUX_WHEELS-}${MAC_WHEELS-}" + if [ -z "${wheels}" ]; then # NOT building wheels + "${ROOT_DIR}"/install-bazel.sh + fi + . "${ROOT_DIR}"/install-dependencies.sh + reload_env # We just modified our environment; reload it so we can continue +} + +build() { + local wheels="${LINUX_WHEELS-}${MAC_WHEELS-}" + if [ -z "${wheels}" ]; then # NOT building wheels + if [ "${LINT-}" != 1 ]; then # NOT linting + bazel build -k "//:*" # Do a full build first to ensure it passes + "${ROOT_DIR}"/install-ray.sh + fi + fi + + if [ "${RAY_CYTHON_EXAMPLES-}" = 1 ]; then + "${ROOT_DIR}"/install-cython-examples.sh + fi + + if [ "${RAY_DEFAULT_BUILD-}" = 1 ]; then + eval "$(curl -sL https://raw.githubusercontent.com/travis-ci/gimme/master/gimme | GIMME_GO_VERSION=master bash)" + fi + + if [ "${LINUX_WHEELS-}" = 1 ]; then + # Mount bazel cache dir to the docker container. + # For the linux wheel build, we use a shared cache between all + # wheels, but not between different travis runs, because that + # caused timeouts in the past. See the "cache: false" line below. + local MOUNT_BAZEL_CACHE=(-v "${HOME}/ray-bazel-cache":/root/ray-bazel-cache -e TRAVIS=true -e TRAVIS_PULL_REQUEST="${TRAVIS_PULL_REQUEST}" -e encrypted_1c30b31fe1ee_key="${encrypted_1c30b31fe1ee_key-}" -e encrypted_1c30b31fe1ee_iv="${encrypted_1c30b31fe1ee_iv-}") + + # This command should be kept in sync with ray/python/README-building-wheels.md, + # except the "${MOUNT_BAZEL_CACHE[@]}" part. + "${WORKSPACE_DIR}"/ci/suppress_output docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" -ti rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh + fi + + if [ "${MAC_WHEELS-}" = 1 ]; then + # This command should be kept in sync with ray/python/README-building-wheels.md. + "${WORKSPACE_DIR}"/ci/suppress_output "${WORKSPACE_DIR}"/python/build-wheel-macos.sh + fi +} + +"$@" + +{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null # Pop caller's shell options (quietly) diff --git a/ci/travis/determine_tests_to_run.py b/ci/travis/determine_tests_to_run.py index e653277f4..a9689e100 100644 --- a/ci/travis/determine_tests_to_run.py +++ b/ci/travis/determine_tests_to_run.py @@ -3,11 +3,11 @@ from __future__ import absolute_import from __future__ import division from __future__ import print_function +import json import os import re import subprocess import sys -from functools import partial from pprint import pformat @@ -25,7 +25,7 @@ def list_changed_files(commit_range): list: List of changed files within the commit range """ - command = ["git", "diff", "--name-only", commit_range] + command = ["git", "diff", "--name-only", commit_range, "--"] out = subprocess.check_output(command) return [s.strip() for s in out.decode().splitlines() if s is not None] @@ -45,9 +45,19 @@ if __name__ == "__main__": RAY_CI_STREAMING_PYTHON_AFFECTED = 0 RAY_CI_STREAMING_JAVA_AFFECTED = 0 - if os.environ["TRAVIS_EVENT_TYPE"] == "pull_request": + event_type = None + for key in ["GITHUB_EVENT_NAME", "TRAVIS_EVENT_TYPE"]: + event_type = os.getenv(key, event_type) - files = list_changed_files(os.environ["TRAVIS_COMMIT_RANGE"]) + if event_type == "pull_request": + + commit_range = os.getenv("TRAVIS_COMMIT_RANGE") + if commit_range is None: + with open(os.environ["GITHUB_EVENT_PATH"], "rb") as f: + event = json.loads(f.read()) + base = event["pull_request"]["base"]["sha"] + commit_range = "{}...{}".format(base, event["after"]) + files = list_changed_files(commit_range) print(pformat(files), file=sys.stderr) @@ -138,25 +148,20 @@ if __name__ == "__main__": RAY_CI_ONLY_RLLIB_AFFECTED = 1 # Log the modified environment variables visible in console. - for output_stream in [sys.stdout, sys.stderr]: - _print = partial(print, file=output_stream) - _print("export RAY_CI_TUNE_AFFECTED={}".format(RAY_CI_TUNE_AFFECTED)) - _print("export RAY_CI_ONLY_RLLIB_AFFECTED={}" - .format(RAY_CI_ONLY_RLLIB_AFFECTED)) - _print("export RAY_CI_RLLIB_AFFECTED={}".format(RAY_CI_RLLIB_AFFECTED)) - _print("export RAY_CI_RLLIB_FULL_AFFECTED={}".format( - RAY_CI_RLLIB_FULL_AFFECTED)) - _print("export RAY_CI_SERVE_AFFECTED={}".format(RAY_CI_SERVE_AFFECTED)) - _print("export RAY_CI_JAVA_AFFECTED={}".format(RAY_CI_JAVA_AFFECTED)) - _print( - "export RAY_CI_PYTHON_AFFECTED={}".format(RAY_CI_PYTHON_AFFECTED)) - _print("export RAY_CI_LINUX_WHEELS_AFFECTED={}" - .format(RAY_CI_LINUX_WHEELS_AFFECTED)) - _print("export RAY_CI_MACOS_WHEELS_AFFECTED={}" - .format(RAY_CI_MACOS_WHEELS_AFFECTED)) - _print("export RAY_CI_STREAMING_CPP_AFFECTED={}" - .format(RAY_CI_STREAMING_CPP_AFFECTED)) - _print("export RAY_CI_STREAMING_PYTHON_AFFECTED={}" - .format(RAY_CI_STREAMING_PYTHON_AFFECTED)) - _print("export RAY_CI_STREAMING_JAVA_AFFECTED={}" - .format(RAY_CI_STREAMING_JAVA_AFFECTED)) + print(" ".join([ + "RAY_CI_TUNE_AFFECTED={}".format(RAY_CI_TUNE_AFFECTED), + "RAY_CI_ONLY_RLLIB_AFFECTED={}".format(RAY_CI_ONLY_RLLIB_AFFECTED), + "RAY_CI_RLLIB_AFFECTED={}".format(RAY_CI_RLLIB_AFFECTED), + "RAY_CI_RLLIB_FULL_AFFECTED={}".format(RAY_CI_RLLIB_FULL_AFFECTED), + "RAY_CI_SERVE_AFFECTED={}".format(RAY_CI_SERVE_AFFECTED), + "RAY_CI_JAVA_AFFECTED={}".format(RAY_CI_JAVA_AFFECTED), + "RAY_CI_PYTHON_AFFECTED={}".format(RAY_CI_PYTHON_AFFECTED), + "RAY_CI_LINUX_WHEELS_AFFECTED={}".format(RAY_CI_LINUX_WHEELS_AFFECTED), + "RAY_CI_MACOS_WHEELS_AFFECTED={}".format(RAY_CI_MACOS_WHEELS_AFFECTED), + "RAY_CI_STREAMING_CPP_AFFECTED={}".format( + RAY_CI_STREAMING_CPP_AFFECTED), + "RAY_CI_STREAMING_PYTHON_AFFECTED={}".format( + RAY_CI_STREAMING_PYTHON_AFFECTED), + "RAY_CI_STREAMING_JAVA_AFFECTED={}".format( + RAY_CI_STREAMING_JAVA_AFFECTED), + ])) diff --git a/ci/travis/install-bazel.sh b/ci/travis/install-bazel.sh index 78e5270f9..54face5d4 100755 --- a/ci/travis/install-bazel.sh +++ b/ci/travis/install-bazel.sh @@ -61,9 +61,7 @@ fi if [ "${TRAVIS-}" = true ]; then # Use bazel disk cache if this script is running in Travis. - mkdir -p "${HOME}/ray-bazel-cache" cat <> "${HOME}/.bazelrc" -build --disk_cache="${HOME}/ray-bazel-cache" build --show_timestamps # Travis doesn't have an option to show timestamps, but GitHub Actions does # If we are in Travis, most of the compilation result will be cached. # This means we are I/O bounded. By default, Bazel set the number of concurrent @@ -74,26 +72,6 @@ build --jobs=50 EOF fi if [ -n "${GITHUB_WORKFLOW-}" ]; then - cat <<"EOF" >> "${HOME}/.profile" -# Set up environment variables the CI user needs on login to run Bazel on each platform. -if [ "${OSTYPE}" = "msys" ]; then - export USE_CLANG_CL=1 - export MSYS2_ARG_CONV_EXCL="*" # Don't let MSYS2 attempt to auto-translate arguments that look like paths - latest_python_bin="" # Detect the system Python from the registry - for latest_python_bin in /proc/registry/HKEY_LOCAL_MACHINE/Software/Python/PythonCore/*/InstallPath/@; do - if [ -f "${latest_python_bin}" ]; then - read -r latest_python_bin < "${latest_python_bin}" - latest_python_bin="${latest_python_bin}\\" - else - latest_python_bin="" - fi - done - latest_python_bin="${latest_python_bin}python.exe" - if [ -f "${latest_python_bin}" ]; then - export PYTHON2_BIN_PATH="${latest_python_bin}" PYTHON3_BIN_PATH="${latest_python_bin}" - fi -fi -EOF cat <> "${HOME}/.bazelrc" --output_base=".bazel-out" # On GitHub Actions, staying on the same volume seems to be faster EOF @@ -103,6 +81,7 @@ if [ "${TRAVIS-}" = true ] || [ -n "${GITHUB_WORKFLOW-}" ]; then # CI output doesn't scroll, so don't use curses build --color=yes build --curses=no +build --disk_cache="$(test "${OSTYPE}" = msys || echo ~/ray-bazel-cache)" build --progress_report_interval=60 # Use ray google cloud cache build --remote_cache="https://storage.googleapis.com/ray-bazel-cache" diff --git a/ci/travis/install-cython-examples.sh b/ci/travis/install-cython-examples.sh index 772530dd3..3a57afe97 100755 --- a/ci/travis/install-cython-examples.sh +++ b/ci/travis/install-cython-examples.sh @@ -1,30 +1,15 @@ #!/usr/bin/env bash -# Cause the script to exit if a single command fails -set -e +set -euxo pipefail ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -echo "PYTHON is $PYTHON" +install_cython_examples() { + ( + cd "${ROOT_DIR}/../../doc/examples/cython" + pip install scipy + python setup.py install --user + ) +} -cython_examples="$ROOT_DIR/../../doc/examples/cython" - -if [[ "$PYTHON" == "3.6" ]]; then - export PATH="$HOME/miniconda/bin:$PATH" - - pushd $cython_examples - pip install scipy - python setup.py install --user - popd - -elif [[ "$LINT" == "1" ]]; then - export PATH="$HOME/miniconda/bin:$PATH" - - pushd $cython_examples - python setup.py install --user - popd - -else - echo "Unrecognized Python version." - exit 1 -fi +install_cython_examples "$@" diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh index ca48f9530..3cd8b2eb5 100755 --- a/ci/travis/install-dependencies.sh +++ b/ci/travis/install-dependencies.sh @@ -1,122 +1,176 @@ #!/usr/bin/env bash -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +{ SHELLOPTS_STACK="${SHELLOPTS_STACK-}|$(set +o); set -$-"; } 2> /dev/null # Push caller's shell options (quietly) -echo "PYTHON is $PYTHON" +set -euxo pipefail -# Make sure all important package versions are static (via env variables -# or assign default values to them). -tf_version="$TF_VERSION" -if [[ $tf_version == "" ]]; then tf_version="2.0.0b1"; fi -echo "tf_version is $tf_version" -tfp_version="$TFP_VERSION" -if [[ tfp_version == "" ]]; then tfp_version="0.8"; fi -echo "tfp_version is $tfp_version" -torch_version="$TORCH_VERSION" -if [[ torch_version == "" ]]; then torch_version="1.4"; fi -echo "torch_version is $torch_version" +ROOT_DIR=$(builtin cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +WORKSPACE_DIR="${ROOT_DIR}/../.." -platform="" -case "${OSTYPE}" in - linux*) - platform="linux" - ;; - darwin*) - platform="macosx" - ;; - msys*) - platform="win32" - ;; -esac -if [ -z "$platform" ]; then - echo "Unrecognized platform." - exit 1 -fi +pkg_install_helper() { + case "${OSTYPE}" in + darwin*) brew install "$@";; + linux*) sudo apt-get install -qq -o=Dpkg::Use-Pty=0 "$@" | grep --line-buffered -v "^\(Preparing to unpack\|Unpacking\|Processing triggers for\) ";; + *) false;; + esac +} -echo "Platform is $platform." +install_base() { + case "${OSTYPE}" in + linux*) + sudo apt-get update -qq + pkg_install_helper build-essential curl unzip tmux gdb libunwind-dev python3-pip python3-setuptools + if [ "${LINUX_WHEELS-}" = 1 ]; then + pkg_install_helper docker + sudo usermod -a -G docker travis + fi + if [ -n "${PYTHON-}" ]; then + "${ROOT_DIR}/install-strace.sh" || true + fi + ;; + esac +} -# Upgrade pip and other packages to avoid incompatibility ERRORS. -pip install --upgrade pip # setuptools cloudpickle urllib3 +install_miniconda() { + local miniconda_version="3-4.5.4" miniconda_platform="" exe_suffix=".sh" + case "${OSTYPE}" in + linux*) miniconda_platform=Linux;; + darwin*) miniconda_platform=MacOSX;; + msys*) miniconda_platform=Windows; exe_suffix=".exe";; + esac + local miniconda_url="https://repo.continuum.io/miniconda/Miniconda${miniconda_version}-${miniconda_platform}-${HOSTTYPE}${exe_suffix}" + local miniconda_target="./${miniconda_url##*/}" + local miniconda_dir="$HOME/miniconda" + curl -s -L -o "${miniconda_target}" "${miniconda_url}" + chmod +x "${miniconda_target}" + case "${OSTYPE}" in + msys*) + miniconda_dir="${miniconda_dir}/bin" # HACK: Compensate for python.exe being in the installation root on Windows + MSYS2_ARG_CONV_EXCL="*" "${miniconda_target}" /S /D="$(cygpath -w -- "${miniconda_dir}")" + ;; + *) + "${miniconda_target}" -b -p "${miniconda_dir}" | grep --line-buffered -v "^\(installing: \|installation finished\.\)" + ;; + esac + { local set_x="${-//[^x]/}"; } 2> /dev/null # save set -x to suppress noise + set +x + local source_line='PYTHON3_BIN_PATH=python; PATH="$HOME/miniconda/bin:$PATH"; export PYTHON3_BIN_PATH PATH;' + test -f ~/.bashrc && grep -x -q -F "${source_line}" -- ~/.bashrc || echo "${source_line}" >> ~/.bashrc + test -z "${set_x}" || set -x # restore set -x + eval "${source_line}" + python -m pip install --upgrade --quiet pip +} -# If we're in a CI environment, do some configuration -if [ "${TRAVIS-}" = true ] || [ -n "${GITHUB_WORKFLOW-}" ]; then - pip config --user set global.disable-pip-version-check True - pip config --user set global.no-color True - pip config --user set global.progress_bar off - pip config --user set global.quiet True -fi +install_nvm() { + local NVM_HOME="${HOME}/.nvm" + if [ "${OSTYPE}" = msys ]; then + local version="1.1.7" + if [ ! -f "${NVM_HOME}/nvm.sh" ]; then + mkdir -p -- "${NVM_HOME}" + export NVM_SYMLINK="${PROGRAMFILES}\nodejs" + ( + cd "${NVM_HOME}" + local target="./nvm-${version}.zip" + curl -s -L -o "${target}" "https://github.com/coreybutler/nvm-windows/releases/download/${version}/nvm-noinstall.zip" + unzip -q -- "${target}" + rm -f -- "${target}" + printf "%s\r\n" "root: $(cygpath -w -- "${NVM_HOME}")" "path: ${NVM_SYMLINK}" > settings.txt + ) + printf "%s\n" "export NVM_HOME=\"$(cygpath -w -- "${NVM_HOME}")\"" 'nvm() { "${NVM_HOME}/nvm.exe" "$@"; }' > "${NVM_HOME}/nvm.sh" + fi + else + test -f "${NVM_HOME}/nvm.sh" # double-check NVM is already available on other platforms + fi +} -if [[ "$PYTHON" == "3.6" ]] && [[ "$platform" == "linux" ]]; then - sudo apt-get update - sudo apt-get install -y build-essential curl unzip tmux gdb libunwind-dev - # Install miniconda. - wget -q https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - "${ROOT_DIR}/install-strace.sh" || true - pip install scipy tensorflow==$tf_version \ - cython==0.29.0 gym \ - opencv-python-headless pyyaml pandas==0.24.2 requests \ - feather-format lxml openpyxl xlrd py-spy pytest pytest-timeout networkx tabulate aiohttp \ - uvicorn dataclasses pygments werkzeug kubernetes flask grpcio pytest-sugar pytest-rerunfailures pytest-asyncio \ - blist scikit-learn numba -elif [[ "$PYTHON" == "3.6" ]] && [[ "$platform" == "macosx" ]]; then - # Install miniconda. - wget -q https://repo.continuum.io/miniconda/Miniconda3-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - pip install scipy tensorflow==$tf_version \ - cython==0.29.0 gym \ - opencv-python-headless pyyaml pandas==0.24.2 requests \ - feather-format lxml openpyxl xlrd py-spy pytest pytest-timeout networkx tabulate aiohttp \ - uvicorn dataclasses pygments werkzeug kubernetes flask grpcio pytest-sugar pytest-rerunfailures pytest-asyncio \ - blist scikit-learn numba -elif [[ "$LINT" == "1" ]]; then - sudo apt-get update - sudo apt-get install -y build-essential curl unzip - # Install miniconda. - wget -q https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv - bash miniconda.sh -b -p $HOME/miniconda - export PATH="$HOME/miniconda/bin:$PATH" - # Install Python linting tools. - pip install flake8==3.7.7 flake8-comprehensions flake8-quotes==2.0.0 - # Install TypeScript and HTML linting tools. - pushd "$ROOT_DIR/../../python/ray/dashboard/client" - source "$HOME/.nvm/nvm.sh" - nvm install node - nvm use node - npm ci - popd -elif [[ "$LINUX_WHEELS" == "1" ]]; then - sudo apt-get install docker - sudo usermod -a -G docker travis -elif [[ "$MAC_WHEELS" == "1" ]]; then - : -else - echo "Unrecognized environment." - exit 1 -fi +install_pip() { + local python=python + if command -v python3 > /dev/null; then + python=python3 + fi -# Install modules needed in all jobs. -pip install dm-tree + if "${python}" -m pip --version || "${python}" -m ensurepip; then # If pip is present, configure it + "${python}" -m pip install --upgrade --quiet pip -# Additional RLlib dependencies. -if [[ "$RLLIB_TESTING" == "1" ]]; then - pip install tensorflow-probability==$tfp_version gast==0.2.2 \ - torch==$torch_version torchvision \ - atari_py gym[atari] lz4 smart_open -fi + # If we're in a CI environment, do some configuration + if [ "${TRAVIS-}" = true ] || [ -n "${GITHUB_WORKFLOW-}" ]; then + "${python}" -W ignore -m pip config -q --user set global.disable-pip-version-check True + "${python}" -W ignore -m pip config -q --user set global.no-color True + "${python}" -W ignore -m pip config -q --user set global.progress_bar off + "${python}" -W ignore -m pip config -q --user set global.quiet True + fi + fi +} -# Additional streaming dependencies. -if [[ "$RAY_CI_STREAMING_PYTHON_AFFECTED" == "1" ]]; then - pip install -q msgpack>=0.6.2 -fi +install_node() { + if [ "${OSTYPE}" = msys ]; then + { echo "WARNING: Skipping running Node.js due to module incompatibilities with Windows"; } 2> /dev/null + else + # Install the latest version of Node.js in order to build the dashboard. + ( + set +x # suppress set -x since it'll get very noisy here + . "${HOME}/.nvm/nvm.sh" + nvm install node + nvm use --silent node + npm config set loglevel warn # make NPM quieter + ) + fi +} -if [[ "$PYTHON" == "3.6" ]] || [[ "$MAC_WHEELS" == "1" ]]; then - # Install the latest version of Node.js in order to build the dashboard. - source "$HOME/.nvm/nvm.sh" - nvm install node -fi +run_npm() { + npm ci +} -pip install psutil setproctitle \ - --target="$ROOT_DIR/../../python/ray/thirdparty_files" +install_dependencies() { + + install_base + if [ -n "${GITHUB_WORKFLOW-}" ]; then # Keep Travis's built-in compilers and only use this for GitHub Actions (for now) + "${ROOT_DIR}"/install-toolchains.sh + fi + install_nvm + install_pip + + if [ -n "${PYTHON-}" ]; then + install_miniconda + pip_packages=(scipy tensorflow=="${TF_VERSION:-2.0.0b1}" cython==0.29.0 gym opencv-python-headless pyyaml \ + pandas==0.24.2 requests feather-format lxml openpyxl xlrd py-spy pytest pytest-timeout networkx tabulate aiohttp \ + uvicorn dataclasses pygments werkzeug kubernetes flask grpcio pytest-sugar pytest-rerunfailures pytest-asyncio \ + scikit-learn numba) + if [ "${OSTYPE}" != msys ]; then + # These packages aren't Windows-compatible + pip_packages+=(blist) # https://github.com/DanielStutzbach/blist/issues/81#issue-391460716 + fi + CC=gcc pip install "${pip_packages[@]}" + elif [ "${LINT-}" = 1 ]; then + install_miniconda + pip install flake8==3.7.7 flake8-comprehensions flake8-quotes==2.0.0 # Python linters + pushd "${WORKSPACE_DIR}/python/ray/dashboard/client" # TypeScript & HTML linters + install_node + run_npm + popd + fi + + # Install modules needed in all jobs. + pip install dm-tree + + # Additional RLlib dependencies. + if [ "${RLLIB_TESTING-}" = 1 ]; then + pip install tensorflow-probability=="${TFP_VERSION-0.8}" gast==0.2.2 torch=="${TORCH_VERSION-1.4}" torchvision \ + atari_py gym[atari] lz4 smart_open + fi + + # Additional streaming dependencies. + if [ "${RAY_CI_STREAMING_PYTHON_AFFECTED}" = 1 ]; then + pip install msgpack>=0.6.2 + fi + + if [ -n "${PYTHON-}" ] || [ "${MAC_WHEELS-}" = 1 ]; then + install_node + fi + + CC=gcc pip install psutil setproctitle --target="${WORKSPACE_DIR}/python/ray/thirdparty_files" +} + +install_dependencies "$@" + +{ set -vx; eval "${SHELLOPTS_STACK##*|}"; SHELLOPTS_STACK="${SHELLOPTS_STACK%|*}"; } 2> /dev/null # Pop caller's shell options (quietly) diff --git a/ci/travis/install-ray.sh b/ci/travis/install-ray.sh index c8d03684d..c9737914d 100755 --- a/ci/travis/install-ray.sh +++ b/ci/travis/install-ray.sh @@ -1,27 +1,35 @@ #!/usr/bin/env bash -# Cause the script to exit if a single command fails. -set -e +set -euxo pipefail ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +WORKSPACE_DIR="${ROOT_DIR}/../.." -echo "PYTHON is $PYTHON" - -if [[ "$PYTHON" == "3.6" ]]; then - export PATH="$HOME/miniconda/bin:$PATH" - - pushd "$ROOT_DIR/../../python" - pushd ray/dashboard/client - source $HOME/.nvm/nvm.sh - nvm use node +build_dashboard_front_end() { + if [ "${OSTYPE}" = msys ]; then + { echo "WARNING: Not building dashboard front-end due to NPM package incompatibilities with Windows"; } 2> /dev/null + else + ( + cd ray/dashboard/client + set +x # suppress set -x since it'll get very noisy here + . "${HOME}/.nvm/nvm.sh" + nvm use --silent node npm ci - npm run build - popd - pip install -e . --verbose - popd + npm run -s build + ) + fi +} -else - echo "Unrecognized Python version." - exit 1 -fi +install_ray() { + ( + cd "${WORKSPACE_DIR}"/python + build_dashboard_front_end + if [ "${OSTYPE}" = msys ]; then + "${WORKSPACE_DIR}"/ci/keep_alive pip install -v -e . || echo "WARNING: Ignoring Ray package build failure on Windows for now" 1>&2 + else + "${WORKSPACE_DIR}"/ci/keep_alive pip install -v -e . + fi + ) +} +install_ray "$@" diff --git a/ci/travis/install-strace.sh b/ci/travis/install-strace.sh index 20e1669b6..91aca5980 100755 --- a/ci/travis/install-strace.sh +++ b/ci/travis/install-strace.sh @@ -1,22 +1,22 @@ #!/usr/bin/env bash -set -eu +set -euxo pipefail -case "$(uname -s)" in - Linux*) - strace -qq -k -e trace=exit /bin/true 1> /dev/null 2> /dev/null || { - echo "This Linux distribution doesn't appear to support strace -k." "Attempting to build & install a recent version..." 1>&2 - git -c advice.detachedHead=false clone -q --depth=1 "https://github.com/strace/strace" -b v5.5 && ( - cd strace && - ./bootstrap > /dev/null && - CPPFLAGS="-w ${CPPFLAGS-}" ./configure --quiet --with-libunwind --enable-mpers=no && - make -s -j"$(getconf _NPROCESSORS_ONLN || echo 1)" && - sudo make -s install - ) - } - ;; - *) - # Unable to install on other platforms - false - ;; -esac +install_strace() { + case "${OSTYPE}" in + linux*) + if ! strace -qq -k -e trace=exit /bin/true 1> /dev/null 2> /dev/null; then + { echo "This Linux distribution doesn't appear to support strace -k." "Attempting to build & install a recent version..."; } 2> /dev/null + git -c advice.detachedHead=false clone -q --depth=1 "https://github.com/strace/strace" -b v5.5 && ( + cd strace && + ./bootstrap > /dev/null && + CPPFLAGS="-w ${CPPFLAGS-}" ./configure --quiet --with-libunwind --enable-mpers=no && + make -s -j"$(getconf _NPROCESSORS_ONLN || echo 1)" && + sudo make -s install + ) > /dev/null + fi;; + *) false;; + esac +} + +install_strace "$@" diff --git a/ci/travis/install-toolchains.sh b/ci/travis/install-toolchains.sh index 40a815eb5..9d71ccd2e 100755 --- a/ci/travis/install-toolchains.sh +++ b/ci/travis/install-toolchains.sh @@ -4,13 +4,12 @@ set -euxo pipefail LLVM_VERSION="9.0.0" -install_clang() { +install_toolchains() { local osversion="" url="" urlbase="https://releases.llvm.org" targetdir="/usr/local" case "${OSTYPE}" in msys) - export MSYS2_ARG_CONV_EXCL="*" # Don't let MSYS2 attempt to auto-translate arguments that look like paths osversion=win - if [ ! "${HOSTTYPE}" = "${HOSTTYPE%64}" ]; then + if [ "${HOSTTYPE}" != "${HOSTTYPE%64}" ]; then osversion="${osversion}64" else osversion="${osversion}32" @@ -31,7 +30,7 @@ install_clang() { curl -s -L -R "${url}" | if [ "${OSTYPE}" = "msys" ]; then local target="./${url##*/}" install /dev/stdin "${target}" - 7z x "${target}" -o"${targetdir}" + 7z x -bsp0 -bso0 "${target}" -o"${targetdir}" rm -f -- "${target}" else sudo tar -x -J --strip-components=1 -C "${targetdir}" @@ -40,4 +39,4 @@ install_clang() { "${targetdir}"/bin/clang --version 1>&2 } -install_"$@" +install_toolchains "$@" diff --git a/java/BUILD.bazel b/java/BUILD.bazel index 446160d4d..126ad5e43 100644 --- a/java/BUILD.bazel +++ b/java/BUILD.bazel @@ -175,7 +175,6 @@ genrule( ], outs = ["gen_maven_deps.out"], cmd = """ - set -x WORK_DIR="$$(pwd)" # Copy protobuf-generated files. rm -rf "$$WORK_DIR/java/runtime/src/main/java/io/ray/runtime/generated" @@ -204,7 +203,6 @@ genrule( ], outs = ["copy_pom_file.out"], cmd = """ - set -x WORK_DIR="$$(pwd)" cp -f $(location //java:io_ray_ray_api_pom) "$$WORK_DIR/java/api/pom.xml" cp -f $(location //java:io_ray_ray_runtime_pom) "$$WORK_DIR/java/runtime/pom.xml" diff --git a/java/test.sh b/java/test.sh index 56a5f650a..bcaaf9503 100755 --- a/java/test.sh +++ b/java/test.sh @@ -46,5 +46,5 @@ popd pushd $ROOT_DIR echo "Testing maven install." -mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN clean install -DskipTests +mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN --no-transfer-progress clean install -DskipTests popd diff --git a/src/shims/windows/sys/time.cc b/src/shims/windows/sys/time.cc index d48e9e851..2c6bfdd0e 100644 --- a/src/shims/windows/sys/time.cc +++ b/src/shims/windows/sys/time.cc @@ -1,6 +1,12 @@ #include #include +#ifdef timezone +#pragma push_macro("timezone") // Work around https://bugs.python.org/issue34657 +#undef timezone +#define timezone timezone +#endif + int gettimeofday(struct timeval *tv, struct timezone *tz) { // Free implementation from: https://stackoverflow.com/a/26085827 SYSTEMTIME systime; @@ -16,3 +22,7 @@ int gettimeofday(struct timeval *tv, struct timezone *tz) { tv->tv_usec = static_cast(systime.wMilliseconds * 1000); return 0; } + +#ifdef timezone +#pragma pop_macro("timezone") +#endif diff --git a/src/shims/windows/sys/time.h b/src/shims/windows/sys/time.h index 846dde313..2b8eb9619 100644 --- a/src/shims/windows/sys/time.h +++ b/src/shims/windows/sys/time.h @@ -1,6 +1,12 @@ #ifndef TIME_H #define TIME_H +#ifdef timezone +#pragma push_macro("timezone") // Work around https://bugs.python.org/issue34657 +#undef timezone +#define timezone timezone +#endif + #include // clients require timeval definition struct timeval; @@ -12,4 +18,8 @@ extern "C" int gettimeofday(struct timeval *tv, struct timezone *tz); +#ifdef timezone +#pragma pop_macro("timezone") +#endif + #endif /* TIME_H */ diff --git a/streaming/BUILD.bazel b/streaming/BUILD.bazel index 68e4c74ca..74b14723c 100644 --- a/streaming/BUILD.bazel +++ b/streaming/BUILD.bazel @@ -268,7 +268,6 @@ genrule( ], cmd = """ set -e - set -x WORK_DIR="$$(pwd)" # Copy generated files. GENERATED_DIR="$$WORK_DIR/streaming/python/generated" diff --git a/streaming/java/BUILD.bazel b/streaming/java/BUILD.bazel index 0ec9b8831..c7ec3f8b2 100644 --- a/streaming/java/BUILD.bazel +++ b/streaming/java/BUILD.bazel @@ -166,7 +166,6 @@ genrule( ], outs = ["copy_pom_file.out"], cmd = """ - set -x WORK_DIR="$$(pwd)" cp -f $(location //streaming/java:io_ray_ray_streaming-api_pom) "$$WORK_DIR/streaming/java/streaming-api/pom.xml" cp -f $(location //streaming/java:io_ray_ray_streaming-runtime_pom) "$$WORK_DIR/streaming/java/streaming-runtime/pom.xml" @@ -184,7 +183,6 @@ genrule( ], outs = ["cp_java_generated.out"], cmd = """ - set -x WORK_DIR="$$(pwd)" GENERATED_DIR="$$WORK_DIR/streaming/java/streaming-runtime/src/main/java/io/ray/streaming/runtime/generated" rm -rf "$$GENERATED_DIR" @@ -208,7 +206,6 @@ genrule( ], outs = ["gen_maven_deps.out"], cmd = """ - set -x WORK_DIR="$$(pwd)" # Copy native dependencies. NATIVE_DEPS_DIR="$$WORK_DIR/streaming/java/streaming-runtime/native_dependencies/" diff --git a/streaming/java/test.sh b/streaming/java/test.sh index a352becba..688d6a417 100755 --- a/streaming/java/test.sh +++ b/streaming/java/test.sh @@ -36,7 +36,7 @@ cd "$ROOT_DIR"/../../java echo "build ray maven deps" bazel build gen_maven_deps echo "maven install ray" -mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN clean install -DskipTests +mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN --no-transfer-progress clean install -DskipTests cd "$ROOT_DIR" echo "maven install ray streaming" -mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN clean install -DskipTests +mvn -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN --no-transfer-progress clean install -DskipTests