Bazel improvements (#7170)

.bazelrc

@@ -1,13 +1,10 @@
 # Must be first. Enables build:windows, build:linux, build:macos, build:freebsd, build:openbsd
 build --enable_platform_specific_config
 ###############################################################################
-build --action_env=BAZEL_LLVM
-build --action_env=BAZEL_SH
+# On       Windows, provide: USE_CLANG_CL=1, CC=clang, BAZEL_LLVM, BAZEL_SH
+# On all platforms, provide: PYTHON2_BIN_PATH, PYTHON3_BIN_PATH
+###############################################################################
 build --action_env=PATH
-build --action_env=PYTHON2_BIN_PATH
-build --action_env=PYTHON3_BIN_PATH
-# Use Clang-Cl (Clang front-end with Visual C++ backend) on Windows
-build --action_env=USE_CLANG_CL=1
 build:linux   --compilation_mode=opt
 build:macos   --compilation_mode=opt
 build:windows --compilation_mode=fastbuild
@@ -57,3 +54,8 @@ build:tsan --copt -DTHREAD_SANITIZER
 build:tsan --copt -g
 build:tsan --copt -fno-omit-frame-pointer
 build:tsan --linkopt -fsanitize=thread
+
+# [Linux] Uncomment this line to print a stack trace on exit.
+#test:linux --run_under="bash -c 'if command -v strace >/dev/null && strace -qq -k -e exit true 2>/dev/null; then strace -qq -k -e exit -e trace=\"!all\" -s 32768 -f -o >(awk \"/^[0-9]+ / { y = \\$3 != \\\"SIGCHLD\\\" && \\$3 != \\\"SIGTERM\\\" && \\$5 != \\\"SIGTERM\\\" && \\$5 != \\\"SIGKILL2\\\"; } y { print; }\" 1>&2 && cat 1>&2) -- \"$@\"; else \"$@\"; fi' -"
+# [Linux] Uncomment this line to preload libSegFault.so if available, to print a stack trace on aborts and segfault. (Note: This doesn't always work.)
+#test:linux --run_under="bash -c 'unset GREP_OPTIONS && if ! grep -q -o Microsoft /proc/version 2>/dev/null; then libs=\"$(command -v ldconfig >/dev/null && ldconfig -p | grep -F -o -e \"libSegFault.so\" | uniq | tr \"\\\\n\" :)\" && if [ -n \"${libs%:}\" ]; then export SEGFAULT_SIGNALS=\"abrt segv\" LD_PRELOAD=\"${libs}${LD_PRELOAD-}\"; fi; fi && \"$@\"' -"

ci/travis/build-helper.sh

@@ -3,6 +3,7 @@
 prep_build_env() {
   export PATH="${PATH}:${HOME}/bin"
   if [ "${OSTYPE}" = "msys" ]; then
+    export USE_CLANG_CL=1
     export MSYS2_ARG_CONV_EXCL="*"  # Don't let MSYS2 attempt to auto-translate arguments that look like paths
     local latest_python_bin=""
     for latest_python_bin in /proc/registry/HKEY_LOCAL_MACHINE/Software/Python/PythonCore/*/InstallPath/@; do

ci/travis/install-bazel.sh

@@ -41,11 +41,21 @@ fi
 if [ "${TRAVIS-}" = true ]; then
   # Use bazel disk cache if this script is running in Travis.
   mkdir -p "${HOME}/ray-bazel-cache"
-  echo "build --disk_cache=${HOME}/ray-bazel-cache" >> "${HOME}/.bazelrc"
+  cat <<EOF >> "${HOME}/.bazelrc"
+build --disk_cache="${HOME}/ray-bazel-cache"
+EOF
 fi
 if [ "${TRAVIS-}" = true ] || [ -n "${GITHUB_TOKEN-}" ]; then
-  # Use ray google cloud cache
-  echo "build --remote_cache=https://storage.googleapis.com/ray-bazel-cache" >> "${HOME}/.bazelrc"
+  cat <<EOF >> "${HOME}/.bazelrc"
+# CI output doesn't scroll, so don't use curses
+build --curses=no
+build --progress_report_interval=60
+# Use ray google cloud cache
+build --remote_cache="https://storage.googleapis.com/ray-bazel-cache"
+build --show_progress_rate_limit=15
+build --show_timestamps
+build --ui_actions_shown=1024
+EOF
   # If we are in master build, we can write to the cache as well.
   upload=0
   if [ "${TRAVIS_PULL_REQUEST-false}" = false ]; then
@@ -71,9 +81,13 @@ if [ "${TRAVIS-}" = true ] || [ -n "${GITHUB_TOKEN-}" ]; then
     if [ "${OSTYPE}" = msys ]; then  # On Windows, we need path translation
       translated_path="$(cygpath -m -- "${translated_path}")"
     fi
-    echo "build --google_credentials=\"${translated_path}\"" >> "${HOME}/.bazelrc"
+    cat <<EOF >> "${HOME}/.bazelrc"
+build --google_credentials="${translated_path}"
+EOF
   else
     echo "Using remote build cache in read-only mode." 1>&2
-    echo "build --remote_upload_local_results=false" >> "${HOME}/.bazelrc"
+    cat <<EOF >> "${HOME}/.bazelrc"
+build --remote_upload_local_results=false
+EOF
   fi
 fi

ci/travis/install-cython-examples.sh

@@ -13,7 +13,7 @@ if [[ "$PYTHON" == "3.6" ]]; then
    export PATH="$HOME/miniconda/bin:$PATH"
 
    pushd $cython_examples
-   pip install --progress-bar=off scipy
+   pip install scipy
    python setup.py install --user
    popd
 

ci/travis/install-dependencies.sh

@@ -32,14 +32,23 @@ fi
 # Upgrade pip and other packages to avoid incompatibility ERRORS.
 pip install --upgrade pip # setuptools cloudpickle urllib3
 
+# If we're in a CI environment, do some configuration
+if [ "${TRAVIS-}" = true ] || [ -n "${GITHUB_TOKEN-}" ]; then
+  pip config --user set global.disable-pip-version-check True
+  pip config --user set global.no-color True
+  pip config --user set global.progress_bar off
+  pip config --user set global.quiet True
+fi
+
 if [[ "$PYTHON" == "3.6" ]] && [[ "$platform" == "linux" ]]; then
   sudo apt-get update
-  sudo apt-get install -y python-dev python-numpy build-essential curl unzip tmux gdb
+  sudo apt-get install -y python-dev python-numpy build-essential curl unzip tmux gdb libunwind-dev
   # Install miniconda.
   wget -q https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q scipy tensorflow==$tf_version \
+  "${ROOT_DIR}/install-strace.sh" || true
+  pip install scipy tensorflow==$tf_version \
     cython==0.29.0 gym \
     opencv-python-headless pyyaml pandas==0.24.2 requests \
     feather-format lxml openpyxl xlrd py-spy pytest-timeout networkx tabulate aiohttp \
@@ -50,7 +59,7 @@ elif [[ "$PYTHON" == "3.6" ]] && [[ "$platform" == "macosx" ]]; then
   wget -q https://repo.continuum.io/miniconda/Miniconda3-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q scipy tensorflow==$tf_version \
+  pip install scipy tensorflow==$tf_version \
     cython==0.29.0 gym \
     opencv-python-headless pyyaml pandas==0.24.2 requests \
     feather-format lxml openpyxl xlrd py-spy pytest-timeout networkx tabulate aiohttp \
@@ -64,7 +73,7 @@ elif [[ "$LINT" == "1" ]]; then
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
   # Install Python linting tools.
-  pip install -q flake8==3.7.7 flake8-comprehensions flake8-quotes==2.0.0
+  pip install flake8==3.7.7 flake8-comprehensions flake8-quotes==2.0.0
   # Install TypeScript and HTML linting tools.
   pushd "$ROOT_DIR/../../python/ray/dashboard/client"
     source "$HOME/.nvm/nvm.sh"
@@ -84,7 +93,7 @@ fi
 
 # Additional RLlib dependencies.
 if [[ "$RLLIB_TESTING" == "1" ]]; then
-  pip install -q tensorflow-probability==$tfp_version gast==0.2.2 \
+  pip install tensorflow-probability==$tfp_version gast==0.2.2 \
     torch==$torch_version torchvision \
     gym[atari] atari_py smart_open lz4
 fi
@@ -100,5 +109,5 @@ if [[ "$PYTHON" == "3.6" ]] || [[ "$MAC_WHEELS" == "1" ]]; then
   nvm install node
 fi
 
-pip install -q psutil setproctitle \
+pip install psutil setproctitle \
         --target="$ROOT_DIR/../../python/ray/thirdparty_files"

ci/travis/install-strace.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -eu
+
+case "$(uname -s)" in
+  Linux*)
+    strace -qq -k -e trace=exit /bin/true || {
+      echo "This Linux distribution doesn't appear to support strace -k." "Attempting to build & install a recent version..." 1>&2
+      git clone -q --depth=1 "https://github.com/strace/strace" -b v5.5 && (
+        cd strace &&
+        ./bootstrap > /dev/null &&
+        ./configure --quiet --with-libunwind --enable-mpers=no &&
+        make -s -j"$(getconf _NPROCESSORS_ONLN || echo 1)" &&
+        sudo make install
+      )
+    }
+    ;;
+  *)
+    # Unable to install on other platforms
+    false
+    ;;
+esac