Make Bazel the default build system (#3898)

.travis.yml

@@ -25,6 +25,7 @@ matrix:
       env:
         - JDK='Oracle JDK 8'
         - PYTHON=3.5 PYTHONWARNINGS=ignore
+        - RAY_USE_CMAKE=1
       install:
         - ./ci/travis/install-dependencies.sh
         - export PATH="$HOME/miniconda/bin:$PATH"
@@ -38,7 +39,7 @@ matrix:
 
     - os: linux
       dist: trusty
-      env: LINT=1 PYTHONWARNINGS=ignore
+      env: LINT=1 PYTHONWARNINGS=ignore RAY_USE_CMAKE=1
       before_install:
         # In case we ever want to use a different version of clang-format:
         #- wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
@@ -76,7 +77,7 @@ matrix:
 
     - os: linux
       dist: trusty
-      env: VALGRIND=1 PYTHON=2.7 PYTHONWARNINGS=ignore
+      env: VALGRIND=1 PYTHON=2.7 PYTHONWARNINGS=ignore RAY_USE_CMAKE=1
       before_install:
         - sudo apt-get update -qq
         - sudo apt-get install -qq valgrind
@@ -107,7 +108,7 @@ matrix:
     # Build Linux wheels.
     - os: linux
       dist: trusty
-      env: LINUX_WHEELS=1 PYTHONWARNINGS=ignore
+      env: LINUX_WHEELS=1 PYTHONWARNINGS=ignore RAY_USE_CMAKE=1
       install:
         - ./ci/travis/install-dependencies.sh
         # This command should be kept in sync with ray/python/README-building-wheels.md.
@@ -118,7 +119,7 @@ matrix:
     # Build MacOS wheels.
     - os: osx
       osx_image: xcode7
-      env: MAC_WHEELS=1 PYTHONWARNINGS=ignore
+      env: MAC_WHEELS=1 PYTHONWARNINGS=ignore RAY_USE_CMAKE=1
       install:
         - ./ci/travis/install-dependencies.sh
         # This command should be kept in sync with ray/python/README-building-wheels.md.
@@ -133,28 +134,28 @@ matrix:
         - PYTHON=3.5
         - RAY_USE_NEW_GCS=on
         - PYTHONWARNINGS=ignore
+        - RAY_USE_CMAKE=1
 
 
 install:
+  - ./ci/travis/install-bazel.sh
   - ./ci/travis/install-dependencies.sh
   - export PATH="$HOME/miniconda/bin:$PATH"
   - ./ci/travis/install-ray.sh
   - ./ci/travis/install-cython-examples.sh
 
-  - cd build
-  - bash ../src/ray/test/run_gcs_tests.sh
+  - bash src/ray/test/run_gcs_tests.sh
   # Raylet tests.
-  - bash ../src/ray/test/run_object_manager_tests.sh
-  - ./src/ray/raylet/task_test
-  - ./src/ray/raylet/worker_pool_test
-  - ./src/ray/raylet/lineage_cache_test
-  - ./src/ray/raylet/task_dependency_manager_test
-  - ./src/ray/raylet/reconstruction_policy_test
-  - ./src/ray/raylet/client_connection_test
-  - ./src/ray/util/logging_test --gtest_filter=PrintLogTest*
-  - ./src/ray/util/signal_test
-
-  - cd ..
+  - bash src/ray/test/run_object_manager_tests.sh
+  - bazel build "//:task_test" "//:worker_pool_test" "//:lineage_cache_test" "//:task_dependency_manager_test" "//:reconstruction_policy_test" "//:client_connection_test" "//:logging_test" "//:signal_test" -c opt
+  - ./bazel-bin/task_test
+  - ./bazel-bin/worker_pool_test
+  - ./bazel-bin/lineage_cache_test
+  - ./bazel-bin/task_dependency_manager_test
+  - ./bazel-bin/reconstruction_policy_test
+  - ./bazel-bin/client_connection_test
+  - ./bazel-bin/logging_test --gtest_filter=PrintLogTest*
+  - ./bazel-bin/signal_test
 
 script:
   - export PATH="$HOME/miniconda/bin:$PATH"
@@ -174,10 +175,10 @@ script:
   - python -m pytest -v --durations=10 python/ray/tune/test/automl_searcher_test.py
 
   # ray rllib tests
-  - python -m pytest -v --durations=10 python/ray/rllib/test/test_catalog.py
-  - python -m pytest -v --durations=10 python/ray/rllib/test/test_filters.py
-  - python -m pytest -v --durations=10 python/ray/rllib/test/test_optimizers.py
-  - python -m pytest -v --durations=10 python/ray/rllib/test/test_evaluators.py
+  - python python/ray/rllib/test/test_catalog.py
+  - python python/ray/rllib/test/test_filters.py
+  - python python/ray/rllib/test/test_optimizers.py
+  - python python/ray/rllib/test/test_evaluators.py
 
   # Python3.5+ only. Otherwise we will get `SyntaxError` regardless of how we set the tester.
   - python -c 'import sys;exit(sys.version_info>=(3,5))' || python -m pytest -v --durations=10 python/ray/experimental/test/async_test.py

BUILD.bazel

@@ -123,6 +123,26 @@ cc_test(
     ],
 )
 
+cc_test(
+    name = "client_connection_test",
+    srcs = ["src/ray/raylet/client_connection_test.cc"],
+    copts = COPTS,
+    deps = [
+        ":raylet_lib",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
+cc_test(
+    name = "signal_test",
+    srcs = ["src/ray/util/signal_test.cc"],
+    copts = COPTS,
+    deps = [
+        ":raylet_lib",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
 cc_library(
     name = "object_manager",
     srcs = glob([
@@ -403,6 +423,7 @@ flatbuffer_py_library(
         "src/ray/gcs/format/gcs.fbs",
     ],
     outs = [
+        "ActorCheckpointIdData.py",
         "ActorState.py",
         "ActorTableData.py",
         "Arg.py",
@@ -412,6 +433,7 @@ flatbuffer_py_library(
         "CustomSerializerData.py",
         "DriverTableData.py",
         "ErrorTableData.py",
+        "ErrorType.py",
         "FunctionTableData.py",
         "GcsTableEntry.py",
         "HeartbeatBatchTableData.py",
@@ -470,6 +492,9 @@ filegroup(
     srcs = glob([
         "python/ray/*.py",
         "python/ray/autoscaler/*.py",
+        "python/ray/autoscaler/aws/example-full.yaml",
+        "python/ray/autoscaler/gcp/example-full.yaml",
+        "python/ray/autoscaler/local/example-full.yaml",
         "python/ray/cloudpickle/*.py",
         "python/ray/core/__init__.py",
         "python/ray/core/generated/__init__.py",
@@ -484,25 +509,31 @@ filegroup(
 
 genrule(
     name = "redis",
-    outs = ["redis-server"],
+    outs = [
+        "redis-server",
+        "redis-cli",
+    ],
     cmd = """
         set -x &&
         curl -sL \"https://github.com/antirez/redis/archive/5.0.3.tar.gz\" | tar xz --strip-components=1 -C . &&
         make &&
-        mv ./src/redis-server $(location redis-server)
+        mv ./src/redis-server $(location redis-server) &&
+        chmod +x $(location redis-server) &&
+        mv ./src/redis-cli $(location redis-cli) &&
+        chmod +x $(location redis-cli)
     """,
 )
 
-cc_library(
-    name = "ray_redis_module",
+cc_binary(
+    name = "ray_redis_module.so",
     srcs = [
         "src/ray/gcs/redis_module/ray_redis_module.cc",
-        "src/ray/util/logging.cc",
-    ],
-    hdrs = [
         "src/ray/gcs/redis_module/redis_string.h",
         "src/ray/gcs/redis_module/redismodule.h",
     ],
+    copts = COPTS,
+    linkshared = 1,
+    linkstatic = 1,
     deps = [
         ":ray_common",
     ],
@@ -516,12 +547,11 @@ genrule(
         "//:python_gcs_fbs",
         "//:python_node_manager_fbs",
         "//:redis-server",
-        "//:ray_redis_module",
+        "//:redis-cli",
+        "//:ray_redis_module.so",
         "//:raylet",
         "//:raylet_monitor",
         "@plasma//:plasma_store_server",
-        "@plasma//:plasma_client_java",
-        "//:raylet_library_java",
     ],
     outs = ["ray_pkg"],
     cmd = """
@@ -530,16 +560,17 @@ genrule(
         cp $(location python/ray/_raylet.so) python/ray &&
         mkdir -p python/ray/core/src/ray/thirdparty/redis/src/ &&
         cp $(location //:redis-server) python/ray/core/src/ray/thirdparty/redis/src/ &&
+        cp $(location //:redis-cli) python/ray/core/src/ray/thirdparty/redis/src/ &&
         mkdir -p python/ray/core/src/ray/gcs/redis_module/ &&
-        for f in $(locations //:ray_redis_module); do cp $$f python/ray/core/src/ray/gcs/redis_module/; done &&
+        cp $(locations //:ray_redis_module.so) python/ray/core/src/ray/gcs/redis_module/libray_redis_module.so &&
         mkdir -p python/ray/core/src/ray/raylet/ &&
         cp $(location //:raylet_monitor) python/ray/core/src/ray/raylet/ &&
         mkdir -p python/ray/core/src/plasma &&
         cp $(location @plasma//:plasma_store_server) python/ray/core/src/plasma/ &&
         cp $(location //:raylet) python/ray/core/src/ray/raylet/ &&
-        mv python $(location ray_pkg) &&
-        mkdir -p $(location ray_pkg)/java_lib &&
-        cp $(location @plasma//:plasma_client_java) $(location ray_pkg)/java_lib &&
-        cp $(location //:raylet_library_java) $(location ray_pkg)/java_lib
+        for f in $(locations //:python_gcs_fbs); do cp $$f python/ray/core/generated/; done &&
+        mkdir -p python/ray/core/generated/ray/protocol/ &&
+        for f in $(locations //:python_node_manager_fbs); do cp $$f python/ray/core/generated/ray/protocol/; done &&
+        mv python $(location ray_pkg)
         """,
 )

build.sh

@@ -25,7 +25,7 @@ function usage()
 # Determine how many parallel jobs to use for make based on the number of cores
 unamestr="$(uname)"
 if [[ "$unamestr" == "Linux" ]]; then
-  PARALLEL=$(nproc)
+  PARALLEL=1
 elif [[ "$unamestr" == "Darwin" ]]; then
   PARALLEL=$(sysctl -n hw.ncpu)
 else
@@ -101,16 +101,29 @@ fi
 
 pushd "$BUILD_DIR"
 
-# avoid the command failed and exits
-# and cmake will check some directories to determine whether some targets built
-make clean || true
-rm -rf external/arrow-install
+if [ ! -z "$RAY_USE_CMAKE" ] ; then
+  # avoid the command failed and exits
+  # and cmake will check some directories to determine whether some targets built
+  make clean || true
+  rm -rf external/arrow-install
 
-cmake -DCMAKE_BUILD_TYPE=$CBUILD_TYPE \
-      -DCMAKE_RAY_LANG_JAVA=$RAY_BUILD_JAVA \
-      -DCMAKE_RAY_LANG_PYTHON=$RAY_BUILD_PYTHON \
-      -DRAY_USE_NEW_GCS=$RAY_USE_NEW_GCS \
-      -DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE $ROOT_DIR
+  cmake -DCMAKE_BUILD_TYPE=$CBUILD_TYPE \
+        -DCMAKE_RAY_LANG_JAVA=$RAY_BUILD_JAVA \
+        -DCMAKE_RAY_LANG_PYTHON=$RAY_BUILD_PYTHON \
+        -DRAY_USE_NEW_GCS=$RAY_USE_NEW_GCS \
+        -DPYTHON_EXECUTABLE:FILEPATH=$PYTHON_EXECUTABLE $ROOT_DIR
+
+  make -j${PARALLEL}
+else
+  # The following line installs pyarrow from S3, these wheels have been
+  # generated from https://github.com/ray-project/arrow-build from
+  # the commit listed in the command.
+  $PYTHON_EXECUTABLE -m pip install \
+      --target=$ROOT_DIR/python/ray/pyarrow_files pyarrow==0.12.0-RAY \
+      --find-links https://s3-us-west-2.amazonaws.com/arrow-wheels/bf0f3a15e0d6583cfaedbca508627b06746fa41a/index.html
+  bazel build //:ray_pkg -c opt
+  # Copy files and skip existing files
+  cp -r -n $ROOT_DIR/bazel-genfiles/ray_pkg/ray $ROOT_DIR/python || true
+fi
 
-make -j${PARALLEL}
 popd

ci/travis/install-dependencies.sh

@@ -24,8 +24,9 @@ if [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "linux" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda2-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.29.0 cmake tensorflow gym==0.10.11 opencv-python pyyaml pandas==0.23.4 requests \
+  pip install -q cython==0.29.0 cmake gym==0.10.11 opencv-python-headless pyyaml pandas==0.23.4 requests \
     feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout mock flaky
+  conda install -y scipy tensorflow
 elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then
   sudo apt-get update
   sudo apt-get install -y cmake pkg-config python-dev python-numpy build-essential autoconf curl libtool unzip tmux gdb
@@ -33,8 +34,9 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "linux" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
+  pip install -q cython==0.29.0 cmake gym opencv-python-headless pyyaml pandas==0.23.4 requests \
     feather-format lxml openpyxl xlrd py-spy setproctitle pytest-timeout flaky
+  conda install -y scipy tensorflow
 elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
   # check that brew is installed
   which -s brew
@@ -50,7 +52,7 @@ elif [[ "$PYTHON" == "2.7" ]] && [[ "$platform" == "macosx" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda2-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.29.0 cmake tensorflow gym==0.10.11 opencv-python pyyaml pandas==0.23.4 requests \
+  pip install -q cython==0.29.0 cmake tensorflow gym==0.10.11 opencv-python-headless pyyaml pandas==0.23.4 requests \
     feather-format lxml openpyxl xlrd py-spy setproctitle faulthandler pytest-timeout mock flaky
 elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
   # check that brew is installed
@@ -67,7 +69,7 @@ elif [[ "$PYTHON" == "3.5" ]] && [[ "$platform" == "macosx" ]]; then
   wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-MacOSX-x86_64.sh -O miniconda.sh -nv
   bash miniconda.sh -b -p $HOME/miniconda
   export PATH="$HOME/miniconda/bin:$PATH"
-  pip install -q cython==0.29.0 cmake tensorflow gym opencv-python pyyaml pandas==0.23.4 requests \
+  pip install -q cython==0.29.0 cmake tensorflow gym opencv-python-headless pyyaml pandas==0.23.4 requests \
     feather-format lxml openpyxl xlrd py-spy setproctitle pytest-timeout flaky
 elif [[ "$LINT" == "1" ]]; then
   sudo apt-get update

doc/requirements-doc.txt

@@ -5,7 +5,7 @@ flatbuffers
 funcsigs
 mock
 numpy
-opencv-python
+opencv-python-headless
 pyarrow
 pyyaml
 recommonmark

doc/source/example-a3c.rst

@@ -22,7 +22,7 @@ To run the application, first install **ray** and then some dependencies:
   pip install tensorflow
   pip install six
   pip install gym[atari]
-  pip install opencv-python
+  pip install opencv-python-headless
   pip install scipy
 
 You can run the code with

docker/deploy/Dockerfile

@@ -2,6 +2,7 @@
 # for end users.
 
 FROM ray-project/base-deps
+ENV RAY_USE_CMAKE=1
 ADD ray.tar /ray
 ADD git-rev /ray/git-rev
 WORKDIR /ray/python

docker/examples/Dockerfile

@@ -5,7 +5,7 @@ FROM ray-project/deploy
 # This updates numpy to 1.14 and mutes errors from other libraries
 RUN conda install -y numpy
 RUN apt-get install -y zlib1g-dev
-RUN pip install gym[atari]==0.10.11 opencv-python==3.2.0.8 tensorflow lz4 keras pytest-timeout smart_open
+RUN pip install gym[atari]==0.10.11 opencv-python-headless tensorflow lz4 keras pytest-timeout smart_open
 RUN pip install -U h5py  # Mutes FutureWarnings
 RUN pip install --upgrade bayesian-optimization
 RUN pip install --upgrade git+git://github.com/hyperopt/hyperopt.git

python/build-wheel-manylinux1.sh

@@ -6,6 +6,8 @@ echo 10
 EOF
 chmod +x /usr/bin/nproc
 
+export RAY_USE_CMAKE=1
+
 PYTHONS=("cp27-cp27mu"
          "cp34-cp34m"
          "cp35-cp35m"

python/setup.py

@@ -64,7 +64,9 @@ else:
 optional_ray_files += ray_autoscaler_files
 
 extras = {
-    "rllib": ["pyyaml", "gym[atari]", "opencv-python", "lz4", "scipy"],
+    "rllib": [
+        "pyyaml", "gym[atari]", "opencv-python-headless", "lz4", "scipy"
+    ],
     "debug": ["psutil", "setproctitle", "py-spy"],
     "dashboard": ["psutil", "aiohttp"],
 }
@@ -120,8 +122,9 @@ class build_ext(_build_ext.build_ext):
         parent_directory = os.path.dirname(destination)
         if not os.path.exists(parent_directory):
             os.makedirs(parent_directory)
-        print("Copying {} to {}.".format(source, destination))
-        shutil.copy(source, destination)
+        if not os.path.exists(destination):
+            print("Copying {} to {}.".format(source, destination))
+            shutil.copy(source, destination)
 
 
 class BinaryDistribution(Distribution):

src/ray/CMakeLists.txt

@@ -61,6 +61,7 @@ set(RAY_LIB_STATIC_LINK_LIBS ${PLASMA_STATIC_LIB} ${ARROW_STATIC_LIB} ${RAY_SYST
 set(RAY_LIB_DEPENDENCIES
     boost_thread
     arrow_ep
+    googletest_ep
     gen_gcs_fbs
     gen_object_manager_fbs
     gen_node_manager_fbs)

src/ray/test/run_gcs_tests.sh

@@ -1,11 +1,13 @@
 #!/usr/bin/env bash
 
-# This needs to be run in the build tree, which is normally ray/build
+# This needs to be run in the root directory
 
 # Cause the script to exit if a single command fails.
 set -e
 set -x
 
+bazel build "//:gcs_client_test" "//:asio_test" "//:ray_redis_module.so" -c opt
+
 # Start Redis.
 if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then
     ./src/credis/redis/src/redis-server \
@@ -14,15 +16,15 @@ if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then
         --loadmodule ./src/ray/gcs/redis_module/libray_redis_module.so \
         --port 6379 &
 else
-    ./src/ray/thirdparty/redis/src/redis-server \
+    ./bazel-genfiles/ray_pkg/ray/core/src/ray/thirdparty/redis/src/redis-server \
         --loglevel warning \
-        --loadmodule ./src/ray/gcs/redis_module/libray_redis_module.so \
+        --loadmodule ./bazel-bin/ray_redis_module.so \
         --port 6379 &
 fi
 sleep 1s
 
-./src/ray/gcs/client_test
-./src/ray/gcs/asio_test
+./bazel-bin/gcs_client_test
+./bazel-bin/asio_test
 
-./src/ray/thirdparty/redis/src/redis-cli -p 6379 shutdown
+./bazel-genfiles/ray_pkg/ray/core/src/ray/thirdparty/redis/src/redis-cli -p 6379 shutdown
 sleep 1s

src/ray/test/run_object_manager_tests.sh

@@ -6,6 +6,8 @@
 set -e
 set -x
 
+bazel build "//:object_manager_stress_test" "//:object_manager_test" "@plasma//:plasma_store_server" -c opt
+
 # Get the directory in which this script is executing.
 SCRIPT_DIR="`dirname \"$0\"`"
 RAY_ROOT="$SCRIPT_DIR/../../.."
@@ -22,8 +24,7 @@ fi
 
 CORE_DIR="$RAY_ROOT/build"
 PYTHON_CORE_DIR="$RAY_ROOT/python/ray/core"
-REDIS_MODULE="$PYTHON_CORE_DIR/src/ray/gcs/redis_module/libray_redis_module.so"
-REDIS_DIR="$CORE_DIR/src/ray/thirdparty/redis/src"
+REDIS_MODULE="./bazel-genfiles/ray_pkg/ray/core/src/ray/gcs/redis_module/libray_redis_module.so"
 
 if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then
     REDIS_SERVER="$CORE_DIR/src/credis/redis/src/redis-server"
@@ -31,25 +32,25 @@ if [[ "${RAY_USE_NEW_GCS}" = "on" ]]; then
     CREDIS_MODULE="$CORE_DIR/src/credis/build/src/libmember.so"
     LOAD_MODULE_ARGS="--loadmodule ${CREDIS_MODULE} --loadmodule ${REDIS_MODULE}"
 else
-    REDIS_SERVER="${REDIS_DIR}/redis-server"
+    REDIS_SERVER="./bazel-genfiles/ray_pkg/ray/core/src/ray/thirdparty/redis/src/redis-server"
     LOAD_MODULE_ARGS="--loadmodule ${REDIS_MODULE}"
 fi
 
-STORE_EXEC="$PYTHON_CORE_DIR/src/plasma/plasma_store_server"
+STORE_EXEC="./bazel-bin/external/plasma/plasma_store_server"
 
 # Allow cleanup commands to fail.
-$REDIS_DIR/redis-cli -p 6379 shutdown || true
+./bazel-genfiles/ray_pkg/ray/core/src/ray/thirdparty/redis/src/redis-cli -p 6379 shutdown || true
 sleep 1s
 ${REDIS_SERVER} --loglevel warning ${LOAD_MODULE_ARGS} --port 6379 &
 sleep 1s
 # Run tests.
-$CORE_DIR/src/ray/object_manager/object_manager_stress_test $STORE_EXEC
+./bazel-bin/object_manager_stress_test $STORE_EXEC
 sleep 1s
 # Use timeout=1000ms for the Wait tests.
-$CORE_DIR/src/ray/object_manager/object_manager_test $STORE_EXEC 1000
+./bazel-bin/object_manager_test $STORE_EXEC 1000
 # Run tests again with inlined objects.
-$CORE_DIR/src/ray/object_manager/object_manager_test $STORE_EXEC 1000 true
-$REDIS_DIR/redis-cli -p 6379 shutdown
+./bazel-bin/object_manager_test $STORE_EXEC 1000 true
+./bazel-genfiles/ray_pkg/ray/core/src/ray/thirdparty/redis/src/redis-cli -p 6379 shutdown
 sleep 1s
 
 # Include raylet integration test once it's ready.