From c218f2eff6dc127fa8a8277710a31e837dbbb7ee Mon Sep 17 00:00:00 2001 From: Simon Mo Date: Mon, 3 Aug 2020 16:37:15 -0700 Subject: [PATCH] [docker] Build docker in Travis PR & Fix image build failing (#9787) Co-authored-by: Ian Rodney --- .travis.yml | 3 +- ci/travis/build-autoscaler-images.sh | 48 ------------------ ci/travis/build-docker-images.sh | 75 ++++++++++++++++++++++++++++ ci/travis/ci.sh | 2 + ci/travis/determine_tests_to_run.py | 3 ++ docker/autoscaler/Dockerfile | 18 +++---- docker/base-deps/Dockerfile | 9 ++-- docker/examples/Dockerfile | 2 +- docker/ray/Dockerfile | 7 +++ 9 files changed, 101 insertions(+), 66 deletions(-) delete mode 100755 ci/travis/build-autoscaler-images.sh create mode 100755 ci/travis/build-docker-images.sh create mode 100644 docker/ray/Dockerfile diff --git a/.travis.yml b/.travis.yml index 149c502cc..1fa550268 100644 --- a/.travis.yml +++ b/.travis.yml @@ -126,6 +126,7 @@ matrix: - . ./ci/travis/ci.sh build script: - . ./ci/travis/ci.sh test_wheels + - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then ./ci/keep_alive bash $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.sh; fi - bash ./java/build-jar-multiplatform.sh linux - bash ./java/build-jar-multiplatform.sh multiplatform #- bash ./java/build-jar-multiplatform.sh deploy @@ -431,7 +432,7 @@ deploy: - provider: script edge: true # This supposedly opts in to deploy v2. - script: ./ci/keep_alive bash $TRAVIS_BUILD_DIR/ci/travis/build-autoscaler-images.sh || true + script: ./ci/keep_alive bash $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.sh skip_cleanup: true on: repo: ray-project/ray diff --git a/ci/travis/build-autoscaler-images.sh b/ci/travis/build-autoscaler-images.sh deleted file mode 100755 index c41bf28fe..000000000 --- a/ci/travis/build-autoscaler-images.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/usr/bin/env bash - -# This script build docker images for autoscaler. -# For now, we only build python3.6 images. -set -e -set -x - -SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) -ROOT_DIR=$(cd "$SCRIPT_DIR"/../../; pwd) -DOCKER_USERNAME="raytravisbot" - -# We will only build and push when we are building branch build. -if [[ "$TRAVIS" == "true" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then - echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin - - wheel="$(basename "$ROOT_DIR"/.whl/*cp36m-manylinux*)" - commit_sha=$(echo "$TRAVIS_COMMIT" | head -c 6) - cp -r "$ROOT_DIR"/.whl "$ROOT_DIR"/docker/autoscaler/.whl - - docker build -q -t rayproject/base-deps docker/base-deps - - docker build \ - --build-arg WHEEL_PATH=".whl/$wheel" \ - --build-arg WHEEL_NAME="$wheel" \ - -t rayproject/autoscaler:"$commit_sha" \ - "$ROOT_DIR"/docker/autoscaler - - docker tag rayproject/base-deps rayproject/base-deps:"$commit_sha" - docker push rayproject/base-deps:"$commit_sha" - docker push rayproject/autoscaler:"$commit_sha" - - - # We have a branch build, e.g. release/v0.7.0 - if [[ "$TRAVIS_BRANCH" != "master" ]]; then - # Replace / in branch name to - so it is legal tag name - normalized_branch_name=$(echo "$TRAVIS_BRANCH" | sed -e "s/\//-/") - docker tag rayproject/autoscaler:"$commit_sha" rayproject/autoscaler:"$normalized_branch_name" - docker tag rayproject/base-deps:"$commit_sha" rayproject/base-deps:"$normalized_branch_name" - docker push rayproject/autoscaler:"$normalized_branch_name" - docker push rayproject/base-deps:"$normalized_branch_name" - else - docker tag rayproject/autoscaler:"$commit_sha" rayproject/autoscaler:latest - docker tag rayproject/base-deps:"$commit_sha" rayproject/base-deps:latest - docker push rayproject/autoscaler:latest - docker push rayproject/base-deps:latest - fi -fi - diff --git a/ci/travis/build-docker-images.sh b/ci/travis/build-docker-images.sh new file mode 100755 index 000000000..31279416b --- /dev/null +++ b/ci/travis/build-docker-images.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash + +# This script build docker images for autoscaler. +# For now, we only build python3.6 images. +set -e +set -x + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) +ROOT_DIR=$(cd "$SCRIPT_DIR"/../../; pwd) +DOCKER_USERNAME="raytravisbot" + +docker_push() { + if [[ "$TRAVIS_PULL_REQUEST" == "false" ]]; then + docker push $@ + else + echo "Skipping docker push because it's in PR environment." + fi +} + +# We will only build and push when we are building branch build. +if [[ "$TRAVIS" == "true" ]]; then + + if [[ "$TRAVIS_PULL_REQUEST" == "false" ]]; then + echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin + else + if [[ "$RAY_CI_DOCKER_AFFECTED" == "0" ]]; then + echo "Skipping docker build in PR build because dockerfile didn't change." + exit 0 + fi + fi + + wheel="$(basename "$ROOT_DIR"/.whl/*cp37m-manylinux*)" + commit_sha=$(echo "$TRAVIS_COMMIT" | head -c 6) + cp -r "$ROOT_DIR"/.whl "$ROOT_DIR"/docker/ray/.whl + cp $ROOT_DIR/python/requirements.txt $ROOT_DIR/docker/autoscaler/requirements.txt + cp $ROOT_DIR/python/requirements_autoscaler.txt $ROOT_DIR/docker/autoscaler/requirements_autoscaler.txt + + docker build -t rayproject/base-deps docker/base-deps + + docker build \ + --build-arg WHEEL_PATH=".whl/$wheel" \ + -t rayproject/ray \ + "$ROOT_DIR"/docker/ray + + docker build \ + -t rayproject/autoscaler:"$commit_sha" \ + "$ROOT_DIR"/docker/autoscaler + + docker tag rayproject/base-deps rayproject/base-deps:"$commit_sha" + docker tag rayproject/ray rayproject/ray:"$commit_sha" + docker_push rayproject/base-deps:"$commit_sha" + docker_push rayproject/ray:"$commit_sha" + docker_push rayproject/autoscaler:"$commit_sha" + + + # We have a branch build, e.g. release/v0.7.0 + if [[ "$TRAVIS_BRANCH" != "master" ]]; then + # Replace / in branch name to - so it is legal tag name + normalized_branch_name=$(echo "$TRAVIS_BRANCH" | sed -e "s/\//-/") + docker tag rayproject/autoscaler:"$commit_sha" rayproject/autoscaler:"$normalized_branch_name" + docker tag rayproject/ray:"$commit_sha" rayproject/ray:"$normalized_branch_name" + docker tag rayproject/base-deps:"$commit_sha" rayproject/base-deps:"$normalized_branch_name" + docker_push rayproject/autoscaler:"$normalized_branch_name" + docker_push rayproject/ray:"$normalized_branch_name" + docker_push rayproject/base-deps:"$normalized_branch_name" + else + docker tag rayproject/autoscaler:"$commit_sha" rayproject/autoscaler:latest + docker tag rayproject/ray:"$commit_sha" rayproject/ray:latest + docker tag rayproject/base-deps:"$commit_sha" rayproject/base-deps:latest + docker_push rayproject/autoscaler:latest + docker_push rayproject/ray:latest + docker_push rayproject/base-deps:latest + fi +fi + diff --git a/ci/travis/ci.sh b/ci/travis/ci.sh index 7ee787713..42ad824ad 100755 --- a/ci/travis/ci.sh +++ b/ci/travis/ci.sh @@ -287,7 +287,9 @@ build_wheels() { # This command should be kept in sync with ray/python/README-building-wheels.md, # except the "${MOUNT_BAZEL_CACHE[@]}" part. + # TODO(ilr) Re-add "suppress_output" to show info docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" \ + -e TRAVIS_COMMIT="${TRAVIS_COMMIT}" -e CI="${CI}" \ rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh ;; darwin*) diff --git a/ci/travis/determine_tests_to_run.py b/ci/travis/determine_tests_to_run.py index 5b27f4631..c382feee0 100644 --- a/ci/travis/determine_tests_to_run.py +++ b/ci/travis/determine_tests_to_run.py @@ -46,6 +46,7 @@ if __name__ == "__main__": RAY_CI_STREAMING_PYTHON_AFFECTED = 0 RAY_CI_STREAMING_JAVA_AFFECTED = 0 RAY_CI_DASHBOARD_AFFECTED = 0 + RAY_CI_DOCKER_AFFECTED = 0 event_type = None for key in ["GITHUB_EVENT_NAME", "TRAVIS_EVENT_TYPE"]: @@ -102,6 +103,8 @@ if __name__ == "__main__": elif changed_file.startswith("java/"): RAY_CI_JAVA_AFFECTED = 1 RAY_CI_STREAMING_JAVA_AFFECTED = 1 + elif changed_file.startswith("docker/"): + RAY_CI_DOCKER_AFFECTED = 1 elif any( changed_file.startswith(prefix) for prefix in skip_prefix_list): diff --git a/docker/autoscaler/Dockerfile b/docker/autoscaler/Dockerfile index cc58ebc96..ee1fd1726 100644 --- a/docker/autoscaler/Dockerfile +++ b/docker/autoscaler/Dockerfile @@ -1,6 +1,4 @@ -FROM rayproject/base-deps -ARG WHEEL_PATH -ARG WHEEL_NAME +FROM rayproject/ray # Install kubectl. RUN apt-get update \ @@ -13,14 +11,10 @@ RUN apt-get update \ && apt-get clean # We have to uninstall wrapt this way for Tensorflow compatibility -COPY ../../python/requirements.txt ./ -COPY ../../python/requirements_autoscaler.txt ./ -COPY $WHEEL_PATH $WHEEL_NAME +COPY requirements.txt ./ +COPY requirements_autoscaler.txt ./ -RUN pip --no-cache-dir install -r requirements.txt \ - && pip --no-cache-dir install -r requirements_autoscaler.txt \ - && pip --no-cache-dir install $WHEEL_NAME[all] +RUN $HOME/anaconda3/bin/pip --no-cache-dir install -r requirements.txt \ + && $HOME/anaconda3/bin/pip --no-cache-dir install -r requirements_autoscaler.txt \ + && rm requirements.txt && rm requirements_autoscaler.txt -# For Click -ENV LC_ALL=C.UTF-8 -ENV LANG=C.UTF-8 diff --git a/docker/base-deps/Dockerfile b/docker/base-deps/Dockerfile index 2947a7ae4..9a8a4232d 100644 --- a/docker/base-deps/Dockerfile +++ b/docker/base-deps/Dockerfile @@ -1,6 +1,8 @@ # The base-deps Docker image installs main libraries needed to run Ray FROM ubuntu:focal +ENV TZ=America/Los_Angeles +ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update -y && apt-get upgrade -y \ && apt-get install -y \ git \ @@ -9,13 +11,11 @@ RUN apt-get update -y && apt-get upgrade -y \ build-essential \ curl \ unzip \ - libgtk2.0-dev \ - zlib1g-dev \ - libgl1-mesa-dev \ tmux \ screen \ rsync \ openssh-client \ + && rm -rf /var/lib/apt/lists/* \ && apt-get clean \ && wget \ --quiet "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" \ @@ -35,4 +35,5 @@ RUN apt-get update -y && apt-get upgrade -y \ # AttributeError: 'numpy.ufunc' object has no attribute '__module__' && $HOME/anaconda3/bin/pip uninstall -y dask -ENV PATH "$HOME/anaconda3/bin:$PATH" +# TODO(ilr) $HOME seems to point to result in "" instead of "/root" +ENV PATH "/root/anaconda3/bin:$PATH" diff --git a/docker/examples/Dockerfile b/docker/examples/Dockerfile index a3ec9e04b..50751ce9d 100644 --- a/docker/examples/Dockerfile +++ b/docker/examples/Dockerfile @@ -5,7 +5,7 @@ FROM ray-project/deploy # This updates numpy to 1.14 and mutes errors from other libraries RUN conda install -y numpy # Needed to run Tune example with a 'plot' call - which does not actually render a plot, but throws an error. -RUN apt-get install -y zlib1g-dev libgl1-mesa-dev +RUN apt-get install -y zlib1g-dev libgl1-mesa-dev libgtk2.0-dev RUN pip install -U pip RUN pip install gym[atari] opencv-python-headless tensorflow lz4 pytest-timeout smart_open tensorflow_probability dm_tree RUN pip install -U h5py # Mutes FutureWarnings diff --git a/docker/ray/Dockerfile b/docker/ray/Dockerfile new file mode 100644 index 000000000..73524f644 --- /dev/null +++ b/docker/ray/Dockerfile @@ -0,0 +1,7 @@ +FROM rayproject/base-deps +ARG WHEEL_PATH +# For Click +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 +COPY $WHEEL_PATH . +RUN $HOME/anaconda3/bin/pip --no-cache-dir install `basename $WHEEL_PATH`[all]