From 2526c06b5e6d56f67d526ed31a804503ad0d27bf Mon Sep 17 00:00:00 2001 From: Ian Rodney Date: Wed, 26 Aug 2020 19:36:11 -0700 Subject: [PATCH] [WIP] [docker] Cleanup Docker Base-Deps (#9988) * cleanup-base deps * only build base-deps a bit * remove parens * formatting * add ray-deps * gpu enabled * always include wheel * fix script * log new variables * run tests for docker * try to include env variables * source files * remove bash when sourcing * add new lines * use wget * dual build autoscaler * switch to gnupg * add gcc cmake * remove blist * clarify build-docker-images --- .travis.yml | 4 +- ci/travis/build-docker-images.sh | 85 +++++++++++++++++++---------- ci/travis/determine_tests_to_run.py | 8 ++- docker/autoscaler/Dockerfile | 7 ++- docker/base-deps/Dockerfile | 21 +++---- docker/ray-deps/Dockerfile | 9 +++ docker/ray/Dockerfile | 6 +- 7 files changed, 92 insertions(+), 48 deletions(-) create mode 100644 docker/ray-deps/Dockerfile diff --git a/.travis.yml b/.travis.yml index 28b93f025..b0386cb5b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -126,7 +126,7 @@ matrix: - . ./ci/travis/ci.sh build script: - . ./ci/travis/ci.sh test_wheels - - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then ./ci/keep_alive bash $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.sh; fi + - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then . ci/keep_alive . $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.sh; fi - bash ./java/build-jar-multiplatform.sh linux - bash ./java/build-jar-multiplatform.sh multiplatform #- bash ./java/build-jar-multiplatform.sh deploy @@ -434,7 +434,7 @@ deploy: - provider: script edge: true # This supposedly opts in to deploy v2. - script: ./ci/keep_alive bash $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.sh + script: . ci/keep_alive . $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.sh skip_cleanup: true on: repo: ray-project/ray diff --git a/ci/travis/build-docker-images.sh b/ci/travis/build-docker-images.sh index ec19783e3..7ec76b725 100755 --- a/ci/travis/build-docker-images.sh +++ b/ci/travis/build-docker-images.sh @@ -1,13 +1,14 @@ #!/usr/bin/env bash -# This script build docker images for autoscaler. -# For now, we only build python3.6 images. +# This script builds docker images on CI. +# This script MUST be sourced to capture env variables set -e set -x SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) ROOT_DIR=$(cd "$SCRIPT_DIR"/../../; pwd) DOCKER_USERNAME="raytravisbot" +WHEEL="$(basename "$ROOT_DIR"/.whl/*cp37m-manylinux*)" docker_push() { if [[ "$TRAVIS_PULL_REQUEST" == "false" ]]; then @@ -16,6 +17,45 @@ docker_push() { echo "Skipping docker push because it's in PR environment." fi } +build_and_push_tags() { + # $1 image-name, also used as the directory where the Dockerfile lives (e.g. base-deps) + # $2 tag for image (e.g. hahs of commit) + for GPU in "" "-gpu" + do + BASE_IMAGE=$(if [ "$GPU" ]; then echo "nvidia/cuda:11.0-cudnn8-runtime-ubuntu18.04"; else echo "ubuntu:focal"; fi;) + FULL_NAME_WITH_TAG="rayproject/$1:$2$GPU" + LATEST_FULL_NAME_WITH_TAG="rayproject/$1:latest$GPU" + docker build --no-cache --build-arg GPU="$GPU" --build-arg BASE_IMAGE="$BASE_IMAGE" --build-arg WHEEL_PATH=".whl/$WHEEL" -t "$FULL_NAME_WITH_TAG" /"$ROOT_DIR"/docker/"$1" + + docker tag "$FULL_NAME_WITH_TAG" "$LATEST_FULL_NAME_WITH_TAG" + + docker_push "$FULL_NAME_WITH_TAG" + docker_push "$LATEST_FULL_NAME_WITH_TAG" + done +} + +build_or_pull_base_images() { + docker pull rayproject/base-deps:latest + TAG=$(date +%F_%H-00) + + age=$(docker inspect -f '{{ .Created }}' rayproject/base-deps:latest) + # Build if older than 2 weeks, files have been edited in this PR OR branch release + if [[ $(date -d "-14 days" +%F) > $(date -d "$age" +%F) || \ + "$RAY_CI_DOCKER_AFFECTED" == "1" || \ + "$RAY_CI_PYTHON_DEPENDENCIES_AFFECTED" == "1" || \ + "$TRAVIS_BRANCH" != "master" + ]]; then + cp -r "$ROOT_DIR"/.whl "$ROOT_DIR"/docker/ray-deps/.whl + for IMAGE in "base-deps" "ray-deps" + do + build_and_push_tags "$IMAGE" "$TAG" + done + + else + echo "Just pulling images" + fi + +} # We will only build and push when we are building branch build. if [[ "$TRAVIS" == "true" ]]; then @@ -29,47 +69,32 @@ if [[ "$TRAVIS" == "true" ]]; then fi fi - wheel="$(basename "$ROOT_DIR"/.whl/*cp37m-manylinux*)" commit_sha=$(echo "$TRAVIS_COMMIT" | head -c 6) cp -r "$ROOT_DIR"/.whl "$ROOT_DIR"/docker/ray/.whl cp "$ROOT_DIR"/python/requirements.txt "$ROOT_DIR"/docker/autoscaler/requirements.txt cp "$ROOT_DIR"/python/requirements_autoscaler.txt "$ROOT_DIR"/docker/autoscaler/requirements_autoscaler.txt - docker build -t rayproject/base-deps docker/base-deps + build_or_pull_base_images - docker build \ - --build-arg WHEEL_PATH=".whl/$wheel" \ - -t rayproject/ray \ - "$ROOT_DIR"/docker/ray - docker build \ - -t rayproject/autoscaler:"$commit_sha" \ - "$ROOT_DIR"/docker/autoscaler - - docker tag rayproject/base-deps rayproject/base-deps:"$commit_sha" - docker tag rayproject/ray rayproject/ray:"$commit_sha" - docker_push rayproject/base-deps:"$commit_sha" - docker_push rayproject/ray:"$commit_sha" - docker_push rayproject/autoscaler:"$commit_sha" + build_and_push_tags "ray" "$commit_sha" + build_and_push_tags "autoscaler" "$commit_sha" + # We have a branch build, e.g. release/v0.7.0 if [[ "$TRAVIS_BRANCH" != "master" ]]; then # Replace / in branch name to - so it is legal tag name normalized_branch_name=$(echo "$TRAVIS_BRANCH" | sed -e "s/\//-/") - docker tag rayproject/autoscaler:"$commit_sha" rayproject/autoscaler:"$normalized_branch_name" - docker tag rayproject/ray:"$commit_sha" rayproject/ray:"$normalized_branch_name" - docker tag rayproject/base-deps:"$commit_sha" rayproject/base-deps:"$normalized_branch_name" - docker_push rayproject/autoscaler:"$normalized_branch_name" - docker_push rayproject/ray:"$normalized_branch_name" - docker_push rayproject/base-deps:"$normalized_branch_name" - else - docker tag rayproject/autoscaler:"$commit_sha" rayproject/autoscaler:latest - docker tag rayproject/ray:"$commit_sha" rayproject/ray:latest - docker tag rayproject/base-deps:"$commit_sha" rayproject/base-deps:latest - docker_push rayproject/autoscaler:latest - docker_push rayproject/ray:latest - docker_push rayproject/base-deps:latest + + for IMAGE in "base-deps" "ray-deps" "ray" "autoscaler" + do + for GPU in "" "-gpu" + do + docker tag "rayproject/$IMAGE:latest$GPU" "rayproject/$IMAGE:$normalized_branch_name$GPU" + docker_push "rayproject/$IMAGE:$normalized_branch_name$GPU" + done + done fi fi diff --git a/ci/travis/determine_tests_to_run.py b/ci/travis/determine_tests_to_run.py index 93dbf30c3..cfb86609f 100644 --- a/ci/travis/determine_tests_to_run.py +++ b/ci/travis/determine_tests_to_run.py @@ -48,6 +48,7 @@ if __name__ == "__main__": RAY_CI_DASHBOARD_AFFECTED = 0 RAY_CI_DOCKER_AFFECTED = 0 RAY_CI_DOC_AFFECTED = 0 + RAY_CI_PYTHON_DEPENDENCIES_AFFECTED = 0 event_type = None for key in ["GITHUB_EVENT_NAME", "TRAVIS_EVENT_TYPE"]: @@ -66,7 +67,7 @@ if __name__ == "__main__": print(pformat(files), file=sys.stderr) skip_prefix_list = [ - "doc/", "examples/", "dev/", "docker/", "kubernetes/", "site/" + "doc/", "examples/", "dev/", "kubernetes/", "site/" ] for changed_file in files: @@ -105,6 +106,8 @@ if __name__ == "__main__": RAY_CI_MACOS_WHEELS_AFFECTED = 1 RAY_CI_STREAMING_PYTHON_AFFECTED = 1 RAY_CI_DOC_AFFECTED = 1 + if changed_file.startswith("python/setup.py"): + RAY_CI_PYTHON_DEPENDENCIES_AFFECTED = 1 elif changed_file.startswith("java/"): RAY_CI_JAVA_AFFECTED = 1 RAY_CI_STREAMING_JAVA_AFFECTED = 1 @@ -196,4 +199,7 @@ if __name__ == "__main__": RAY_CI_STREAMING_PYTHON_AFFECTED), "RAY_CI_STREAMING_JAVA_AFFECTED={}".format( RAY_CI_STREAMING_JAVA_AFFECTED), + "RAY_CI_DOCKER_AFFECTED={}".format(RAY_CI_DOCKER_AFFECTED), + "RAY_CI_PYTHON_DEPENDENCIES_AFFECTED={}".format( + RAY_CI_PYTHON_DEPENDENCIES_AFFECTED), ])) diff --git a/docker/autoscaler/Dockerfile b/docker/autoscaler/Dockerfile index ee1fd1726..482621fd2 100644 --- a/docker/autoscaler/Dockerfile +++ b/docker/autoscaler/Dockerfile @@ -1,9 +1,10 @@ -FROM rayproject/ray +ARG GPU +FROM rayproject/ray:latest"$GPU" # Install kubectl. RUN apt-get update \ - && apt-get install -y apt-transport-https \ - && curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ + && apt-get install -y gnupg gcc cmake \ + && wget -O - -q https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ && touch /etc/apt/sources.list.d/kubernetes.list \ && echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list \ && apt-get update \ diff --git a/docker/base-deps/Dockerfile b/docker/base-deps/Dockerfile index 9a8a4232d..c3d97f671 100644 --- a/docker/base-deps/Dockerfile +++ b/docker/base-deps/Dockerfile @@ -1,22 +1,21 @@ # The base-deps Docker image installs main libraries needed to run Ray -FROM ubuntu:focal +# The GPU option is nvidia/cuda:11.0-cudnn8-runtime-ubuntu18.04 +ARG BASE_IMAGE="ubuntu:focal" +FROM ${BASE_IMAGE} ENV TZ=America/Los_Angeles +# TODO(ilr) $HOME seems to point to result in "" instead of "/root" +ENV PATH "/root/anaconda3/bin:$PATH" ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update -y && apt-get upgrade -y \ && apt-get install -y \ git \ wget \ cmake \ - build-essential \ - curl \ - unzip \ tmux \ screen \ rsync \ openssh-client \ - && rm -rf /var/lib/apt/lists/* \ - && apt-get clean \ && wget \ --quiet "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" \ -O /tmp/miniconda.sh \ @@ -31,9 +30,11 @@ RUN apt-get update -y && apt-get upgrade -y \ flatbuffers \ cython==0.29.0 \ numpy==1.15.4 \ + psutil \ # To avoid the following error on Jenkins: # AttributeError: 'numpy.ufunc' object has no attribute '__module__' - && $HOME/anaconda3/bin/pip uninstall -y dask - -# TODO(ilr) $HOME seems to point to result in "" instead of "/root" -ENV PATH "/root/anaconda3/bin:$PATH" + && $HOME/anaconda3/bin/pip uninstall -y dask \ + # We install cmake temporarily to get psutil + && apt-get autoremove cmake -y \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get clean diff --git a/docker/ray-deps/Dockerfile b/docker/ray-deps/Dockerfile new file mode 100644 index 000000000..b5e0d1b94 --- /dev/null +++ b/docker/ray-deps/Dockerfile @@ -0,0 +1,9 @@ +ARG GPU="" +FROM rayproject/base-deps:latest"$GPU" +ARG WHEEL_PATH +# For Click +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 +COPY $WHEEL_PATH . +RUN $HOME/anaconda3/bin/pip --no-cache-dir install `basename $WHEEL_PATH`[all] && \ + $HOME/anaconda3/bin/pip uninstall ray -y && rm `basename $WHEEL_PATH` diff --git a/docker/ray/Dockerfile b/docker/ray/Dockerfile index 73524f644..4d89c8956 100644 --- a/docker/ray/Dockerfile +++ b/docker/ray/Dockerfile @@ -1,7 +1,9 @@ -FROM rayproject/base-deps +ARG GPU +FROM rayproject/ray-deps:latest"$GPU" ARG WHEEL_PATH # For Click ENV LC_ALL=C.UTF-8 ENV LANG=C.UTF-8 COPY $WHEEL_PATH . -RUN $HOME/anaconda3/bin/pip --no-cache-dir install `basename $WHEEL_PATH`[all] +RUN $HOME/anaconda3/bin/pip --no-cache-dir install `basename $WHEEL_PATH`[all] \ +&& rm `basename $WHEEL_PATH`