[WIP] [docker] Cleanup Docker Base-Deps (#9988)

* cleanup-base deps

* only build base-deps a bit

* remove parens

* formatting

* add ray-deps

* gpu enabled

* always include wheel

* fix script

* log new variables

* run tests for docker

* try to include env variables

* source files

* remove bash when sourcing

* add new lines

* use wget

* dual build autoscaler

* switch to gnupg

* add gcc cmake

* remove blist

* clarify build-docker-images

.travis.yml

@@ -126,7 +126,7 @@ matrix:
         - . ./ci/travis/ci.sh build
       script:
         - . ./ci/travis/ci.sh test_wheels
-        - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then ./ci/keep_alive bash $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.sh; fi
+        - if [[ "$TRAVIS_PULL_REQUEST" != "false" ]]; then . ci/keep_alive . $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.sh; fi
         - bash ./java/build-jar-multiplatform.sh linux
         - bash ./java/build-jar-multiplatform.sh multiplatform
         #- bash ./java/build-jar-multiplatform.sh deploy
@@ -434,7 +434,7 @@ deploy:
 
   - provider: script
     edge: true # This supposedly opts in to deploy v2.
-    script: ./ci/keep_alive bash $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.sh
+    script: . ci/keep_alive . $TRAVIS_BUILD_DIR/ci/travis/build-docker-images.sh
     skip_cleanup: true
     on:
       repo: ray-project/ray

ci/travis/build-docker-images.sh

@@ -1,13 +1,14 @@
 #!/usr/bin/env bash
 
-# This script build docker images for autoscaler.
-# For now, we only build python3.6 images.
+# This script builds docker images on CI.
+# This script MUST be sourced to capture env variables
 set -e
 set -x
 
 SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
 ROOT_DIR=$(cd "$SCRIPT_DIR"/../../; pwd)
 DOCKER_USERNAME="raytravisbot"
+WHEEL="$(basename "$ROOT_DIR"/.whl/*cp37m-manylinux*)"
 
 docker_push() {
     if [[ "$TRAVIS_PULL_REQUEST" == "false" ]]; then
@@ -16,6 +17,45 @@ docker_push() {
         echo "Skipping docker push because it's in PR environment."
     fi
 }
+build_and_push_tags() {
+    # $1 image-name, also used as the directory where the Dockerfile lives (e.g. base-deps)
+    # $2 tag for image (e.g. hahs of commit)
+    for GPU in "" "-gpu" 
+    do 
+        BASE_IMAGE=$(if [ "$GPU" ]; then echo "nvidia/cuda:11.0-cudnn8-runtime-ubuntu18.04"; else echo "ubuntu:focal"; fi;)
+        FULL_NAME_WITH_TAG="rayproject/$1:$2$GPU"
+        LATEST_FULL_NAME_WITH_TAG="rayproject/$1:latest$GPU"
+        docker build --no-cache --build-arg GPU="$GPU" --build-arg BASE_IMAGE="$BASE_IMAGE" --build-arg WHEEL_PATH=".whl/$WHEEL" -t "$FULL_NAME_WITH_TAG" /"$ROOT_DIR"/docker/"$1"
+        
+        docker tag "$FULL_NAME_WITH_TAG" "$LATEST_FULL_NAME_WITH_TAG"
+
+        docker_push "$FULL_NAME_WITH_TAG"
+        docker_push "$LATEST_FULL_NAME_WITH_TAG"
+    done
+}
+
+build_or_pull_base_images() {
+    docker pull rayproject/base-deps:latest
+    TAG=$(date +%F_%H-00)
+    
+    age=$(docker inspect -f '{{ .Created }}' rayproject/base-deps:latest)
+    # Build if older than 2 weeks, files have been edited in this PR OR branch release
+    if [[  $(date -d "-14 days" +%F) > $(date -d "$age" +%F) || \
+        "$RAY_CI_DOCKER_AFFECTED" == "1" || \
+        "$RAY_CI_PYTHON_DEPENDENCIES_AFFECTED" == "1" || \
+         "$TRAVIS_BRANCH" != "master"
+        ]]; then
+        cp -r "$ROOT_DIR"/.whl "$ROOT_DIR"/docker/ray-deps/.whl
+        for IMAGE in "base-deps" "ray-deps"
+        do
+            build_and_push_tags "$IMAGE" "$TAG"
+        done
+
+    else
+        echo "Just pulling images"
+    fi
+
+}
 
 # We will only build and push when we are building branch build.
 if [[ "$TRAVIS" == "true" ]]; then
@@ -29,47 +69,32 @@ if [[ "$TRAVIS" == "true" ]]; then
         fi
     fi
 
-    wheel="$(basename "$ROOT_DIR"/.whl/*cp37m-manylinux*)"
     commit_sha=$(echo "$TRAVIS_COMMIT" | head -c 6)
     cp -r "$ROOT_DIR"/.whl "$ROOT_DIR"/docker/ray/.whl
     cp "$ROOT_DIR"/python/requirements.txt "$ROOT_DIR"/docker/autoscaler/requirements.txt
     cp "$ROOT_DIR"/python/requirements_autoscaler.txt "$ROOT_DIR"/docker/autoscaler/requirements_autoscaler.txt
 
-    docker build -t rayproject/base-deps docker/base-deps
+    build_or_pull_base_images
 
-    docker build \
-        --build-arg WHEEL_PATH=".whl/$wheel" \
-        -t rayproject/ray \
-        "$ROOT_DIR"/docker/ray
 
-    docker build \
-        -t rayproject/autoscaler:"$commit_sha" \
-        "$ROOT_DIR"/docker/autoscaler
-
-    docker tag rayproject/base-deps rayproject/base-deps:"$commit_sha" 
-    docker tag rayproject/ray rayproject/ray:"$commit_sha" 
-    docker_push rayproject/base-deps:"$commit_sha"
-    docker_push rayproject/ray:"$commit_sha"
-    docker_push rayproject/autoscaler:"$commit_sha"
+    build_and_push_tags "ray" "$commit_sha"
 
+    build_and_push_tags "autoscaler" "$commit_sha"
+ 
 
     # We have a branch build, e.g. release/v0.7.0
     if [[ "$TRAVIS_BRANCH" != "master" ]]; then
        # Replace / in branch name to - so it is legal tag name
        normalized_branch_name=$(echo "$TRAVIS_BRANCH" | sed -e "s/\//-/")
-       docker tag rayproject/autoscaler:"$commit_sha" rayproject/autoscaler:"$normalized_branch_name"
-       docker tag rayproject/ray:"$commit_sha" rayproject/ray:"$normalized_branch_name"
-       docker tag rayproject/base-deps:"$commit_sha" rayproject/base-deps:"$normalized_branch_name"
-       docker_push rayproject/autoscaler:"$normalized_branch_name"
-       docker_push rayproject/ray:"$normalized_branch_name"
-       docker_push rayproject/base-deps:"$normalized_branch_name"
-    else
-       docker tag rayproject/autoscaler:"$commit_sha" rayproject/autoscaler:latest
-       docker tag rayproject/ray:"$commit_sha" rayproject/ray:latest
-       docker tag rayproject/base-deps:"$commit_sha" rayproject/base-deps:latest
-       docker_push rayproject/autoscaler:latest
-       docker_push rayproject/ray:latest
-       docker_push rayproject/base-deps:latest
+
+       for IMAGE in "base-deps" "ray-deps" "ray" "autoscaler"
+       do
+            for GPU in "" "-gpu"
+            do
+                docker tag "rayproject/$IMAGE:latest$GPU" "rayproject/$IMAGE:$normalized_branch_name$GPU"
+                docker_push  "rayproject/$IMAGE:$normalized_branch_name$GPU"
+            done
+       done 
     fi
 fi
 

ci/travis/determine_tests_to_run.py

@@ -48,6 +48,7 @@ if __name__ == "__main__":
     RAY_CI_DASHBOARD_AFFECTED = 0
     RAY_CI_DOCKER_AFFECTED = 0
     RAY_CI_DOC_AFFECTED = 0
+    RAY_CI_PYTHON_DEPENDENCIES_AFFECTED = 0
 
     event_type = None
     for key in ["GITHUB_EVENT_NAME", "TRAVIS_EVENT_TYPE"]:
@@ -66,7 +67,7 @@ if __name__ == "__main__":
         print(pformat(files), file=sys.stderr)
 
         skip_prefix_list = [
-            "doc/", "examples/", "dev/", "docker/", "kubernetes/", "site/"
+            "doc/", "examples/", "dev/", "kubernetes/", "site/"
         ]
 
         for changed_file in files:
@@ -105,6 +106,8 @@ if __name__ == "__main__":
                 RAY_CI_MACOS_WHEELS_AFFECTED = 1
                 RAY_CI_STREAMING_PYTHON_AFFECTED = 1
                 RAY_CI_DOC_AFFECTED = 1
+                if changed_file.startswith("python/setup.py"):
+                    RAY_CI_PYTHON_DEPENDENCIES_AFFECTED = 1
             elif changed_file.startswith("java/"):
                 RAY_CI_JAVA_AFFECTED = 1
                 RAY_CI_STREAMING_JAVA_AFFECTED = 1
@@ -196,4 +199,7 @@ if __name__ == "__main__":
             RAY_CI_STREAMING_PYTHON_AFFECTED),
         "RAY_CI_STREAMING_JAVA_AFFECTED={}".format(
             RAY_CI_STREAMING_JAVA_AFFECTED),
+        "RAY_CI_DOCKER_AFFECTED={}".format(RAY_CI_DOCKER_AFFECTED),
+        "RAY_CI_PYTHON_DEPENDENCIES_AFFECTED={}".format(
+            RAY_CI_PYTHON_DEPENDENCIES_AFFECTED),
     ]))

docker/autoscaler/Dockerfile

@@ -1,9 +1,10 @@
-FROM rayproject/ray
+ARG GPU
+FROM rayproject/ray:latest"$GPU"
 
 # Install kubectl.
 RUN apt-get update \
-    && apt-get install -y apt-transport-https \
-    && curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
+    && apt-get install -y gnupg gcc cmake \
+    && wget -O - -q https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \
     && touch /etc/apt/sources.list.d/kubernetes.list \
     && echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/apt/sources.list.d/kubernetes.list \
     && apt-get update \

docker/base-deps/Dockerfile

@@ -1,22 +1,21 @@
 # The base-deps Docker image installs main libraries needed to run Ray
 
-FROM ubuntu:focal
+# The GPU option is nvidia/cuda:11.0-cudnn8-runtime-ubuntu18.04
+ARG BASE_IMAGE="ubuntu:focal"
+FROM ${BASE_IMAGE}
 ENV TZ=America/Los_Angeles
+# TODO(ilr) $HOME seems to point to result in "" instead of "/root"
+ENV PATH "/root/anaconda3/bin:$PATH"
 ARG DEBIAN_FRONTEND=noninteractive
 RUN apt-get update -y && apt-get upgrade -y \
     && apt-get install -y \
         git \
         wget \
         cmake \
-        build-essential \
-        curl \
-        unzip \
         tmux \
         screen \
         rsync \
         openssh-client \
-    && rm -rf /var/lib/apt/lists/* \
-    && apt-get clean \
     && wget \
         --quiet "https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" \
         -O /tmp/miniconda.sh \
@@ -31,9 +30,11 @@ RUN apt-get update -y && apt-get upgrade -y \
         flatbuffers \
         cython==0.29.0 \
         numpy==1.15.4 \
+        psutil \
     # To avoid the following error on Jenkins:
     # AttributeError: 'numpy.ufunc' object has no attribute '__module__'
-    && $HOME/anaconda3/bin/pip uninstall -y dask
-
-# TODO(ilr) $HOME seems to point to result in "" instead of "/root"
-ENV PATH "/root/anaconda3/bin:$PATH"
+    && $HOME/anaconda3/bin/pip uninstall -y dask \ 
+    # We install cmake temporarily to get psutil
+    && apt-get autoremove cmake -y \
+    && rm -rf /var/lib/apt/lists/* \
+    && apt-get clean 

docker/ray-deps/Dockerfile

@@ -0,0 +1,9 @@
+ARG GPU=""
+FROM rayproject/base-deps:latest"$GPU"
+ARG WHEEL_PATH
+# For Click
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+COPY $WHEEL_PATH .
+RUN $HOME/anaconda3/bin/pip --no-cache-dir install `basename $WHEEL_PATH`[all] && \
+    $HOME/anaconda3/bin/pip uninstall ray -y && rm `basename $WHEEL_PATH`

docker/ray/Dockerfile

@@ -1,7 +1,9 @@
-FROM rayproject/base-deps
+ARG GPU
+FROM rayproject/ray-deps:latest"$GPU"
 ARG WHEEL_PATH
 # For Click
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 COPY $WHEEL_PATH .
-RUN $HOME/anaconda3/bin/pip --no-cache-dir install `basename $WHEEL_PATH`[all]
+RUN $HOME/anaconda3/bin/pip --no-cache-dir install `basename $WHEEL_PATH`[all] \
+&& rm `basename $WHEEL_PATH`