From 5d4d67c47d7b28b5f4eec3cdab7685ee46ca8668 Mon Sep 17 00:00:00 2001 From: Ian Rodney Date: Mon, 31 Aug 2020 10:57:17 -0700 Subject: [PATCH] [docker] Mirror Functionality of CI scripts & Fix docs (#10349) * first-pass * add back build examples * remove unnecessary test * add gcc and more formatting * doc fixing * small fixes --- build-docker.sh | 92 +++++++++++++++-------- doc/source/installation.rst | 31 ++++---- docker/{deploy => development}/Dockerfile | 8 +- docker/examples/Dockerfile | 45 +++++++---- docker/stress_test/Dockerfile | 13 ---- 5 files changed, 116 insertions(+), 73 deletions(-) rename docker/{deploy => development}/Dockerfile (66%) delete mode 100644 docker/stress_test/Dockerfile diff --git a/build-docker.sh b/build-docker.sh index 4e3a44d93..cffc25fbc 100755 --- a/build-docker.sh +++ b/build-docker.sh @@ -1,56 +1,86 @@ #!/bin/bash +# This script is for users to build docker images locally. It is most useful for users wishing to edit the +# base-deps, ray-deps, or ray images. This script is *not* tested, so please look at the +# scripts/build-docker-images.sh if there are problems with using this script. + set -x +GPU="" +BASE_IMAGE="ubuntu:focal" +WHEEL_URL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp37-cp37m-manylinux1_x86_64.whl" + while [[ $# -gt 0 ]] do key="$1" case $key in - --no-cache) + --gpu) + GPU="-gpu" + BASE_IMAGE="nvidia/cuda:11.0-cudnn8-runtime-ubuntu18.04" + ;; + --no-cache-build) NO_CACHE="--no-cache" ;; - --skip-examples) - SKIP_EXAMPLES=YES + --build-development-image) + BUILD_DEV=YES ;; - --output-sha) - # output the SHA sum of the last built file (either ray-project/deploy - # or ray-project/examples, suppressing all other output. This is useful - # for scripting tests, especially when builds of different versions - # are running on the same machine. It also can facilitate cleanup. + --build-examples) + BUILD_EXAMPLES=YES + ;; + --shas-only) + # output the SHA sum of each build. This is useful for scripting tests, + # especially when builds of different versions are running on the same machine. + # It also can facilitate cleanup. OUTPUT_SHA=YES ;; + --wheel-to-use) + # Which wheel to use. This defaults to the latest nightly on python 3.7 + echo "not implemented, just hardcode me :'(" + exit 1 + ;; *) - echo "Usage: build-docker.sh [ --no-cache ] [ --skip-examples ] [ --sha-sums ]" + echo "Usage: build-docker.sh [ --no-cache-build ] [ --shas-only ] [ --build-development-image ] [ --build-examples ] [ --wheel-to-use ]" exit 1 esac shift done -# Build base dependencies, allow caching -if [ $OUTPUT_SHA ]; then - IMAGE_SHA=$(docker build $NO_CACHE -q -t ray-project/base-deps docker/base-deps) -else - docker build $NO_CACHE -t ray-project/base-deps docker/base-deps -fi +WHEEL_DIR=$(mktemp -d) +wget --quiet "$WHEEL_URL" -P "$WHEEL_DIR" +WHEEL="$WHEEL_DIR/$(basename "$WHEEL_DIR"/*.whl)" +# Build base-deps, ray-deps, and ray. +for IMAGE in "base-deps" "ray-deps" "ray" +do + cp "$WHEEL" "docker/$IMAGE/$(basename "$WHEEL")" + if [ $OUTPUT_SHA ]; then + IMAGE_SHA=$(docker build $NO_CACHE --build-arg GPU="$GPU" --build-arg BASE_IMAGE="$BASE_IMAGE" --build-arg WHEEL_PATH="$(basename "$WHEEL")" -q -t rayproject/$IMAGE:latest docker/$IMAGE) + echo "rayproject/$IMAGE:latest SHA:$IMAGE_SHA" + else + docker build $NO_CACHE --build-arg GPU="$GPU" --build-arg BASE_IMAGE="$BASE_IMAGE" --build-arg WHEEL_PATH="$(basename "$WHEEL")" -t rayproject/$IMAGE:latest docker/$IMAGE + fi + rm "docker/$IMAGE/$(basename "$WHEEL")" +done + # Build the current Ray source -git rev-parse HEAD > ./docker/deploy/git-rev -git archive -o ./docker/deploy/ray.tar "$(git rev-parse HEAD)" -if [ $OUTPUT_SHA ]; then - IMAGE_SHA=$(docker build --no-cache -q -t ray-project/deploy docker/deploy) -else - docker build --no-cache -t ray-project/deploy docker/deploy -fi -rm ./docker/deploy/ray.tar ./docker/deploy/git-rev - -# Build the examples, unless skipped -if [ ! $SKIP_EXAMPLES ]; then +if [ $BUILD_DEV ]; then + git rev-parse HEAD > ./docker/development/git-rev + git archive -o ./docker/development/ray.tar "$(git rev-parse HEAD)" if [ $OUTPUT_SHA ]; then - IMAGE_SHA=$(docker build $NO_CACHE -q -t ray-project/examples docker/examples) + IMAGE_SHA=$(docker build --no-cache -q -t rayproject/development docker/development) + echo "rayproject/development:latest SHA:$IMAGE_SHA" else - docker build --no-cache -t ray-project/examples docker/examples + docker build --no-cache -t rayproject/development docker/development + fi + rm ./docker/development/ray.tar ./docker/development/git-rev +fi + +if [ $BUILD_EXAMPLES ]; then + if [ $OUTPUT_SHA ]; then + IMAGE_SHA=$(docker build $NO_CACHE -q -t rayproject/examples docker/examples) + echo "rayproject/examples:latest SHA:$IMAGE_SHA" + else + docker build $NO_CACHE -t rayproject/examples docker/examples fi fi -if [ $OUTPUT_SHA ]; then - echo "${IMAGE_SHA/sha256:/}" -fi +rm -rf "$WHEEL_DIR" \ No newline at end of file diff --git a/doc/source/installation.rst b/doc/source/installation.rst index dc80e3579..59e656e79 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -171,21 +171,26 @@ However, should you need to build from source, follow :ref:`these instructions f Docker Source Images -------------------- -Run the script to create Docker images. +Most users should pull a Docker image from the Ray Docker Hub. + +- The ``rayproject/ray`` image has ray and all required dependencies. It comes with anaconda and Python 3.7. +- The ``rayproject/autoscaler`` image has the above features as well as many additional libraries. +- The ``rayproject/base-deps`` and ``rayproject/ray-deps`` are for the linux and python dependencies respectively. + +These images are tagged by their release number (or commit hash for nightlies) as well as a ``"-gpu"`` if they are GPU compatible. + + +If you want to tweak some aspect of these images and build them locally, refer to the following script: .. code-block:: bash cd ray ./build-docker.sh -This script creates several Docker images: +Beyond creating the above Docker images, this script can also produce the following two images. -- The ``ray-project/deploy`` image is a self-contained copy of code and binaries - suitable for end users. -- The ``ray-project/examples`` adds additional libraries for running examples. -- The ``ray-project/base-deps`` image builds from Ubuntu Xenial and includes - Anaconda and other basic dependencies and can serve as a starting point for - developers. +- The ``rayproject/development`` image has the ray source code included and is setup for development. +- The ``rayproject/examples`` image adds additional libraries for running examples. Review images by listing them: @@ -198,10 +203,10 @@ Output should look something like the following: .. code-block:: bash REPOSITORY TAG IMAGE ID CREATED SIZE - ray-project/examples latest 7584bde65894 4 days ago 3.257 GB - ray-project/deploy latest 970966166c71 4 days ago 2.899 GB - ray-project/base-deps latest f45d66963151 4 days ago 2.649 GB - ubuntu xenial f49eec89601e 3 weeks ago 129.5 MB + rayproject/ray latest 7243a11ac068 2 days ago 1.11 GB + rayproject/ray-deps latest b6b39d979d73 8 days ago 996 MB + rayproject/base-deps latest 5606591eeab9 8 days ago 512 MB + ubuntu focal 1e4467b07108 3 weeks ago 73.9 MB Launch Ray in Docker @@ -211,7 +216,7 @@ Start out by launching the deployment container. .. code-block:: bash - docker run --shm-size= -t -i ray-project/deploy + docker run --shm-size= -t -i ray-project/ray Replace ```` with a limit appropriate for your system, for example ``512M`` or ``2G``. The ``-t`` and ``-i`` options here are required to support diff --git a/docker/deploy/Dockerfile b/docker/development/Dockerfile similarity index 66% rename from docker/deploy/Dockerfile rename to docker/development/Dockerfile index fc0c510e7..571dc283b 100644 --- a/docker/deploy/Dockerfile +++ b/docker/development/Dockerfile @@ -1,9 +1,11 @@ -# The deploy Docker image build a self-contained Ray instance suitable -# for end users. +# The development Docker image build a self-contained Ray instance suitable +# for developers that need the source code to actively modify. -FROM ray-project/base-deps +FROM rayproject/ray-deps:latest ADD ray.tar /ray ADD git-rev /ray/git-rev +# Install dependencies needed to build ray +RUN apt-get update && apt-get install -y curl unzip cmake gcc && apt-get clean RUN cd /ray && git init && ./ci/travis/install-bazel.sh --system ENV PATH=$PATH:/root/bin RUN echo 'build --remote_cache="https://storage.googleapis.com/ray-bazel-cache"' >> $HOME/.bazelrc diff --git a/docker/examples/Dockerfile b/docker/examples/Dockerfile index 174ad0956..a80b6d578 100644 --- a/docker/examples/Dockerfile +++ b/docker/examples/Dockerfile @@ -1,17 +1,36 @@ # The examples Docker image adds dependencies needed to run the examples -FROM ray-project/deploy +FROM rayproject/ray:latest -# This updates numpy to 1.14 and mutes errors from other libraries -RUN conda install -y numpy # Needed to run Tune example with a 'plot' call - which does not actually render a plot, but throws an error. -RUN apt-get install -y zlib1g-dev libgl1-mesa-dev libgtk2.0-dev -RUN pip install -U pip -RUN pip install gym[atari] opencv-python-headless==4.3.0.36 tensorflow lz4 pytest-timeout smart_open tensorflow_probability dm_tree -RUN pip install -U h5py # Mutes FutureWarnings -RUN pip install --upgrade bayesian-optimization hyperopt -RUN pip install ConfigSpace==0.4.10 -RUN pip install --upgrade sigopt nevergrad scikit-optimize hpbandster lightgbm xgboost torch torchvision tensorboardX dragonfly-opt zoopt -RUN pip install -U tabulate mlflow -RUN pip install -U pytest-remotedata>=0.3.1 -RUN pip install -U matplotlib jupyter pandas +RUN apt-get update && install -y zlib1g-dev libgl1-mesa-dev libgtk2.0-dev && apt-get clean +RUN pip install --no-cache-dir -U pip \ + gym[atari] \ + opencv-python-headless==4.3.0.36 \ + tensorflow \ + lz4 \ + pytest-timeout \ + smart_open \ + tensorflow_probability \ + dm_tree \ + h5py # Mutes FutureWarnings \ + bayesian-optimization \ + hyperopt \ + ConfigSpace==0.4.10 \ + sigopt \ + nevergrad \ + scikit-optimize \ + hpbandster \ + lightgbm \ + xgboost \ + torch \ + torchvision \ + tensorboardX \ + dragonfly-opt \ + zoopt \ + tabulate \ + mlflow \ + pytest-remotedata>=0.3.1 \ + matplotlib \ + jupyter \ + pandas diff --git a/docker/stress_test/Dockerfile b/docker/stress_test/Dockerfile deleted file mode 100644 index 80ae5d4c3..000000000 --- a/docker/stress_test/Dockerfile +++ /dev/null @@ -1,13 +0,0 @@ -# The stress_test Docker image build a self-contained Ray instance for launching Ray. - -FROM ray-project/base-deps - -# We install ray and boto3 to enable the ray autoscaler as -# a test runner. -RUN pip install -U https://ray-wheels.s3-us-west-2.amazonaws.com/latest/ray-0.9.0.dev0-cp36-cp36m-manylinux1_x86_64.whl boto3 -RUN mkdir -p /root/.ssh/ - -# We port the source code in so that we run the most up-to-date stress tests. -ADD ray.tar /ray -ADD git-rev /ray/git-rev -WORKDIR /ray