mirror of
https://github.com/wassname/ray.git
synced 2026-06-27 18:44:07 +08:00
[docker] Mirror Functionality of CI scripts & Fix docs (#10349)
* first-pass * add back build examples * remove unnecessary test * add gcc and more formatting * doc fixing * small fixes
This commit is contained in:
Ian Rodney
+61
-31
@@ -1,56 +1,86 @@
|
||||
#!/bin/bash
|
||||
# This script is for users to build docker images locally. It is most useful for users wishing to edit the
|
||||
# base-deps, ray-deps, or ray images. This script is *not* tested, so please look at the
|
||||
# scripts/build-docker-images.sh if there are problems with using this script.
|
||||
|
||||
set -x
|
||||
|
||||
GPU=""
|
||||
BASE_IMAGE="ubuntu:focal"
|
||||
WHEEL_URL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-0.9.0.dev0-cp37-cp37m-manylinux1_x86_64.whl"
|
||||
|
||||
while [[ $# -gt 0 ]]
|
||||
do
|
||||
key="$1"
|
||||
case $key in
|
||||
--no-cache)
|
||||
--gpu)
|
||||
GPU="-gpu"
|
||||
BASE_IMAGE="nvidia/cuda:11.0-cudnn8-runtime-ubuntu18.04"
|
||||
;;
|
||||
--no-cache-build)
|
||||
NO_CACHE="--no-cache"
|
||||
;;
|
||||
--skip-examples)
|
||||
SKIP_EXAMPLES=YES
|
||||
--build-development-image)
|
||||
BUILD_DEV=YES
|
||||
;;
|
||||
--output-sha)
|
||||
# output the SHA sum of the last built file (either ray-project/deploy
|
||||
# or ray-project/examples, suppressing all other output. This is useful
|
||||
# for scripting tests, especially when builds of different versions
|
||||
# are running on the same machine. It also can facilitate cleanup.
|
||||
--build-examples)
|
||||
BUILD_EXAMPLES=YES
|
||||
;;
|
||||
--shas-only)
|
||||
# output the SHA sum of each build. This is useful for scripting tests,
|
||||
# especially when builds of different versions are running on the same machine.
|
||||
# It also can facilitate cleanup.
|
||||
OUTPUT_SHA=YES
|
||||
;;
|
||||
--wheel-to-use)
|
||||
# Which wheel to use. This defaults to the latest nightly on python 3.7
|
||||
echo "not implemented, just hardcode me :'("
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
echo "Usage: build-docker.sh [ --no-cache ] [ --skip-examples ] [ --sha-sums ]"
|
||||
echo "Usage: build-docker.sh [ --no-cache-build ] [ --shas-only ] [ --build-development-image ] [ --build-examples ] [ --wheel-to-use ]"
|
||||
exit 1
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
# Build base dependencies, allow caching
|
||||
if [ $OUTPUT_SHA ]; then
|
||||
IMAGE_SHA=$(docker build $NO_CACHE -q -t ray-project/base-deps docker/base-deps)
|
||||
else
|
||||
docker build $NO_CACHE -t ray-project/base-deps docker/base-deps
|
||||
fi
|
||||
WHEEL_DIR=$(mktemp -d)
|
||||
wget --quiet "$WHEEL_URL" -P "$WHEEL_DIR"
|
||||
WHEEL="$WHEEL_DIR/$(basename "$WHEEL_DIR"/*.whl)"
|
||||
# Build base-deps, ray-deps, and ray.
|
||||
for IMAGE in "base-deps" "ray-deps" "ray"
|
||||
do
|
||||
cp "$WHEEL" "docker/$IMAGE/$(basename "$WHEEL")"
|
||||
if [ $OUTPUT_SHA ]; then
|
||||
IMAGE_SHA=$(docker build $NO_CACHE --build-arg GPU="$GPU" --build-arg BASE_IMAGE="$BASE_IMAGE" --build-arg WHEEL_PATH="$(basename "$WHEEL")" -q -t rayproject/$IMAGE:latest docker/$IMAGE)
|
||||
echo "rayproject/$IMAGE:latest SHA:$IMAGE_SHA"
|
||||
else
|
||||
docker build $NO_CACHE --build-arg GPU="$GPU" --build-arg BASE_IMAGE="$BASE_IMAGE" --build-arg WHEEL_PATH="$(basename "$WHEEL")" -t rayproject/$IMAGE:latest docker/$IMAGE
|
||||
fi
|
||||
rm "docker/$IMAGE/$(basename "$WHEEL")"
|
||||
done
|
||||
|
||||
|
||||
# Build the current Ray source
|
||||
git rev-parse HEAD > ./docker/deploy/git-rev
|
||||
git archive -o ./docker/deploy/ray.tar "$(git rev-parse HEAD)"
|
||||
if [ $OUTPUT_SHA ]; then
|
||||
IMAGE_SHA=$(docker build --no-cache -q -t ray-project/deploy docker/deploy)
|
||||
else
|
||||
docker build --no-cache -t ray-project/deploy docker/deploy
|
||||
fi
|
||||
rm ./docker/deploy/ray.tar ./docker/deploy/git-rev
|
||||
|
||||
# Build the examples, unless skipped
|
||||
if [ ! $SKIP_EXAMPLES ]; then
|
||||
if [ $BUILD_DEV ]; then
|
||||
git rev-parse HEAD > ./docker/development/git-rev
|
||||
git archive -o ./docker/development/ray.tar "$(git rev-parse HEAD)"
|
||||
if [ $OUTPUT_SHA ]; then
|
||||
IMAGE_SHA=$(docker build $NO_CACHE -q -t ray-project/examples docker/examples)
|
||||
IMAGE_SHA=$(docker build --no-cache -q -t rayproject/development docker/development)
|
||||
echo "rayproject/development:latest SHA:$IMAGE_SHA"
|
||||
else
|
||||
docker build --no-cache -t ray-project/examples docker/examples
|
||||
docker build --no-cache -t rayproject/development docker/development
|
||||
fi
|
||||
rm ./docker/development/ray.tar ./docker/development/git-rev
|
||||
fi
|
||||
|
||||
if [ $BUILD_EXAMPLES ]; then
|
||||
if [ $OUTPUT_SHA ]; then
|
||||
IMAGE_SHA=$(docker build $NO_CACHE -q -t rayproject/examples docker/examples)
|
||||
echo "rayproject/examples:latest SHA:$IMAGE_SHA"
|
||||
else
|
||||
docker build $NO_CACHE -t rayproject/examples docker/examples
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $OUTPUT_SHA ]; then
|
||||
echo "${IMAGE_SHA/sha256:/}"
|
||||
fi
|
||||
rm -rf "$WHEEL_DIR"
|
||||
+18
-13
@@ -171,21 +171,26 @@ However, should you need to build from source, follow :ref:`these instructions f
|
||||
Docker Source Images
|
||||
--------------------
|
||||
|
||||
Run the script to create Docker images.
|
||||
Most users should pull a Docker image from the Ray Docker Hub.
|
||||
|
||||
- The ``rayproject/ray`` image has ray and all required dependencies. It comes with anaconda and Python 3.7.
|
||||
- The ``rayproject/autoscaler`` image has the above features as well as many additional libraries.
|
||||
- The ``rayproject/base-deps`` and ``rayproject/ray-deps`` are for the linux and python dependencies respectively.
|
||||
|
||||
These images are tagged by their release number (or commit hash for nightlies) as well as a ``"-gpu"`` if they are GPU compatible.
|
||||
|
||||
|
||||
If you want to tweak some aspect of these images and build them locally, refer to the following script:
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
cd ray
|
||||
./build-docker.sh
|
||||
|
||||
This script creates several Docker images:
|
||||
Beyond creating the above Docker images, this script can also produce the following two images.
|
||||
|
||||
- The ``ray-project/deploy`` image is a self-contained copy of code and binaries
|
||||
suitable for end users.
|
||||
- The ``ray-project/examples`` adds additional libraries for running examples.
|
||||
- The ``ray-project/base-deps`` image builds from Ubuntu Xenial and includes
|
||||
Anaconda and other basic dependencies and can serve as a starting point for
|
||||
developers.
|
||||
- The ``rayproject/development`` image has the ray source code included and is setup for development.
|
||||
- The ``rayproject/examples`` image adds additional libraries for running examples.
|
||||
|
||||
Review images by listing them:
|
||||
|
||||
@@ -198,10 +203,10 @@ Output should look something like the following:
|
||||
.. code-block:: bash
|
||||
|
||||
REPOSITORY TAG IMAGE ID CREATED SIZE
|
||||
ray-project/examples latest 7584bde65894 4 days ago 3.257 GB
|
||||
ray-project/deploy latest 970966166c71 4 days ago 2.899 GB
|
||||
ray-project/base-deps latest f45d66963151 4 days ago 2.649 GB
|
||||
ubuntu xenial f49eec89601e 3 weeks ago 129.5 MB
|
||||
rayproject/ray latest 7243a11ac068 2 days ago 1.11 GB
|
||||
rayproject/ray-deps latest b6b39d979d73 8 days ago 996 MB
|
||||
rayproject/base-deps latest 5606591eeab9 8 days ago 512 MB
|
||||
ubuntu focal 1e4467b07108 3 weeks ago 73.9 MB
|
||||
|
||||
|
||||
Launch Ray in Docker
|
||||
@@ -211,7 +216,7 @@ Start out by launching the deployment container.
|
||||
|
||||
.. code-block:: bash
|
||||
|
||||
docker run --shm-size=<shm-size> -t -i ray-project/deploy
|
||||
docker run --shm-size=<shm-size> -t -i ray-project/ray
|
||||
|
||||
Replace ``<shm-size>`` with a limit appropriate for your system, for example
|
||||
``512M`` or ``2G``. The ``-t`` and ``-i`` options here are required to support
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
# The deploy Docker image build a self-contained Ray instance suitable
|
||||
# for end users.
|
||||
# The development Docker image build a self-contained Ray instance suitable
|
||||
# for developers that need the source code to actively modify.
|
||||
|
||||
FROM ray-project/base-deps
|
||||
FROM rayproject/ray-deps:latest
|
||||
ADD ray.tar /ray
|
||||
ADD git-rev /ray/git-rev
|
||||
# Install dependencies needed to build ray
|
||||
RUN apt-get update && apt-get install -y curl unzip cmake gcc && apt-get clean
|
||||
RUN cd /ray && git init && ./ci/travis/install-bazel.sh --system
|
||||
ENV PATH=$PATH:/root/bin
|
||||
RUN echo 'build --remote_cache="https://storage.googleapis.com/ray-bazel-cache"' >> $HOME/.bazelrc
|
||||
+32
-13
@@ -1,17 +1,36 @@
|
||||
# The examples Docker image adds dependencies needed to run the examples
|
||||
|
||||
FROM ray-project/deploy
|
||||
FROM rayproject/ray:latest
|
||||
|
||||
# This updates numpy to 1.14 and mutes errors from other libraries
|
||||
RUN conda install -y numpy
|
||||
# Needed to run Tune example with a 'plot' call - which does not actually render a plot, but throws an error.
|
||||
RUN apt-get install -y zlib1g-dev libgl1-mesa-dev libgtk2.0-dev
|
||||
RUN pip install -U pip
|
||||
RUN pip install gym[atari] opencv-python-headless==4.3.0.36 tensorflow lz4 pytest-timeout smart_open tensorflow_probability dm_tree
|
||||
RUN pip install -U h5py # Mutes FutureWarnings
|
||||
RUN pip install --upgrade bayesian-optimization hyperopt
|
||||
RUN pip install ConfigSpace==0.4.10
|
||||
RUN pip install --upgrade sigopt nevergrad scikit-optimize hpbandster lightgbm xgboost torch torchvision tensorboardX dragonfly-opt zoopt
|
||||
RUN pip install -U tabulate mlflow
|
||||
RUN pip install -U pytest-remotedata>=0.3.1
|
||||
RUN pip install -U matplotlib jupyter pandas
|
||||
RUN apt-get update && install -y zlib1g-dev libgl1-mesa-dev libgtk2.0-dev && apt-get clean
|
||||
RUN pip install --no-cache-dir -U pip \
|
||||
gym[atari] \
|
||||
opencv-python-headless==4.3.0.36 \
|
||||
tensorflow \
|
||||
lz4 \
|
||||
pytest-timeout \
|
||||
smart_open \
|
||||
tensorflow_probability \
|
||||
dm_tree \
|
||||
h5py # Mutes FutureWarnings \
|
||||
bayesian-optimization \
|
||||
hyperopt \
|
||||
ConfigSpace==0.4.10 \
|
||||
sigopt \
|
||||
nevergrad \
|
||||
scikit-optimize \
|
||||
hpbandster \
|
||||
lightgbm \
|
||||
xgboost \
|
||||
torch \
|
||||
torchvision \
|
||||
tensorboardX \
|
||||
dragonfly-opt \
|
||||
zoopt \
|
||||
tabulate \
|
||||
mlflow \
|
||||
pytest-remotedata>=0.3.1 \
|
||||
matplotlib \
|
||||
jupyter \
|
||||
pandas
|
||||
|
||||
@@ -1,13 +0,0 @@
|
||||
# The stress_test Docker image build a self-contained Ray instance for launching Ray.
|
||||
|
||||
FROM ray-project/base-deps
|
||||
|
||||
# We install ray and boto3 to enable the ray autoscaler as
|
||||
# a test runner.
|
||||
RUN pip install -U https://ray-wheels.s3-us-west-2.amazonaws.com/latest/ray-0.9.0.dev0-cp36-cp36m-manylinux1_x86_64.whl boto3
|
||||
RUN mkdir -p /root/.ssh/
|
||||
|
||||
# We port the source code in so that we run the most up-to-date stress tests.
|
||||
ADD ray.tar /ray
|
||||
ADD git-rev /ray/git-rev
|
||||
WORKDIR /ray
|
||||
Reference in New Issue
Block a user