wassname/ray
1
0
Fork 0
mirror of https://github.com/wassname/ray.git synced 2026-09-09 11:32:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

[docker] Build docker in Travis PR & Fix image build failing (#9787)

Browse Source 
Co-authored-by: Ian Rodney <ian.rodney@gmail.com>
This commit is contained in:
Simon Mo
2020-08-03 16:37:15 -07:00
committed by GitHub
co-authored by Ian Rodney
parent edcd94f26b
commit c218f2eff6
9 changed files with 101 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ci/travis/build-autoscaler-images.sh
-48
View File
@@ -1,48 +0,0 @@
#!/usr/bin/env bash
# This script build docker images for autoscaler.
# For now, we only build python3.6 images.
set -e
set -x
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
ROOT_DIR=$(cd "$SCRIPT_DIR"/../../; pwd)
DOCKER_USERNAME="raytravisbot"
# We will only build and push when we are building branch build.
if [[ "$TRAVIS" == "true" && "$TRAVIS_PULL_REQUEST" == "false" ]]; then
echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
wheel="$(basename "$ROOT_DIR"/.whl/*cp36m-manylinux*)"
commit_sha=$(echo "$TRAVIS_COMMIT" | head -c 6)
cp -r "$ROOT_DIR"/.whl "$ROOT_DIR"/docker/autoscaler/.whl
docker build -q -t rayproject/base-deps docker/base-deps
docker build \
--build-arg WHEEL_PATH=".whl/$wheel" \
--build-arg WHEEL_NAME="$wheel" \
-t rayproject/autoscaler:"$commit_sha" \
"$ROOT_DIR"/docker/autoscaler
docker tag rayproject/base-deps rayproject/base-deps:"$commit_sha"
docker push rayproject/base-deps:"$commit_sha"
docker push rayproject/autoscaler:"$commit_sha"
# We have a branch build, e.g. release/v0.7.0
if [[ "$TRAVIS_BRANCH" != "master" ]]; then
# Replace / in branch name to - so it is legal tag name
normalized_branch_name=$(echo "$TRAVIS_BRANCH" | sed -e "s/\//-/")
docker tag rayproject/autoscaler:"$commit_sha" rayproject/autoscaler:"$normalized_branch_name"
docker tag rayproject/base-deps:"$commit_sha" rayproject/base-deps:"$normalized_branch_name"
docker push rayproject/autoscaler:"$normalized_branch_name"
docker push rayproject/base-deps:"$normalized_branch_name"
else
docker tag rayproject/autoscaler:"$commit_sha" rayproject/autoscaler:latest
docker tag rayproject/base-deps:"$commit_sha" rayproject/base-deps:latest
docker push rayproject/autoscaler:latest
docker push rayproject/base-deps:latest
fi
fi
ci/travis/build-docker-images.sh
Executable
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env bash
# This script build docker images for autoscaler.
# For now, we only build python3.6 images.
set -e
set -x
SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
ROOT_DIR=$(cd "$SCRIPT_DIR"/../../; pwd)
DOCKER_USERNAME="raytravisbot"
docker_push() {
if [[ "$TRAVIS_PULL_REQUEST" == "false" ]]; then
docker push $@
else
echo "Skipping docker push because it's in PR environment."
fi
}
# We will only build and push when we are building branch build.
if [[ "$TRAVIS" == "true" ]]; then
if [[ "$TRAVIS_PULL_REQUEST" == "false" ]]; then
echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin
else
if [[ "$RAY_CI_DOCKER_AFFECTED" == "0" ]]; then
echo "Skipping docker build in PR build because dockerfile didn't change."
exit 0
fi
fi
wheel="$(basename "$ROOT_DIR"/.whl/*cp37m-manylinux*)"
commit_sha=$(echo "$TRAVIS_COMMIT" | head -c 6)
cp -r "$ROOT_DIR"/.whl "$ROOT_DIR"/docker/ray/.whl
cp $ROOT_DIR/python/requirements.txt $ROOT_DIR/docker/autoscaler/requirements.txt
cp $ROOT_DIR/python/requirements_autoscaler.txt $ROOT_DIR/docker/autoscaler/requirements_autoscaler.txt
docker build -t rayproject/base-deps docker/base-deps
docker build \
--build-arg WHEEL_PATH=".whl/$wheel" \
-t rayproject/ray \
"$ROOT_DIR"/docker/ray
docker build \
-t rayproject/autoscaler:"$commit_sha" \
"$ROOT_DIR"/docker/autoscaler
docker tag rayproject/base-deps rayproject/base-deps:"$commit_sha"
docker tag rayproject/ray rayproject/ray:"$commit_sha"
docker_push rayproject/base-deps:"$commit_sha"
docker_push rayproject/ray:"$commit_sha"
docker_push rayproject/autoscaler:"$commit_sha"
# We have a branch build, e.g. release/v0.7.0
if [[ "$TRAVIS_BRANCH" != "master" ]]; then
# Replace / in branch name to - so it is legal tag name
normalized_branch_name=$(echo "$TRAVIS_BRANCH" | sed -e "s/\//-/")
docker tag rayproject/autoscaler:"$commit_sha" rayproject/autoscaler:"$normalized_branch_name"
docker tag rayproject/ray:"$commit_sha" rayproject/ray:"$normalized_branch_name"
docker tag rayproject/base-deps:"$commit_sha" rayproject/base-deps:"$normalized_branch_name"
docker_push rayproject/autoscaler:"$normalized_branch_name"
docker_push rayproject/ray:"$normalized_branch_name"
docker_push rayproject/base-deps:"$normalized_branch_name"
else
docker tag rayproject/autoscaler:"$commit_sha" rayproject/autoscaler:latest
docker tag rayproject/ray:"$commit_sha" rayproject/ray:latest
docker tag rayproject/base-deps:"$commit_sha" rayproject/base-deps:latest
docker_push rayproject/autoscaler:latest
docker_push rayproject/ray:latest
docker_push rayproject/base-deps:latest
fi
fi
ci/travis/ci.sh
+2
View File
@@ -287,7 +287,9 @@ build_wheels() {
# This command should be kept in sync with ray/python/README-building-wheels.md,
# except the "${MOUNT_BAZEL_CACHE[@]}" part.
# TODO(ilr) Re-add "suppress_output" to show info
docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" \
-e TRAVIS_COMMIT="${TRAVIS_COMMIT}" -e CI="${CI}" \
rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh
;;
darwin*)
ci/travis/determine_tests_to_run.py
+3
View File
@@ -46,6 +46,7 @@ if __name__ == "__main__":
RAY_CI_STREAMING_PYTHON_AFFECTED = 0
RAY_CI_STREAMING_JAVA_AFFECTED = 0
RAY_CI_DASHBOARD_AFFECTED = 0
RAY_CI_DOCKER_AFFECTED = 0
event_type = None
for key in ["GITHUB_EVENT_NAME", "TRAVIS_EVENT_TYPE"]:
@@ -102,6 +103,8 @@ if __name__ == "__main__":
elif changed_file.startswith("java/"):
RAY_CI_JAVA_AFFECTED = 1
RAY_CI_STREAMING_JAVA_AFFECTED = 1
elif changed_file.startswith("docker/"):
RAY_CI_DOCKER_AFFECTED = 1
elif any(
changed_file.startswith(prefix)
for prefix in skip_prefix_list):