mirror of
https://github.com/wassname/ray.git
synced 2026-07-14 11:17:54 +08:00
Move Jenkins test to Github action (#7342)
This commit is contained in:
Simon Mo
@@ -0,0 +1,26 @@
|
||||
name: Jenkins Integration
|
||||
|
||||
env:
|
||||
DEBIAN_FRONTEND: noninteractive
|
||||
|
||||
on: [push, pull_request]
|
||||
|
||||
jobs:
|
||||
jenkins:
|
||||
name: docker integration
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
test-case: ["RUN_TUNE_TESTS", "RUN_DOC_TESTS", "RUN_SGD_TESTS"]
|
||||
fail-fast: false
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v1
|
||||
with:
|
||||
fetch-depth: 1
|
||||
- name: Run Jenkins Tests
|
||||
shell: bash -e -o pipefail -l {0}
|
||||
env:
|
||||
SIGOPT_KEY: ${{ secrets.TUNE_SIGOPT_KEY }}
|
||||
TEST_CASE: ${{ matrix.test-case }}
|
||||
run: export $TEST_CASE=1 && HOME=$(pwd) ./ci/jenkins_tests/entry_point.sh
|
||||
Executable
+44
@@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Cause the script to exit if a single command fails.
|
||||
set -e
|
||||
|
||||
# Show explicitly which commands are currently running.
|
||||
set -x
|
||||
|
||||
|
||||
MEMORY_SIZE="8G"
|
||||
SHM_SIZE="4G"
|
||||
|
||||
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
|
||||
|
||||
SUPPRESS_OUTPUT=$ROOT_DIR/../suppress_output
|
||||
|
||||
function retry {
|
||||
local n=1
|
||||
local max=3
|
||||
|
||||
while true; do
|
||||
"$@" && break || {
|
||||
if [[ $n -lt $max ]]; then
|
||||
((n++))
|
||||
echo "Command failed. Attempt $n/$max:"
|
||||
else
|
||||
echo "The command has failed after $n attempts."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
done
|
||||
}
|
||||
|
||||
if [[ ! -z "$RUN_TUNE_TESTS" ]]; then
|
||||
retry bash $ROOT_DIR/run_tune_tests.sh ${MEMORY_SIZE} ${SHM_SIZE}
|
||||
fi
|
||||
|
||||
if [[ ! -z "$RUN_DOC_TESTS" ]]; then
|
||||
retry bash $ROOT_DIR/run_doc_tests.sh ${MEMORY_SIZE} ${SHM_SIZE}
|
||||
fi
|
||||
|
||||
if [[ ! -z "$RUN_SGD_TESTS" ]]; then
|
||||
retry bash $ROOT_DIR/run_sgd_tests.sh ${MEMORY_SIZE} ${SHM_SIZE}
|
||||
fi
|
||||
Executable
+51
@@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Cause the script to exit if a single command fails.
|
||||
set -e
|
||||
|
||||
# Show explicitly which commands are currently running.
|
||||
set -x
|
||||
|
||||
MEMORY_SIZE=$1
|
||||
SHM_SIZE=$2
|
||||
DOCKER_SHA=$3
|
||||
|
||||
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
|
||||
SUPPRESS_OUTPUT=$ROOT_DIR/../suppress_output
|
||||
|
||||
if [ "$MEMORY_SIZE" == "" ]; then
|
||||
MEMORY_SIZE="20G"
|
||||
fi
|
||||
if [ "$SHM_SIZE" == "" ]; then
|
||||
SHM_SIZE="20G"
|
||||
fi
|
||||
if [ "$DOCKER_SHA" == "" ]; then
|
||||
echo "Building application docker."
|
||||
docker build -q --no-cache -t ray-project/base-deps docker/base-deps
|
||||
|
||||
# Add Ray source
|
||||
git rev-parse HEAD > ./docker/tune_test/git-rev
|
||||
git archive -o ./docker/tune_test/ray.tar $(git rev-parse HEAD)
|
||||
DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test)
|
||||
fi
|
||||
|
||||
echo "Using Docker image" $DOCKER_SHA
|
||||
|
||||
|
||||
######################## EXAMPLE TESTS #################################
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/doc/examples/plot_pong_example.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/doc/examples/plot_parameter_server.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/doc/examples/plot_hyperparameter.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/doc/examples/doc_code/torch_example.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/doc/examples/doc_code/tf_example.py
|
||||
|
||||
@@ -1,47 +1,9 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Cause the script to exit if a single command fails.
|
||||
set -e
|
||||
|
||||
# Show explicitly which commands are currently running.
|
||||
set -x
|
||||
|
||||
MEMORY_SIZE="20G"
|
||||
SHM_SIZE="20G"
|
||||
# This file exists for Jenkins compatiblity
|
||||
|
||||
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
|
||||
|
||||
DOCKER_SHA=$($ROOT_DIR/../../build-docker.sh --output-sha --no-cache)
|
||||
SUPPRESS_OUTPUT=$ROOT_DIR/../suppress_output
|
||||
echo "Using Docker image" $DOCKER_SHA
|
||||
export RUN_TUNE_TESTS=1
|
||||
export RUN_DOC_TESTS=1
|
||||
export RUN_SGD_TESTS=1
|
||||
|
||||
######################## RLLIB TESTS #################################
|
||||
|
||||
# DEPRECATED: All RLlib tests have been moved to /ray/rllib/BUILD
|
||||
# source $ROOT_DIR/run_rllib_tests.sh
|
||||
|
||||
######################## TUNE TESTS #################################
|
||||
|
||||
bash $ROOT_DIR/run_tune_tests.sh ${MEMORY_SIZE} ${SHM_SIZE} $DOCKER_SHA
|
||||
|
||||
######################## EXAMPLE TESTS #################################
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/doc/examples/plot_pong_example.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/doc/examples/plot_parameter_server.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/doc/examples/plot_hyperparameter.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/doc/examples/doc_code/torch_example.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/doc/examples/doc_code/tf_example.py
|
||||
|
||||
######################## RAY BACKEND TESTS #################################
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=60G --memory=60G --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/ci/jenkins_tests/miscellaneous/large_memory_test.py
|
||||
bash $ROOT_DIR/entry_point.sh
|
||||
Executable
+83
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Cause the script to exit if a single command fails.
|
||||
set -e
|
||||
|
||||
# Show explicitly which commands are currently running.
|
||||
set -x
|
||||
|
||||
MEMORY_SIZE=$1
|
||||
SHM_SIZE=$2
|
||||
DOCKER_SHA=$3
|
||||
|
||||
ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd)
|
||||
SUPPRESS_OUTPUT=$ROOT_DIR/../suppress_output
|
||||
|
||||
if [ "$MEMORY_SIZE" == "" ]; then
|
||||
MEMORY_SIZE="20G"
|
||||
fi
|
||||
if [ "$SHM_SIZE" == "" ]; then
|
||||
SHM_SIZE="20G"
|
||||
fi
|
||||
if [ "$DOCKER_SHA" == "" ]; then
|
||||
echo "Building application docker."
|
||||
docker build -q --no-cache -t ray-project/base-deps docker/base-deps
|
||||
|
||||
# Add Ray source
|
||||
git rev-parse HEAD > ./docker/tune_test/git-rev
|
||||
git archive -o ./docker/tune_test/ray.tar $(git rev-parse HEAD)
|
||||
DOCKER_SHA=$(docker build --no-cache -q -t ray-project/tune_test docker/tune_test)
|
||||
fi
|
||||
|
||||
echo "Using Docker image" $DOCKER_SHA
|
||||
|
||||
|
||||
######################## SGD TESTS #################################
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python -m pytest /ray/python/ray/util/sgd/tests
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/raysgd_torch_signatures.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/train_example.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/train_example.py --num-workers=2
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/tune_example.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/tune_example.py --num-workers=2
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/cifar_pytorch_example.py --smoke-test
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/cifar_pytorch_example.py --smoke-test --num-workers=2
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/cifar_pytorch_pbt.py --smoke-test
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/dcgan.py --smoke-test --num-workers=2
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/tf/examples/tensorflow_train_example.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/tf/examples/tensorflow_train_example.py --num-replicas=2
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/tf/examples/tensorflow_train_example.py --tune
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/tf/examples/cifar_tf_example.py --smoke-test
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/tf/examples/cifar_tf_example.py --num-replicas 2 --smoke-test
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/tf/examples/cifar_tf_example.py --num-replicas 2 --smoke-test --augment-data
|
||||
@@ -31,58 +31,7 @@ fi
|
||||
|
||||
echo "Using Docker image" $DOCKER_SHA
|
||||
|
||||
|
||||
######################## SGD TESTS #################################
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python -m pytest /ray/python/ray/util/sgd/tests
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/raysgd_torch_signatures.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/train_example.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/train_example.py --num-workers=2
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/tune_example.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/tune_example.py --num-workers=2
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/cifar_pytorch_example.py --smoke-test
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/cifar_pytorch_example.py --smoke-test --num-workers=2
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/cifar_pytorch_pbt.py --smoke-test
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/torch/examples/dcgan.py --smoke-test --num-workers=2
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/tf/examples/tensorflow_train_example.py
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/tf/examples/tensorflow_train_example.py --num-replicas=2
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/tf/examples/tensorflow_train_example.py --tune
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/tf/examples/cifar_tf_example.py --smoke-test
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/tf/examples/cifar_tf_example.py --num-replicas 2 --smoke-test
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
python /ray/python/ray/util/sgd/tf/examples/cifar_tf_example.py --num-replicas 2 --smoke-test --augment-data
|
||||
|
||||
|
||||
######################## TUNE TESTS #################################
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
pytest /ray/python/ray/tune/tests/test_actor_reuse.py
|
||||
@@ -144,9 +93,11 @@ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE}
|
||||
python /ray/python/ray/tune/examples/hyperopt_example.py \
|
||||
--smoke-test
|
||||
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 -e SIGOPT_KEY $DOCKER_SHA \
|
||||
python /ray/python/ray/tune/examples/sigopt_example.py \
|
||||
--smoke-test
|
||||
if [[ ! -z "$SIGOPT_KEY" ]]; then
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 -e SIGOPT_KEY $DOCKER_SHA \
|
||||
python /ray/python/ray/tune/examples/sigopt_example.py \
|
||||
--smoke-test
|
||||
fi
|
||||
|
||||
# Runs only on Python3
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
@@ -196,4 +147,4 @@ $SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE}
|
||||
|
||||
# Moved to bottom because flaky
|
||||
$SUPPRESS_OUTPUT docker run --rm --shm-size=${SHM_SIZE} --memory=${MEMORY_SIZE} --memory-swap=-1 $DOCKER_SHA \
|
||||
pytest /ray/python/ray/tune/tests/test_cluster.py
|
||||
pytest /ray/python/ray/tune/tests/test_cluster.py
|
||||
@@ -13,7 +13,6 @@ RUN apt update
|
||||
RUN apt install -y kubectl
|
||||
|
||||
# We have to uninstall wrapt this way for Tensorflow compatibility
|
||||
RUN conda remove -y wrapt
|
||||
COPY requirements.txt .
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
|
||||
@@ -9,20 +9,24 @@ RUN apt-get update \
|
||||
build-essential \
|
||||
curl \
|
||||
unzip \
|
||||
libgtk2.0-dev \
|
||||
zlib1g-dev \
|
||||
libgl1-mesa-dev \
|
||||
&& apt-get clean \
|
||||
&& echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh \
|
||||
&& wget \
|
||||
--quiet 'https://repo.continuum.io/archive/Anaconda3-5.2.0-Linux-x86_64.sh' \
|
||||
--quiet "https://repo.anaconda.com/miniconda/Miniconda3-4.7.12.1-Linux-x86_64.sh" \
|
||||
-O /tmp/anaconda.sh \
|
||||
&& /bin/bash /tmp/anaconda.sh -b -p /opt/conda \
|
||||
&& rm /tmp/anaconda.sh \
|
||||
&& /opt/conda/bin/conda install -y \
|
||||
libgcc \
|
||||
libgcc python=3.6.9 \
|
||||
&& /opt/conda/bin/conda clean -y --all \
|
||||
&& /opt/conda/bin/pip install \
|
||||
flatbuffers \
|
||||
cython==0.29.0 \
|
||||
numpy==1.15.4
|
||||
|
||||
|
||||
# To avoid the following error on Jenkins:
|
||||
# AttributeError: 'numpy.ufunc' object has no attribute '__module__'
|
||||
|
||||
@@ -6,8 +6,6 @@ FROM ray-project/deploy
|
||||
RUN conda install -y numpy
|
||||
# Needed to run Tune example with a 'plot' call - which does not actually render a plot, but throws an error.
|
||||
RUN apt-get install -y zlib1g-dev libgl1-mesa-dev
|
||||
# The following is needed to support TensorFlow 1.14
|
||||
RUN conda remove -y --force wrapt
|
||||
RUN pip install -U pip
|
||||
RUN pip install gym[atari] opencv-python-headless tensorflow lz4 pytest-timeout smart_open tensorflow_probability dm_tree
|
||||
RUN pip install -U h5py # Mutes FutureWarnings
|
||||
@@ -17,3 +15,4 @@ RUN pip install ConfigSpace==0.4.10
|
||||
RUN pip install --upgrade sigopt nevergrad scikit-optimize hpbandster lightgbm xgboost torch torchvision tensorboardX dragonfly-opt
|
||||
RUN pip install -U tabulate mlflow
|
||||
RUN pip install -U pytest-remotedata>=0.3.1
|
||||
RUN pip install -U matplotlib jupyter pandas
|
||||
|
||||
@@ -4,24 +4,16 @@ FROM ray-project/base-deps
|
||||
|
||||
# We install ray and boto3 to enable the ray autoscaler as
|
||||
# a test runner.
|
||||
RUN conda install -y numpy
|
||||
RUN pip install -U pip
|
||||
RUN pip install -U https://ray-wheels.s3-us-west-2.amazonaws.com/latest/ray-0.9.0.dev0-cp36-cp36m-manylinux1_x86_64.whl
|
||||
RUN pip install -U boto3
|
||||
|
||||
# We install this after the latest wheels -- this should not override the latest wheels.
|
||||
# Needed to run Tune example with a 'plot' call - which does not actually render a plot, but throws an error.
|
||||
RUN apt-get install -y zlib1g-dev libgl1-mesa-dev
|
||||
# The following is needed to support TensorFlow 1.14
|
||||
RUN conda remove -y --force wrapt
|
||||
RUN pip install gym[atari]==0.10.11 opencv-python-headless tensorflow lz4 keras pytest-timeout smart_open torch torchvision dm_tree
|
||||
RUN pip install --upgrade bayesian-optimization
|
||||
RUN pip install --upgrade hyperopt==0.1.2
|
||||
RUN pip install ConfigSpace==0.4.10
|
||||
RUN pip install --upgrade sigopt nevergrad scikit-optimize hpbandster lightgbm xgboost tensorboardX
|
||||
RUN pip install -U mlflow
|
||||
RUN pip install -U pytest-remotedata>=0.3.1
|
||||
RUN pip install torch==1.4.0+cpu torchvision==0.5.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
|
||||
RUN pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl
|
||||
|
||||
# RUN mkdir -p /root/.ssh/
|
||||
COPY requirements.txt .
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
# We port the source code in so that we run the most up-to-date stress tests.
|
||||
ADD ray.tar /ray
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
ConfigSpace==0.4.10
|
||||
bayesian-optimization
|
||||
boto3
|
||||
dm_tree
|
||||
dragonfly-opt
|
||||
gym[atari]
|
||||
h5py
|
||||
hpbandster
|
||||
hyperopt==0.1.2
|
||||
jupyter
|
||||
keras
|
||||
lightgbm
|
||||
lz4
|
||||
matplotlib
|
||||
mlflow
|
||||
nevergrad
|
||||
opencv-python-headless
|
||||
pandas
|
||||
pytest-remotedata>=0.3.1
|
||||
pytest-timeout
|
||||
scikit-optimize
|
||||
sigopt
|
||||
smart_open
|
||||
tabulate
|
||||
tensorboardX
|
||||
tensorflow_probability
|
||||
torch
|
||||
torchvision
|
||||
xgboost
|
||||
@@ -203,7 +203,8 @@ if __name__ == "__main__":
|
||||
})
|
||||
|
||||
training_start = time.time()
|
||||
for i in range(3):
|
||||
num_epochs = 1 if args.smoke_test else 3
|
||||
for i in range(num_epochs):
|
||||
# Trains num epochs
|
||||
train_stats1 = trainer.train()
|
||||
train_stats1.update(trainer.validate())
|
||||
|
||||
Reference in New Issue
Block a user