From 43f70faa259db44de1977ee8356b66785c0bc761 Mon Sep 17 00:00:00 2001 From: Amog Kamsetty Date: Thu, 7 Jan 2021 14:03:06 -0800 Subject: [PATCH] [Tune] Pin Tune Dependencies (#13027) Co-authored-by: Ian --- .travis.yml | 12 ------ ci/travis/build-docker-images.py | 2 + ci/travis/install-dependencies.sh | 49 +++++++++++++---------- docker/ray-ml/Dockerfile | 5 ++- python/requirements/requirements_tune.txt | 47 ++++++++++++++++++++++ python/requirements_tune.txt | 38 ------------------ 6 files changed, 79 insertions(+), 74 deletions(-) create mode 100644 python/requirements/requirements_tune.txt delete mode 100644 python/requirements_tune.txt diff --git a/.travis.yml b/.travis.yml index 54b8999b6..8bf9b5093 100644 --- a/.travis.yml +++ b/.travis.yml @@ -358,9 +358,6 @@ matrix: env: - TUNE_TESTING=1 - PYTHON=3.6 - - TF_VERSION=2.2.0 - - TFP_VERSION=0.8 - - TORCH_VERSION=1.7 - PYTHONWARNINGS=ignore install: - . ./ci/travis/ci.sh init RAY_CI_TUNE_AFFECTED @@ -378,9 +375,6 @@ matrix: env: - SGD_TESTING=1 - PYTHON=3.6 - - TF_VERSION=2.1.0 - - TFP_VERSION=0.8 - - TORCH_VERSION=1.7 - PYTHONWARNINGS=ignore install: - . ./ci/travis/ci.sh init RAY_CI_SGD_AFFECTED @@ -398,9 +392,6 @@ matrix: env: - DOC_TESTING=1 - PYTHON=3.6 - - TF_VERSION=2.1.0 - - TFP_VERSION=0.8 - - TORCH_VERSION=1.7 - PYTHONWARNINGS=ignore install: - . ./ci/travis/ci.sh init RAY_CI_PYTHON_AFFECTED,RAY_CI_TUNE_AFFECTED,RAY_CI_DOC_AFFECTED @@ -416,9 +407,6 @@ matrix: env: - PYTHON=3.7 TUNE_TESTING=1 - INSTALL_HOROVOD=1 - - TF_VERSION=2.1.0 - - TFP_VERSION=0.8 - - TORCH_VERSION=1.7 - PYTHONWARNINGS=ignore install: - . ./ci/travis/ci.sh init RAY_CI_TUNE_AFFECTED,RAY_CI_SGD_AFFECTED diff --git a/ci/travis/build-docker-images.py b/ci/travis/build-docker-images.py index b3152089c..a2ae7a18d 100644 --- a/ci/travis/build-docker-images.py +++ b/ci/travis/build-docker-images.py @@ -179,6 +179,8 @@ def build_ray_ml(): root_dir = _get_root_dir() requirement_files = glob.glob( f"{_get_root_dir()}/python/requirements*.txt") + requirement_files.extend( + glob.glob(f"{_get_root_dir()}/python/requirements/*.txt")) for fl in requirement_files: shutil.copy(fl, os.path.join(root_dir, "docker/ray-ml/")) ray_ml_images = _build_cpu_gpu_images("ray-ml") diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh index 7bc73e967..67a85fab0 100755 --- a/ci/travis/install-dependencies.sh +++ b/ci/travis/install-dependencies.sh @@ -249,13 +249,16 @@ install_dependencies() { pip install --no-clean dm-tree # --no-clean is due to: https://github.com/deepmind/tree/issues/5 if [ -n "${PYTHON-}" ]; then - # PyTorch is installed first since we are using a "-f" directive to find the wheels. - # We want to install the CPU version only. - local torch_url="https://download.pytorch.org/whl/torch_stable.html" - case "${OSTYPE}" in - darwin*) pip install torch torchvision;; - *) pip install torch==1.7.0+cpu torchvision==0.8.1+cpu -f "${torch_url}";; - esac + # Remove this entire section once RLlib and Serve dependencies are fixed. + if [ "${DOC_TESTING-}" != 1 ] && [ "${SGD_TESTING-}" != 1 ] && [ "${TUNE_TESTING-}" != 1 ]; then + # PyTorch is installed first since we are using a "-f" directive to find the wheels. + # We want to install the CPU version only. + local torch_url="https://download.pytorch.org/whl/torch_stable.html" + case "${OSTYPE}" in + darwin*) pip install torch torchvision;; + *) pip install torch==1.7.0+cpu torchvision==0.8.1+cpu -f "${torch_url}";; + esac + fi # Try n times; we often encounter OpenSSL.SSL.WantReadError (or others) # that break the entire CI job: Simply retry installation in this case @@ -294,32 +297,34 @@ install_dependencies() { # Additional Tune test dependencies. if [ "${TUNE_TESTING-}" = 1 ]; then - pip install -r "${WORKSPACE_DIR}"/python/requirements_tune.txt + pip install -r "${WORKSPACE_DIR}"/python/requirements/requirements_tune.txt fi # Additional RaySGD test dependencies. if [ "${SGD_TESTING-}" = 1 ]; then - pip install -r "${WORKSPACE_DIR}"/python/requirements_tune.txt + pip install -r "${WORKSPACE_DIR}"/python/requirements/requirements_tune.txt # TODO: eventually have a separate requirements file for Ray SGD. fi # Additional Doc test dependencies. if [ "${DOC_TESTING-}" = 1 ]; then - pip install -r "${WORKSPACE_DIR}"/python/requirements_tune.txt + pip install -r "${WORKSPACE_DIR}"/python/requirements/requirements_tune.txt fi - - # If CI has deemed that a different version of Tensorflow or Torch - # should be installed, then upgrade/downgrade to that specific version. - if [ -n "${TORCH_VERSION-}" ] || [ -n "${TFP_VERSION-}" ] || [ -n "${TF_VERSION-}" ]; then - case "${TORCH_VERSION-1.7}" in - 1.7) TORCHVISION_VERSION=0.8.1;; - 1.5) TORCHVISION_VERSION=0.6.0;; - *) TORCHVISION_VERSION=0.5.0;; - esac - pip install --use-deprecated=legacy-resolver --upgrade tensorflow-probability=="${TFP_VERSION-0.8}" \ - torch=="${TORCH_VERSION-1.7}" torchvision=="${TORCHVISION_VERSION}" \ - tensorflow=="${TF_VERSION-2.2.0}" gym + # Remove this entire section once RLlib and Serve dependencies are fixed. + if [ "${DOC_TESTING-}" != 1 ] && [ "${SGD_TESTING-}" != 1 ] && [ "${TUNE_TESTING-}" != 1 ]; then + # If CI has deemed that a different version of Tensorflow or Torch + # should be installed, then upgrade/downgrade to that specific version. + if [ -n "${TORCH_VERSION-}" ] || [ -n "${TFP_VERSION-}" ] || [ -n "${TF_VERSION-}" ]; then + case "${TORCH_VERSION-1.7}" in + 1.7) TORCHVISION_VERSION=0.8.1;; + 1.5) TORCHVISION_VERSION=0.6.0;; + *) TORCHVISION_VERSION=0.5.0;; + esac + pip install --use-deprecated=legacy-resolver --upgrade tensorflow-probability=="${TFP_VERSION-0.8}" \ + torch=="${TORCH_VERSION-1.7}" torchvision=="${TORCHVISION_VERSION}" \ + tensorflow=="${TF_VERSION-2.2.0}" gym + fi fi # Additional Tune dependency for Horovod. diff --git a/docker/ray-ml/Dockerfile b/docker/ray-ml/Dockerfile index d878a8340..ca9a9402d 100644 --- a/docker/ray-ml/Dockerfile +++ b/docker/ray-ml/Dockerfile @@ -5,7 +5,7 @@ FROM rayproject/ray:nightly"$GPU" COPY requirements.txt ./ COPY requirements_ml_docker.txt ./ COPY requirements_rllib.txt ./ -COPY requirements_tune.txt ./ +COPY requirements_tune.txt ./requirements_tune.txt RUN sudo apt-get update \ && sudo apt-get install -y gcc \ @@ -16,7 +16,8 @@ RUN sudo apt-get update \ && $HOME/anaconda3/bin/pip --use-deprecated=legacy-resolver --no-cache-dir install -r requirements.txt \ && $HOME/anaconda3/bin/pip --use-deprecated=legacy-resolver --no-cache-dir install -r requirements_ml_docker.txt \ # Remove dataclasses & typing because they are included in Py3.7 - && $HOME/anaconda3/bin/pip uninstall dataclasses typing -y \ + && $HOME/anaconda3/bin/pip uninstall dataclasses typing -y \ && sudo rm requirements.txt && sudo rm requirements_ml_docker.txt \ + && sudo rm requirements_tune.txt && sudo rm requirements_rllib.txt \ && sudo apt-get clean diff --git a/python/requirements/requirements_tune.txt b/python/requirements/requirements_tune.txt new file mode 100644 index 000000000..4bbbacfbd --- /dev/null +++ b/python/requirements/requirements_tune.txt @@ -0,0 +1,47 @@ +ax-platform==0.1.9; python_version < '3.7' +ax-platform==0.1.19; python_version >= '3.7' +bayesian-optimization==1.2.0 +ConfigSpace==0.4.10 +dragonfly-opt==0.1.6 +gluoncv==0.9.0 +gpy==1.9.9 +gym[atari]==0.18.0 +h5py==3.1.0 +hpbandster==0.7.4 +hyperopt==0.2.5 +jupyter==1.0.0 +keras==2.4.3 +kubernetes==12.0.1 +lightgbm==3.1.1 +matplotlib==3.3.3 +mlflow==1.13.0 +mxnet==1.7.0.post1 +nevergrad==0.4.2.post5 +optuna==2.3.0 +pytest-remotedata==0.3.2 +pytorch-lightning-bolts==0.2.5 +pytorch-lightning==1.0.3 +scikit-learn==0.22.2 +scikit-optimize==0.8.1 +sigopt==5.7.0 +smart_open==4.0.1 +tensorflow-probability==0.11.1 +timm==0.3.2 + + +torch==1.7.0;sys_platform=="darwin" +torchvision==0.8.1;sys_platform=="darwin" + +# On non-OSX machines only install CPU version of torch and torchvision +-f https://download.pytorch.org/whl/torch_stable.html +torch==1.7.0+cpu;sys_platform!="darwin" +-f https://download.pytorch.org/whl/torch_stable.html +torchvision==0.8.1+cpu;sys_platform!="darwin" + + +transformers==3.1 +tune-sklearn==0.2.1 +wandb==0.10.12 +xgboost==1.3.0.post0 +xgboost_ray==0.0.1 +zoopt==0.4.1 diff --git a/python/requirements_tune.txt b/python/requirements_tune.txt deleted file mode 100644 index 9be5ee118..000000000 --- a/python/requirements_tune.txt +++ /dev/null @@ -1,38 +0,0 @@ -ax-platform -bayesian-optimization -ConfigSpace==0.4.10 -dragonfly-opt -gluoncv -gorilla # Need this because bug in mlflow. Should be fixed in v1.12.2 -gym[atari] -GPy -h5py -hpbandster -hyperopt>=0.2.5 -jupyter -keras -kubernetes -lightgbm -matplotlib -mlflow -mxnet -nevergrad -optuna -pytest-remotedata>=0.3.1 -pytorch-lightning==1.0.3 -pytorch-lightning-bolts -scikit-learn -scikit-optimize -sigopt -smart_open -tensorflow-probability -timm -torch>=1.6.0 -torchvision>=0.6.0 -# transformers -git+git://github.com/huggingface/transformers.git@bdcc4b78a27775d1ec8f3fd297cb679c257289db#transformers -git+git://github.com/ray-project/tune-sklearn@master#tune-sklearn -git+git://github.com/ray-project/xgboost_ray@master#xgboost_ray -wandb -xgboost -zoopt>=0.4.1