*: Centralize requirements.txt and unify dependency versions (#9759)

* python_test: fix cython_examples in doc/ and tests/

* update setup.py to parse the bazel version string better

* all: centralize all python deps into stackable requirements files in python/

* format

* Move cython test into the proper package

* Add cross-reference dependency comments for requirements and setup.py

* re-enable version pinning on CI, fix formatting

* fix up torchvision version

* fix case in shell

ci/travis/install-dependencies.sh

@@ -58,7 +58,6 @@ install_miniconda() {
   fi
 
   local conda="${CONDA_EXE-}"  # Try to get the activated conda executable
-
   if [ -z "${conda}" ]; then  # If no conda is found, try to find it in PATH
     conda="$(command -v conda || true)"
   fi
@@ -138,7 +137,7 @@ install_miniconda() {
 }
 
 install_linters() {
-  pip install flake8==3.7.7 flake8-comprehensions flake8-quotes==2.0.0 yapf==0.23.0
+  pip install -r "${WORKSPACE_DIR}"/python/requirements_linters.txt
 }
 
 install_nvm() {
@@ -230,28 +229,13 @@ install_dependencies() {
       *) pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f "${torch_url}";;
     esac
 
-    local tf_version
-    case "${OSTYPE}" in
-      msys) tf_version="${TF_VERSION:-2.2.0}";;
-      *) tf_version="${TF_VERSION:-2.1.0}";;
-    esac
-    pip_packages+=(scipy tensorflow=="${tf_version}" cython==0.29.0 gym \
-      opencv-python-headless pyyaml pandas==1.0.5 requests feather-format lxml openpyxl xlrd \
-      py-spy pytest==5.4.3 pytest-timeout networkx tabulate aiohttp uvicorn dataclasses pygments werkzeug \
-      kubernetes flask grpcio pytest-sugar pytest-rerunfailures pytest-asyncio scikit-learn==0.22.2 numba \
-      Pillow prometheus_client boto3 pettingzoo mypy)
-    if [ "${OSTYPE}" != msys ]; then
-      # These packages aren't Windows-compatible
-      pip_packages+=(blist)  # https://github.com/DanielStutzbach/blist/issues/81#issue-391460716
-    fi
-
     # Try n times; we often encounter OpenSSL.SSL.WantReadError (or others)
     # that break the entire CI job: Simply retry installation in this case
     # after n seconds.
     local status="0";
     local errmsg="";
     for _ in {1..3}; do
-      errmsg=$(CC=gcc pip install "${pip_packages[@]}" 2>&1) && break;
+      errmsg=$(CC=gcc pip install -r "${WORKSPACE_DIR}"/python/requirements.txt 2>&1) && break;
       status=$errmsg && echo "'pip install ...' failed, will retry after n seconds!" && sleep 30;
     done
     if [ "$status" != "0" ]; then
@@ -275,34 +259,35 @@ install_dependencies() {
 
   # Additional RLlib dependencies.
   if [ "${RLLIB_TESTING-}" = 1 ]; then
-    pip install tensorflow-probability=="${TFP_VERSION-0.8}" gast==0.2.2 \
-      torch=="${TORCH_VERSION-1.4}" torchvision atari_py "gym[atari]" lz4 smart_open
+    pip install -r "${WORKSPACE_DIR}"/python/requirements_rllib.txt
   fi
 
   # Additional Tune test dependencies.
   if [ "${TUNE_TESTING-}" = 1 ]; then
-    pip install tensorflow-probability=="${TFP_VERSION-0.8}" \
-      torch=="${TORCH_VERSION-1.4}"
-    pip install -r "${WORKSPACE_DIR}"/docker/tune_test/requirements.txt
+    pip install -r "${WORKSPACE_DIR}"/python/requirements_tune.txt
   fi
 
   # Additional RaySGD test dependencies.
   if [ "${SGD_TESTING-}" = 1 ]; then
-    pip install tensorflow-probability=="${TFP_VERSION-0.8}" \
-      torch=="${TORCH_VERSION-1.4}"
-    pip install -r "${WORKSPACE_DIR}"/docker/tune_test/requirements.txt
+    pip install -r "${WORKSPACE_DIR}"/python/requirements_tune.txt
   fi
 
   # Additional Doc test dependencies.
   if [ "${DOC_TESTING-}" = 1 ]; then
-    pip install tensorflow-probability=="${TFP_VERSION-0.8}" \
-      torch=="${TORCH_VERSION-1.4}" torchvision atari_py gym[atari] lz4 smart_open
-    pip install -r "${WORKSPACE_DIR}"/docker/tune_test/requirements.txt
+    pip install -r "${WORKSPACE_DIR}"/python/requirements_tune.txt
   fi
 
-  # Additional streaming dependencies.
-  if [ "${RAY_CI_STREAMING_PYTHON_AFFECTED}" = 1 ]; then
-    pip install "msgpack>=1.0.0"
+  # If CI has deemed that a different version of Tensorflow or Torch 
+  # should be installed, then upgrade/downgrade to that specific version.
+  if [ -n "${TORCH_VERSION-}" ] || [ -n "${TFP_VERSION-}" ] || [ -n "${TF_VERSION-}" ]; then
+    case "${TORCH_VERSION-1.4}" in
+      1.5) TORCHVISION_VERSION=0.6.0;;
+      *) TORCHVISION_VERSION=0.5.0;;
+    esac
+
+    pip install --upgrade tensorflow-probability=="${TFP_VERSION-0.8}" \
+      torch=="${TORCH_VERSION-1.4}" torchvision=="${TORCHVISION_VERSION}" \
+      tensorflow=="${TF_VERSION-2.2.0}" gym
   fi
 
   if [ -n "${PYTHON-}" ] || [ -n "${LINT-}" ] || [ "${MAC_WHEELS-}" = 1 ]; then

doc/examples/cython/cython_examples/__init__.py

@@ -1,3 +1,9 @@
+# flake8: noqa
+import numpy
+import pyximport
+
+pyximport.install(setup_args={"include_dirs": numpy.get_include()})
+
 from .cython_simple import simple_func, fib, fib_int, \
         fib_cpdef, fib_cdef, simple_class
 from .masked_log import masked_log
@@ -11,17 +17,8 @@ from .cython_blas import \
         compute_single_matrix_multiplication
 
 __all__ = [
-        "simple_func",
-        "fib",
-        "fib_int",
-        "fib_cpdef",
-        "fib_cdef",
-        "simple_class",
-        "masked_log",
-        "compute_self_corr_for_voxel_sel",
-        "compute_kernel_matrix",
-        "compute_single_self_corr_syrk",
-        "compute_single_self_corr_gemm",
-        "compute_corr_vectors",
-        "compute_single_matrix_multiplication"
-        ]
+    "simple_func", "fib", "fib_int", "fib_cpdef", "fib_cdef", "simple_class",
+    "masked_log", "compute_self_corr_for_voxel_sel", "compute_kernel_matrix",
+    "compute_single_self_corr_syrk", "compute_single_self_corr_gemm",
+    "compute_corr_vectors", "compute_single_matrix_multiplication"
+]

docker/autoscaler/Dockerfile

@@ -13,10 +13,12 @@ RUN apt-get update \
     && apt-get clean 
 
 # We have to uninstall wrapt this way for Tensorflow compatibility
-COPY requirements.txt .
+COPY ../../python/requirements.txt ./
+COPY ../../python/requirements_autoscaler.txt ./
 COPY $WHEEL_PATH $WHEEL_NAME
 
-RUN pip --no-cache-dir install -r requirements.txt \
+RUN pip --no-cache-dir install -r requirements.txt \ 
+    && pip --no-cache-dir install -r requirements_autoscaler.txt \
     && pip --no-cache-dir install $WHEEL_NAME[all]
 
 # For Click

docker/tune_test/Dockerfile

@@ -12,8 +12,10 @@ RUN pip install -U https://ray-wheels.s3-us-west-2.amazonaws.com/latest/ray-0.9.
 RUN pip install torch==1.4.0+cpu torchvision==0.5.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
 RUN pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl
 
-COPY requirements.txt .
+COPY ../../python/requirements.txt .
 RUN pip install -r requirements.txt
+COPY ../../python/requirements_tune.txt .
+RUN pip install -r requirements_tune.txt
 
 # We port the source code in so that we run the most up-to-date stress tests.
 ADD ray.tar /ray

docker/tune_test/build_from_source.Dockerfile

@@ -9,8 +9,10 @@ RUN pip install -U pip
 RUN pip install torch==1.4.0+cpu torchvision==0.5.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
 RUN pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl
 
-COPY requirements.txt .
+COPY ../../python/requirements.txt .
 RUN pip install -r requirements.txt
+COPY ../../python/requirements_tune.txt .
+RUN pip install -r requirements_tune.txt
 
 # We port the source code in so that we run the most up-to-date stress tests.
 ADD ray.tar /ray

python/ray/tests/BUILD

@@ -206,13 +206,6 @@ py_test(
     deps = ["//:ray_lib"],
 )
 
-py_test(
-    name = "test_cython",
-    size = "small",
-    srcs = SRCS + ["test_cython.py"],
-    deps = ["//:ray_lib"],
-)
-
 py_test(
     name = "test_debug_tools",
     size = "small",

python/requirements.txt

@@ -0,0 +1,65 @@
+# These are mirrored in setup.py as install_requires,
+# which is what the users of the ray package will install. The rest of this file
+# sets up all the packages necessary for a /developer/ of Ray.
+#
+# In short, if you change it here, PLEASE also change it in setup.py.
+#
+# setup.py install_requires
+aiohttp
+aioredis
+click >= 7.0
+colorama
+colorful
+filelock
+google
+gpustat
+grpcio >= 1.28.1
+jsonschema
+msgpack >= 1.0.0, < 2.0.0
+numpy >= 1.16
+opencensus
+prometheus_client >= 0.7.1
+protobuf >= 3.8.0
+py-spy >= 0.2.0
+pyyaml
+redis >= 3.3.2, < 3.5.0
+requests
+
+## setup.py extras
+atari_py
+dm_tree
+flask
+gym[atari]
+lz4
+opencv-python-headless
+pandas==1.0.5
+scipy==1.4.1
+tabulate
+tensorboardX
+uvicorn
+
+# Requirements for running tests
+blist; platform_system != "Windows"
+boto3
+cython==0.29.0
+dataclasses
+feather-format
+gym
+kubernetes
+lxml
+mypy
+networkx
+numba
+openpyxl
+pettingzoo
+Pillow; platform_system != "Windows"
+pygments
+pytest==5.4.3
+pytest-asyncio
+pytest-rerunfailures
+pytest-sugar
+pytest-timeout
+scikit-learn==0.22.2
+tensorflow
+werkzeug
+xlrd

python/requirements_autoscaler.txt

@@ -1,28 +1,20 @@
-gym[atari]
-opencv-python-headless
-tensorflow
-lz4
+bayesian-optimization
+ConfigSpace==0.4.10
+google-api-python-client
+google-oauth
+h5py
+hpbandster 
+hyperopt==0.1.2
+ipython
 keras
+lightgbm 
+mlflow
+nevergrad 
+oauth2client
+scikit-optimize 
+sigopt
 smart_open
 tensorflow_probability
-h5py
-bayesian-optimization
-hyperopt==0.1.2
-ConfigSpace==0.4.10
-sigopt
-nevergrad 
-scikit-optimize 
-hpbandster 
-lightgbm 
-xgboost 
 torch 
 torchvision 
-tensorboardX
-tabulate
-mlflow
-kubernetes
-boto3
-ipython
-google-oauth
-google-api-python-client
-oauth2client
+xgboost 

python/requirements_linters.txt

@@ -0,0 +1,4 @@
+flake8==3.7.7 
+flake8-comprehensions
+flake8-quotes==2.0.0
+yapf==0.23.0

python/requirements_rllib.txt

@@ -0,0 +1,5 @@
+tensorflow-probability
+gast
+torch
+torchvision
+smart_open

python/requirements_tune.txt

@@ -1,8 +1,6 @@
 ax-platform
-ConfigSpace==0.4.10
 bayesian-optimization
-boto3
-dm_tree
+ConfigSpace==0.4.10
 dragonfly-opt
 gluoncv
 gym[atari]
@@ -12,27 +10,19 @@ hyperopt==0.1.2
 jupyter
 keras
 lightgbm
-lz4
 matplotlib
 mlflow
 mxnet
 nevergrad
-opencv-python-headless
-pandas
 pytest-remotedata>=0.3.1
-pytest-timeout
 pytorch-lightning
-scikit-learn==0.22.2
 scikit-optimize
 sigopt
 smart_open
-tabulate
-tensorboardX
 tensorflow_probability
+timm
 torch>=1.5.0
 torchvision>=0.6.0
+tune-sklearn==0.0.5
 xgboost
 zoopt>=0.4.0
-timm
-dataclasses
-tune-sklearn==0.0.5

python/setup.py

@@ -13,6 +13,7 @@ import tempfile
 import zipfile
 
 from itertools import chain
+from itertools import takewhile
 
 import urllib.error
 import urllib.parse
@@ -104,6 +105,9 @@ if os.getenv("RAY_USE_NEW_GCS") == "on":
         "ray/core/src/credis/redis/src/redis-server" + exe_suffix,
     ]
 
+# If you're adding dependencies for ray extras, please
+# also update the matching section of requirements.txt
+# in this directory
 extras = {
     "debug": [],
     "serve": ["uvicorn", "flask", "blist", "requests"],
@@ -120,10 +124,35 @@ extras["rllib"] = extras["tune"] + [
     "scipy",
 ]
 
-extras["streaming"] = ["msgpack >= 0.6.2"]
+extras["streaming"] = []
 
 extras["all"] = list(set(chain.from_iterable(extras.values())))
 
+# These are the main dependencies for users of ray. This list
+# should be carefully curated. If you change it, please reflect
+# the change in the matching section of requirements.txt
+install_requires = [
+    "aiohttp",
+    "aioredis",
+    "click >= 7.0",
+    "colorama",
+    "colorful",
+    "filelock",
+    "google",
+    "gpustat",
+    "grpcio >= 1.28.1",
+    "jsonschema",
+    "msgpack >= 1.0.0, < 2.0.0",
+    "numpy >= 1.16",
+    "protobuf >= 3.8.0",
+    "py-spy >= 0.2.0",
+    "pyyaml",
+    "requests",
+    "redis >= 3.3.2, < 3.5.0",
+    "opencensus",
+    "prometheus_client >= 0.7.1",
+]
+
 
 def is_native_windows_or_msys():
     """Check to see if we are running on native Windows,
@@ -255,7 +284,11 @@ def build(build_python, build_java):
 
     version_info = bazel_invoke(subprocess.check_output, ["--version"])
     bazel_version_str = version_info.rstrip().decode("utf-8").split(" ", 1)[1]
-    bazel_version = tuple(map(int, bazel_version_str.split(".")))
+    bazel_version_split = bazel_version_str.split(".")
+    bazel_version_digits = [
+        "".join(takewhile(str.isdigit, s)) for s in bazel_version_split
+    ]
+    bazel_version = tuple(map(int, bazel_version_digits))
     if bazel_version < SUPPORTED_BAZEL:
         logger.warning("Expected Bazel version {} but found {}".format(
             ".".join(map(str, SUPPORTED_BAZEL)), bazel_version_str))
@@ -305,29 +338,6 @@ def find_version(*filepath):
         raise RuntimeError("Unable to find version string.")
 
 
-install_requires = [
-    "aiohttp",
-    "aioredis",
-    "click >= 7.0",
-    "colorama",
-    "colorful",
-    "filelock",
-    "google",
-    "gpustat",
-    "grpcio >= 1.28.1",
-    "jsonschema",
-    "msgpack >= 0.6.0, < 2.0.0",
-    "numpy >= 1.16",
-    "protobuf >= 3.8.0",
-    "py-spy >= 0.2.0",
-    "pyyaml",
-    "requests",
-    "redis >= 3.3.2, < 3.5.0",
-    "opencensus",
-    "prometheus_client >= 0.7.1",
-]
-
-
 def pip_run(build_ext):
     build(True, BUILD_JAVA)