*: Centralize requirements.txt and unify dependency versions (#9759)

* python_test: fix cython_examples in doc/ and tests/

* update setup.py to parse the bazel version string better

* all: centralize all python deps into stackable requirements files in python/

* format

* Move cython test into the proper package

* Add cross-reference dependency comments for requirements and setup.py

* re-enable version pinning on CI, fix formatting

* fix up torchvision version

* fix case in shell

python/ray/tests/BUILD

@@ -206,13 +206,6 @@ py_test(
     deps = ["//:ray_lib"],
 )
 
-py_test(
-    name = "test_cython",
-    size = "small",
-    srcs = SRCS + ["test_cython.py"],
-    deps = ["//:ray_lib"],
-)
-
 py_test(
     name = "test_debug_tools",
     size = "small",

python/ray/tests/test_cython.py

@@ -1,59 +0,0 @@
-from __future__ import absolute_import
-from __future__ import print_function
-
-import math
-import numpy as np
-import unittest
-
-import ray
-import cython_examples as cyth
-
-
-def get_ray_result(cython_func, *args):
-    func = ray.remote(cython_func)
-    return ray.get(func.remote(*args))
-
-
-class CythonTest(unittest.TestCase):
-    def setUp(self):
-        ray.init(object_store_memory=int(150 * 1024 * 1024))
-
-    def tearDown(self):
-        ray.shutdown()
-
-    def assertEqualHelper(self, cython_func, expected, *args):
-        assert get_ray_result(cython_func, *args) == expected
-
-    def test_simple_func(self):
-        self.assertEqualHelper(cyth.simple_func, 6, 1, 2, 3)
-        self.assertEqualHelper(cyth.fib, 55, 10)
-        self.assertEqualHelper(cyth.fib_int, 55, 10)
-        self.assertEqualHelper(cyth.fib_cpdef, 55, 10)
-        self.assertEqualHelper(cyth.fib_cdef, 55, 10)
-
-    def test_simple_class(self):
-        cls = ray.remote(cyth.simple_class)
-        a1 = cls.remote()
-        a2 = cls.remote()
-
-        result1 = ray.get(a1.increment.remote())
-        result2 = ray.get(a2.increment.remote())
-        result3 = ray.get(a2.increment.remote())
-
-        assert result1 == 1
-        assert result2 == 1
-        assert result3 == 2
-
-    def test_numpy(self):
-        array = np.array([-1.0, 0.0, 1.0, 2.0])
-
-        answer = [float("-inf") if x <= 0 else math.log(x) for x in array]
-        result = get_ray_result(cyth.masked_log, array)
-
-        np.testing.assert_array_equal(answer, result)
-
-
-if __name__ == "__main__":
-    import pytest
-    import sys
-    sys.exit(pytest.main(["-v", __file__]))

python/requirements.txt

@@ -0,0 +1,65 @@
+# These are mirrored in setup.py as install_requires,
+# which is what the users of the ray package will install. The rest of this file
+# sets up all the packages necessary for a /developer/ of Ray.
+#
+# In short, if you change it here, PLEASE also change it in setup.py.
+#
+# setup.py install_requires
+aiohttp
+aioredis
+click >= 7.0
+colorama
+colorful
+filelock
+google
+gpustat
+grpcio >= 1.28.1
+jsonschema
+msgpack >= 1.0.0, < 2.0.0
+numpy >= 1.16
+opencensus
+prometheus_client >= 0.7.1
+protobuf >= 3.8.0
+py-spy >= 0.2.0
+pyyaml
+redis >= 3.3.2, < 3.5.0
+requests
+
+## setup.py extras
+atari_py
+dm_tree
+flask
+gym[atari]
+lz4
+opencv-python-headless
+pandas==1.0.5
+scipy==1.4.1
+tabulate
+tensorboardX
+uvicorn
+
+# Requirements for running tests
+blist; platform_system != "Windows"
+boto3
+cython==0.29.0
+dataclasses
+feather-format
+gym
+kubernetes
+lxml
+mypy
+networkx
+numba
+openpyxl
+pettingzoo
+Pillow; platform_system != "Windows"
+pygments
+pytest==5.4.3
+pytest-asyncio
+pytest-rerunfailures
+pytest-sugar
+pytest-timeout
+scikit-learn==0.22.2
+tensorflow
+werkzeug
+xlrd

python/requirements_autoscaler.txt

@@ -0,0 +1,20 @@
+bayesian-optimization
+ConfigSpace==0.4.10
+google-api-python-client
+google-oauth
+h5py
+hpbandster 
+hyperopt==0.1.2
+ipython
+keras
+lightgbm 
+mlflow
+nevergrad 
+oauth2client
+scikit-optimize 
+sigopt
+smart_open
+tensorflow_probability
+torch 
+torchvision 
+xgboost 

python/requirements_linters.txt

@@ -0,0 +1,4 @@
+flake8==3.7.7 
+flake8-comprehensions
+flake8-quotes==2.0.0
+yapf==0.23.0

python/requirements_rllib.txt

@@ -0,0 +1,5 @@
+tensorflow-probability
+gast
+torch
+torchvision
+smart_open

python/requirements_tune.txt

@@ -0,0 +1,28 @@
+ax-platform
+bayesian-optimization
+ConfigSpace==0.4.10
+dragonfly-opt
+gluoncv
+gym[atari]
+h5py
+hpbandster
+hyperopt==0.1.2
+jupyter
+keras
+lightgbm
+matplotlib
+mlflow
+mxnet
+nevergrad
+pytest-remotedata>=0.3.1
+pytorch-lightning
+scikit-optimize
+sigopt
+smart_open
+tensorflow_probability
+timm
+torch>=1.5.0
+torchvision>=0.6.0
+tune-sklearn==0.0.5
+xgboost
+zoopt>=0.4.0

python/setup.py

@@ -13,6 +13,7 @@ import tempfile
 import zipfile
 
 from itertools import chain
+from itertools import takewhile
 
 import urllib.error
 import urllib.parse
@@ -104,6 +105,9 @@ if os.getenv("RAY_USE_NEW_GCS") == "on":
         "ray/core/src/credis/redis/src/redis-server" + exe_suffix,
     ]
 
+# If you're adding dependencies for ray extras, please
+# also update the matching section of requirements.txt
+# in this directory
 extras = {
     "debug": [],
     "serve": ["uvicorn", "flask", "blist", "requests"],
@@ -120,10 +124,35 @@ extras["rllib"] = extras["tune"] + [
     "scipy",
 ]
 
-extras["streaming"] = ["msgpack >= 0.6.2"]
+extras["streaming"] = []
 
 extras["all"] = list(set(chain.from_iterable(extras.values())))
 
+# These are the main dependencies for users of ray. This list
+# should be carefully curated. If you change it, please reflect
+# the change in the matching section of requirements.txt
+install_requires = [
+    "aiohttp",
+    "aioredis",
+    "click >= 7.0",
+    "colorama",
+    "colorful",
+    "filelock",
+    "google",
+    "gpustat",
+    "grpcio >= 1.28.1",
+    "jsonschema",
+    "msgpack >= 1.0.0, < 2.0.0",
+    "numpy >= 1.16",
+    "protobuf >= 3.8.0",
+    "py-spy >= 0.2.0",
+    "pyyaml",
+    "requests",
+    "redis >= 3.3.2, < 3.5.0",
+    "opencensus",
+    "prometheus_client >= 0.7.1",
+]
+
 
 def is_native_windows_or_msys():
     """Check to see if we are running on native Windows,
@@ -255,7 +284,11 @@ def build(build_python, build_java):
 
     version_info = bazel_invoke(subprocess.check_output, ["--version"])
     bazel_version_str = version_info.rstrip().decode("utf-8").split(" ", 1)[1]
-    bazel_version = tuple(map(int, bazel_version_str.split(".")))
+    bazel_version_split = bazel_version_str.split(".")
+    bazel_version_digits = [
+        "".join(takewhile(str.isdigit, s)) for s in bazel_version_split
+    ]
+    bazel_version = tuple(map(int, bazel_version_digits))
     if bazel_version < SUPPORTED_BAZEL:
         logger.warning("Expected Bazel version {} but found {}".format(
             ".".join(map(str, SUPPORTED_BAZEL)), bazel_version_str))
@@ -305,29 +338,6 @@ def find_version(*filepath):
         raise RuntimeError("Unable to find version string.")
 
 
-install_requires = [
-    "aiohttp",
-    "aioredis",
-    "click >= 7.0",
-    "colorama",
-    "colorful",
-    "filelock",
-    "google",
-    "gpustat",
-    "grpcio >= 1.28.1",
-    "jsonschema",
-    "msgpack >= 0.6.0, < 2.0.0",
-    "numpy >= 1.16",
-    "protobuf >= 3.8.0",
-    "py-spy >= 0.2.0",
-    "pyyaml",
-    "requests",
-    "redis >= 3.3.2, < 3.5.0",
-    "opencensus",
-    "prometheus_client >= 0.7.1",
-]
-
-
 def pip_run(build_ext):
     build(True, BUILD_JAVA)