diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh index dd0995fd3..46d9c2bae 100755 --- a/ci/travis/install-dependencies.sh +++ b/ci/travis/install-dependencies.sh @@ -58,7 +58,6 @@ install_miniconda() { fi local conda="${CONDA_EXE-}" # Try to get the activated conda executable - if [ -z "${conda}" ]; then # If no conda is found, try to find it in PATH conda="$(command -v conda || true)" fi @@ -138,7 +137,7 @@ install_miniconda() { } install_linters() { - pip install flake8==3.7.7 flake8-comprehensions flake8-quotes==2.0.0 yapf==0.23.0 + pip install -r "${WORKSPACE_DIR}"/python/requirements_linters.txt } install_nvm() { @@ -230,28 +229,13 @@ install_dependencies() { *) pip install torch==1.5.0+cpu torchvision==0.6.0+cpu -f "${torch_url}";; esac - local tf_version - case "${OSTYPE}" in - msys) tf_version="${TF_VERSION:-2.2.0}";; - *) tf_version="${TF_VERSION:-2.1.0}";; - esac - pip_packages+=(scipy tensorflow=="${tf_version}" cython==0.29.0 gym \ - opencv-python-headless pyyaml pandas==1.0.5 requests feather-format lxml openpyxl xlrd \ - py-spy pytest==5.4.3 pytest-timeout networkx tabulate aiohttp uvicorn dataclasses pygments werkzeug \ - kubernetes flask grpcio pytest-sugar pytest-rerunfailures pytest-asyncio scikit-learn==0.22.2 numba \ - Pillow prometheus_client boto3 pettingzoo mypy) - if [ "${OSTYPE}" != msys ]; then - # These packages aren't Windows-compatible - pip_packages+=(blist) # https://github.com/DanielStutzbach/blist/issues/81#issue-391460716 - fi - # Try n times; we often encounter OpenSSL.SSL.WantReadError (or others) # that break the entire CI job: Simply retry installation in this case # after n seconds. local status="0"; local errmsg=""; for _ in {1..3}; do - errmsg=$(CC=gcc pip install "${pip_packages[@]}" 2>&1) && break; + errmsg=$(CC=gcc pip install -r "${WORKSPACE_DIR}"/python/requirements.txt 2>&1) && break; status=$errmsg && echo "'pip install ...' failed, will retry after n seconds!" && sleep 30; done if [ "$status" != "0" ]; then @@ -275,34 +259,35 @@ install_dependencies() { # Additional RLlib dependencies. if [ "${RLLIB_TESTING-}" = 1 ]; then - pip install tensorflow-probability=="${TFP_VERSION-0.8}" gast==0.2.2 \ - torch=="${TORCH_VERSION-1.4}" torchvision atari_py "gym[atari]" lz4 smart_open + pip install -r "${WORKSPACE_DIR}"/python/requirements_rllib.txt fi # Additional Tune test dependencies. if [ "${TUNE_TESTING-}" = 1 ]; then - pip install tensorflow-probability=="${TFP_VERSION-0.8}" \ - torch=="${TORCH_VERSION-1.4}" - pip install -r "${WORKSPACE_DIR}"/docker/tune_test/requirements.txt + pip install -r "${WORKSPACE_DIR}"/python/requirements_tune.txt fi # Additional RaySGD test dependencies. if [ "${SGD_TESTING-}" = 1 ]; then - pip install tensorflow-probability=="${TFP_VERSION-0.8}" \ - torch=="${TORCH_VERSION-1.4}" - pip install -r "${WORKSPACE_DIR}"/docker/tune_test/requirements.txt + pip install -r "${WORKSPACE_DIR}"/python/requirements_tune.txt fi # Additional Doc test dependencies. if [ "${DOC_TESTING-}" = 1 ]; then - pip install tensorflow-probability=="${TFP_VERSION-0.8}" \ - torch=="${TORCH_VERSION-1.4}" torchvision atari_py gym[atari] lz4 smart_open - pip install -r "${WORKSPACE_DIR}"/docker/tune_test/requirements.txt + pip install -r "${WORKSPACE_DIR}"/python/requirements_tune.txt fi - # Additional streaming dependencies. - if [ "${RAY_CI_STREAMING_PYTHON_AFFECTED}" = 1 ]; then - pip install "msgpack>=1.0.0" + # If CI has deemed that a different version of Tensorflow or Torch + # should be installed, then upgrade/downgrade to that specific version. + if [ -n "${TORCH_VERSION-}" ] || [ -n "${TFP_VERSION-}" ] || [ -n "${TF_VERSION-}" ]; then + case "${TORCH_VERSION-1.4}" in + 1.5) TORCHVISION_VERSION=0.6.0;; + *) TORCHVISION_VERSION=0.5.0;; + esac + + pip install --upgrade tensorflow-probability=="${TFP_VERSION-0.8}" \ + torch=="${TORCH_VERSION-1.4}" torchvision=="${TORCHVISION_VERSION}" \ + tensorflow=="${TF_VERSION-2.2.0}" gym fi if [ -n "${PYTHON-}" ] || [ -n "${LINT-}" ] || [ "${MAC_WHEELS-}" = 1 ]; then diff --git a/doc/examples/cython/cython_examples/__init__.py b/doc/examples/cython/cython_examples/__init__.py index 0f4ba110e..b3e9dd437 100644 --- a/doc/examples/cython/cython_examples/__init__.py +++ b/doc/examples/cython/cython_examples/__init__.py @@ -1,3 +1,9 @@ +# flake8: noqa +import numpy +import pyximport + +pyximport.install(setup_args={"include_dirs": numpy.get_include()}) + from .cython_simple import simple_func, fib, fib_int, \ fib_cpdef, fib_cdef, simple_class from .masked_log import masked_log @@ -11,17 +17,8 @@ from .cython_blas import \ compute_single_matrix_multiplication __all__ = [ - "simple_func", - "fib", - "fib_int", - "fib_cpdef", - "fib_cdef", - "simple_class", - "masked_log", - "compute_self_corr_for_voxel_sel", - "compute_kernel_matrix", - "compute_single_self_corr_syrk", - "compute_single_self_corr_gemm", - "compute_corr_vectors", - "compute_single_matrix_multiplication" - ] + "simple_func", "fib", "fib_int", "fib_cpdef", "fib_cdef", "simple_class", + "masked_log", "compute_self_corr_for_voxel_sel", "compute_kernel_matrix", + "compute_single_self_corr_syrk", "compute_single_self_corr_gemm", + "compute_corr_vectors", "compute_single_matrix_multiplication" +] diff --git a/python/ray/tests/test_cython.py b/doc/examples/cython/tests/test_cython.py similarity index 100% rename from python/ray/tests/test_cython.py rename to doc/examples/cython/tests/test_cython.py diff --git a/docker/autoscaler/Dockerfile b/docker/autoscaler/Dockerfile index 3c26d582b..cc58ebc96 100644 --- a/docker/autoscaler/Dockerfile +++ b/docker/autoscaler/Dockerfile @@ -13,10 +13,12 @@ RUN apt-get update \ && apt-get clean # We have to uninstall wrapt this way for Tensorflow compatibility -COPY requirements.txt . +COPY ../../python/requirements.txt ./ +COPY ../../python/requirements_autoscaler.txt ./ COPY $WHEEL_PATH $WHEEL_NAME -RUN pip --no-cache-dir install -r requirements.txt \ +RUN pip --no-cache-dir install -r requirements.txt \ + && pip --no-cache-dir install -r requirements_autoscaler.txt \ && pip --no-cache-dir install $WHEEL_NAME[all] # For Click diff --git a/docker/tune_test/Dockerfile b/docker/tune_test/Dockerfile index b6a3343cf..cac1ed82e 100644 --- a/docker/tune_test/Dockerfile +++ b/docker/tune_test/Dockerfile @@ -12,8 +12,10 @@ RUN pip install -U https://ray-wheels.s3-us-west-2.amazonaws.com/latest/ray-0.9. RUN pip install torch==1.4.0+cpu torchvision==0.5.0+cpu -f https://download.pytorch.org/whl/torch_stable.html RUN pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl -COPY requirements.txt . +COPY ../../python/requirements.txt . RUN pip install -r requirements.txt +COPY ../../python/requirements_tune.txt . +RUN pip install -r requirements_tune.txt # We port the source code in so that we run the most up-to-date stress tests. ADD ray.tar /ray diff --git a/docker/tune_test/build_from_source.Dockerfile b/docker/tune_test/build_from_source.Dockerfile index 1779f5f27..38afa7c69 100644 --- a/docker/tune_test/build_from_source.Dockerfile +++ b/docker/tune_test/build_from_source.Dockerfile @@ -9,8 +9,10 @@ RUN pip install -U pip RUN pip install torch==1.4.0+cpu torchvision==0.5.0+cpu -f https://download.pytorch.org/whl/torch_stable.html RUN pip install https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow_cpu-2.1.0-cp36-cp36m-manylinux2010_x86_64.whl -COPY requirements.txt . +COPY ../../python/requirements.txt . RUN pip install -r requirements.txt +COPY ../../python/requirements_tune.txt . +RUN pip install -r requirements_tune.txt # We port the source code in so that we run the most up-to-date stress tests. ADD ray.tar /ray diff --git a/python/ray/tests/BUILD b/python/ray/tests/BUILD index 1437b2e16..0624dbc57 100644 --- a/python/ray/tests/BUILD +++ b/python/ray/tests/BUILD @@ -206,13 +206,6 @@ py_test( deps = ["//:ray_lib"], ) -py_test( - name = "test_cython", - size = "small", - srcs = SRCS + ["test_cython.py"], - deps = ["//:ray_lib"], -) - py_test( name = "test_debug_tools", size = "small", diff --git a/python/requirements.txt b/python/requirements.txt new file mode 100644 index 000000000..f5289741b --- /dev/null +++ b/python/requirements.txt @@ -0,0 +1,65 @@ +# These are mirrored in setup.py as install_requires, +# which is what the users of the ray package will install. The rest of this file +# sets up all the packages necessary for a /developer/ of Ray. +# +# In short, if you change it here, PLEASE also change it in setup.py. +# +# setup.py install_requires +aiohttp +aioredis +click >= 7.0 +colorama +colorful +filelock +google +gpustat +grpcio >= 1.28.1 +jsonschema +msgpack >= 1.0.0, < 2.0.0 +numpy >= 1.16 +opencensus +prometheus_client >= 0.7.1 +protobuf >= 3.8.0 +py-spy >= 0.2.0 +pyyaml +redis >= 3.3.2, < 3.5.0 +requests + +## setup.py extras +atari_py +dm_tree +flask +gym[atari] +lz4 +opencv-python-headless +pandas==1.0.5 +scipy==1.4.1 +tabulate +tensorboardX +uvicorn + +# Requirements for running tests +blist; platform_system != "Windows" +boto3 +cython==0.29.0 +dataclasses +feather-format +gym +kubernetes +lxml +mypy +networkx +numba +openpyxl +pettingzoo +Pillow; platform_system != "Windows" +pygments +pytest==5.4.3 +pytest-asyncio +pytest-rerunfailures +pytest-sugar +pytest-timeout +scikit-learn==0.22.2 +tensorflow +werkzeug +xlrd diff --git a/docker/autoscaler/requirements.txt b/python/requirements_autoscaler.txt similarity index 70% rename from docker/autoscaler/requirements.txt rename to python/requirements_autoscaler.txt index fa8308f6b..f86a91f98 100644 --- a/docker/autoscaler/requirements.txt +++ b/python/requirements_autoscaler.txt @@ -1,28 +1,20 @@ -gym[atari] -opencv-python-headless -tensorflow -lz4 +bayesian-optimization +ConfigSpace==0.4.10 +google-api-python-client +google-oauth +h5py +hpbandster +hyperopt==0.1.2 +ipython keras +lightgbm +mlflow +nevergrad +oauth2client +scikit-optimize +sigopt smart_open tensorflow_probability -h5py -bayesian-optimization -hyperopt==0.1.2 -ConfigSpace==0.4.10 -sigopt -nevergrad -scikit-optimize -hpbandster -lightgbm -xgboost torch torchvision -tensorboardX -tabulate -mlflow -kubernetes -boto3 -ipython -google-oauth -google-api-python-client -oauth2client \ No newline at end of file +xgboost diff --git a/python/requirements_linters.txt b/python/requirements_linters.txt new file mode 100644 index 000000000..0af7a00b4 --- /dev/null +++ b/python/requirements_linters.txt @@ -0,0 +1,4 @@ +flake8==3.7.7 +flake8-comprehensions +flake8-quotes==2.0.0 +yapf==0.23.0 diff --git a/python/requirements_rllib.txt b/python/requirements_rllib.txt new file mode 100644 index 000000000..bda6d294a --- /dev/null +++ b/python/requirements_rllib.txt @@ -0,0 +1,5 @@ +tensorflow-probability +gast +torch +torchvision +smart_open diff --git a/docker/tune_test/requirements.txt b/python/requirements_tune.txt similarity index 75% rename from docker/tune_test/requirements.txt rename to python/requirements_tune.txt index a6077e323..3e8c2159c 100644 --- a/docker/tune_test/requirements.txt +++ b/python/requirements_tune.txt @@ -1,8 +1,6 @@ ax-platform -ConfigSpace==0.4.10 bayesian-optimization -boto3 -dm_tree +ConfigSpace==0.4.10 dragonfly-opt gluoncv gym[atari] @@ -12,27 +10,19 @@ hyperopt==0.1.2 jupyter keras lightgbm -lz4 matplotlib mlflow mxnet nevergrad -opencv-python-headless -pandas pytest-remotedata>=0.3.1 -pytest-timeout pytorch-lightning -scikit-learn==0.22.2 scikit-optimize sigopt smart_open -tabulate -tensorboardX tensorflow_probability +timm torch>=1.5.0 torchvision>=0.6.0 +tune-sklearn==0.0.5 xgboost zoopt>=0.4.0 -timm -dataclasses -tune-sklearn==0.0.5 diff --git a/python/setup.py b/python/setup.py index f4f6c4798..141df7357 100644 --- a/python/setup.py +++ b/python/setup.py @@ -13,6 +13,7 @@ import tempfile import zipfile from itertools import chain +from itertools import takewhile import urllib.error import urllib.parse @@ -104,6 +105,9 @@ if os.getenv("RAY_USE_NEW_GCS") == "on": "ray/core/src/credis/redis/src/redis-server" + exe_suffix, ] +# If you're adding dependencies for ray extras, please +# also update the matching section of requirements.txt +# in this directory extras = { "debug": [], "serve": ["uvicorn", "flask", "blist", "requests"], @@ -120,10 +124,35 @@ extras["rllib"] = extras["tune"] + [ "scipy", ] -extras["streaming"] = ["msgpack >= 0.6.2"] +extras["streaming"] = [] extras["all"] = list(set(chain.from_iterable(extras.values()))) +# These are the main dependencies for users of ray. This list +# should be carefully curated. If you change it, please reflect +# the change in the matching section of requirements.txt +install_requires = [ + "aiohttp", + "aioredis", + "click >= 7.0", + "colorama", + "colorful", + "filelock", + "google", + "gpustat", + "grpcio >= 1.28.1", + "jsonschema", + "msgpack >= 1.0.0, < 2.0.0", + "numpy >= 1.16", + "protobuf >= 3.8.0", + "py-spy >= 0.2.0", + "pyyaml", + "requests", + "redis >= 3.3.2, < 3.5.0", + "opencensus", + "prometheus_client >= 0.7.1", +] + def is_native_windows_or_msys(): """Check to see if we are running on native Windows, @@ -255,7 +284,11 @@ def build(build_python, build_java): version_info = bazel_invoke(subprocess.check_output, ["--version"]) bazel_version_str = version_info.rstrip().decode("utf-8").split(" ", 1)[1] - bazel_version = tuple(map(int, bazel_version_str.split("."))) + bazel_version_split = bazel_version_str.split(".") + bazel_version_digits = [ + "".join(takewhile(str.isdigit, s)) for s in bazel_version_split + ] + bazel_version = tuple(map(int, bazel_version_digits)) if bazel_version < SUPPORTED_BAZEL: logger.warning("Expected Bazel version {} but found {}".format( ".".join(map(str, SUPPORTED_BAZEL)), bazel_version_str)) @@ -305,29 +338,6 @@ def find_version(*filepath): raise RuntimeError("Unable to find version string.") -install_requires = [ - "aiohttp", - "aioredis", - "click >= 7.0", - "colorama", - "colorful", - "filelock", - "google", - "gpustat", - "grpcio >= 1.28.1", - "jsonschema", - "msgpack >= 0.6.0, < 2.0.0", - "numpy >= 1.16", - "protobuf >= 3.8.0", - "py-spy >= 0.2.0", - "pyyaml", - "requests", - "redis >= 3.3.2, < 3.5.0", - "opencensus", - "prometheus_client >= 0.7.1", -] - - def pip_run(build_ext): build(True, BUILD_JAVA)