diff --git a/ci/travis/check_import_order.py b/ci/travis/check_import_order.py index bf349ca89..26536fbfd 100644 --- a/ci/travis/check_import_order.py +++ b/ci/travis/check_import_order.py @@ -1,6 +1,6 @@ """ This script ensures python files conform to ray's import ordering rules. -In particular, we make sure psutil and setproctitle is imported _after_ +In particular, we make sure psutil is imported _after_ importing ray due to our bundling of the two libraries. Usage: @@ -22,7 +22,6 @@ def check_import(file): check_to_lines = { "import ray": -1, "import psutil": -1, - "import setproctitle": -1 } with io.open(file, "r", encoding="utf-8") as f: @@ -41,7 +40,7 @@ def check_import(file): line) and check_to_lines[check] == -1: check_to_lines[check] = i - for import_lib in ["import psutil", "import setproctitle"]: + for import_lib in ["import psutil"]: if check_to_lines[import_lib] != -1: import_psutil_line = check_to_lines[import_lib] import_ray_line = check_to_lines["import ray"] diff --git a/ci/travis/format.sh b/ci/travis/format.sh index 3f4b753f4..3578f3dd0 100755 --- a/ci/travis/format.sh +++ b/ci/travis/format.sh @@ -323,7 +323,7 @@ else fi # Ensure import ordering -# Make sure that for every import psutil; import setproctitle +# Make sure that for every import psutil # There's a import ray above it. PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python} diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh index 7bc73e967..4d5b67e2b 100755 --- a/ci/travis/install-dependencies.sh +++ b/ci/travis/install-dependencies.sh @@ -333,7 +333,7 @@ install_dependencies() { install_node fi - CC=gcc pip install psutil setproctitle==1.1.10 --target="${WORKSPACE_DIR}/python/ray/thirdparty_files" + CC=gcc pip install psutil --target="${WORKSPACE_DIR}/python/ray/thirdparty_files" } install_dependencies "$@" diff --git a/ci/travis/test-wheels.sh b/ci/travis/test-wheels.sh index f9131acd5..a6e5d2101 100755 --- a/ci/travis/test-wheels.sh +++ b/ci/travis/test-wheels.sh @@ -83,7 +83,7 @@ if [[ "$platform" == "linux" ]]; then # Check that the other wheels are present. NUMBER_OF_WHEELS="$(find "$ROOT_DIR"/../../.whl/ -mindepth 1 -maxdepth 1 -name "*.whl" | wc -l)" - if [[ "$NUMBER_OF_WHEELS" != "3" ]]; then + if [[ "$NUMBER_OF_WHEELS" != "4" ]]; then echo "Wrong number of wheels found." ls -l "$ROOT_DIR/../.whl/" exit 2 @@ -91,10 +91,11 @@ if [[ "$platform" == "linux" ]]; then elif [[ "$platform" == "macosx" ]]; then MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions - PY_WHEEL_VERSIONS=("36" "37" "38") + PY_WHEEL_VERSIONS=("36" "37" "38" "39") PY_MMS=("3.6" "3.7" - "3.8") + "3.8" + "3.9") for ((i=0; i<${#PY_MMS[@]}; ++i)); do PY_MM="${PY_MMS[i]}" diff --git a/doc/source/installation.rst b/doc/source/installation.rst index b85ff20d2..890db304e 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -49,19 +49,23 @@ instead of the ones above: =================== =================== ====================== Linux MacOS Windows (experimental) =================== =================== ====================== +`Linux Python 3.9`_ `MacOS Python 3.9`_ `Windows Python 3.9`_ `Linux Python 3.8`_ `MacOS Python 3.8`_ `Windows Python 3.8`_ `Linux Python 3.7`_ `MacOS Python 3.7`_ `Windows Python 3.7`_ `Linux Python 3.6`_ `MacOS Python 3.6`_ `Windows Python 3.6`_ =================== =================== ====================== +.. _`Linux Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp39-cp39-manylinux2014_x86_64.whl .. _`Linux Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp38-cp38-manylinux2014_x86_64.whl .. _`Linux Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl .. _`Linux Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl +.. _`MacOS Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp39-cp39-macosx_10_13_x86_64.whl .. _`MacOS Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp38-cp38-macosx_10_13_x86_64.whl .. _`MacOS Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-macosx_10_13_intel.whl .. _`MacOS Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-macosx_10_13_intel.whl +.. _`Windows Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp39-cp39-win_amd64.whl .. _`Windows Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp38-cp38-win_amd64.whl .. _`Windows Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-win_amd64.whl .. _`Windows Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp36-cp36m-win_amd64.whl @@ -336,32 +340,3 @@ that you've cloned the git repository. .. code-block:: bash python -m pytest -v python/ray/tests/test_mini.py - -Troubleshooting ---------------- - -If importing Ray (``python3 -c "import ray"``) in your development clone results -in this error: - -.. code-block:: python - - Traceback (most recent call last): - File "", line 1, in - File ".../ray/python/ray/__init__.py", line 63, in - import ray._raylet # noqa: E402 - File "python/ray/_raylet.pyx", line 98, in init ray._raylet - import ray.memory_monitor as memory_monitor - File ".../ray/python/ray/memory_monitor.py", line 9, in - import psutil # noqa E402 - File ".../ray/python/ray/thirdparty_files/psutil/__init__.py", line 159, in - from . import _psosx as _psplatform - File ".../ray/python/ray/thirdparty_files/psutil/_psosx.py", line 15, in - from . import _psutil_osx as cext - ImportError: cannot import name '_psutil_osx' from partially initialized module 'psutil' (most likely due to a circular import) (.../ray/python/ray/thirdparty_files/psutil/__init__.py) - -Then you should run the following commands: - -.. code-block:: bash - - rm -rf python/ray/thirdparty_files/ - python3 -m pip install setproctitle diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh index 60b35ed04..38355d407 100755 --- a/python/build-wheel-macos.sh +++ b/python/build-wheel-macos.sh @@ -16,19 +16,21 @@ DOWNLOAD_DIR=python_downloads NODE_VERSION="14" PY_VERSIONS=("3.6.1" "3.7.0" - "3.8.2") + "3.8.2" + "3.9.0") PY_INSTS=("python-3.6.1-macosx10.6.pkg" "python-3.7.0-macosx10.6.pkg" - "python-3.8.2-macosx10.9.pkg") + "python-3.8.2-macosx10.9.pkg" + "python-3.9.0-macosx10.9.pkg") PY_MMS=("3.6" "3.7" - "3.8") + "3.8" + "3.9") -# The minimum supported numpy version is 1.14, see -# https://issues.apache.org/jira/browse/ARROW-3141 -NUMPY_VERSIONS=("1.14.5" - "1.14.5" - "1.14.5") +NUMPY_VERSIONS=("1.19.3" + "1.19.3" + "1.19.3" + "1.19.3") ./ci/travis/install-bazel.sh diff --git a/python/build-wheel-manylinux2014.sh b/python/build-wheel-manylinux2014.sh index 490ec256e..68d02ed80 100755 --- a/python/build-wheel-manylinux2014.sh +++ b/python/build-wheel-manylinux2014.sh @@ -14,13 +14,13 @@ chmod +x /usr/bin/nproc NODE_VERSION="14" PYTHONS=("cp36-cp36m" "cp37-cp37m" - "cp38-cp38") + "cp38-cp38" + "cp39-cp39") -# The minimum supported numpy version is 1.14, see -# https://issues.apache.org/jira/browse/ARROW-3141 -NUMPY_VERSIONS=("1.14.5" - "1.14.5" - "1.14.5") +NUMPY_VERSIONS=("1.19.3" + "1.19.3" + "1.19.3" + "1.19.3") yum -y install unzip zip sudo yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel xz diff --git a/python/ray/__init__.py b/python/ray/__init__.py index 848ea5910..e1f4af708 100644 --- a/python/ray/__init__.py +++ b/python/ray/__init__.py @@ -37,7 +37,7 @@ pickle5_path = os.path.join( os.path.abspath(os.path.dirname(__file__)), "pickle5_files") sys.path.insert(0, pickle5_path) -# Importing psutil & setproctitle. Must be before ray._raylet is initialized. +# Importing psutil. Must be before ray._raylet is initialized. thirdparty_files = os.path.join( os.path.abspath(os.path.dirname(__file__)), "thirdparty_files") sys.path.insert(0, thirdparty_files) diff --git a/python/ray/tests/test_actor.py b/python/ray/tests/test_actor.py index 4db4bdd4b..76c3bf373 100644 --- a/python/ray/tests/test_actor.py +++ b/python/ray/tests/test_actor.py @@ -10,17 +10,13 @@ except ImportError: import sys import tempfile import datetime +import setproctitle from ray.test_utils import client_test_enabled from ray.test_utils import wait_for_condition from ray.test_utils import wait_for_pid_to_exit from ray.tests.client_test_utils import create_remote_signal_actor -import ray -# NOTE: We have to import setproctitle after ray because we bundle setproctitle -# with ray. -import setproctitle # noqa - def test_caching_actors(shutdown_only): # Test defining actors before ray.init() has been called. diff --git a/python/ray/tests/test_advanced_3.py b/python/ray/tests/test_advanced_3.py index b1bc25fbb..864dfe20b 100644 --- a/python/ray/tests/test_advanced_3.py +++ b/python/ray/tests/test_advanced_3.py @@ -10,6 +10,7 @@ import time import numpy as np import pickle import pytest +import setproctitle import ray import ray.ray_constants as ray_constants @@ -17,7 +18,6 @@ import ray.util.accelerators import ray.cluster_utils import ray.test_utils from ray import resource_spec -import setproctitle import subprocess from ray.test_utils import (check_call_ray, RayTestTimeoutException, diff --git a/python/requirements.txt b/python/requirements.txt index 28c387fde..426cd5b89 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -23,6 +23,7 @@ py-spy >= 0.2.0 pyyaml redis >= 3.5.0 requests +setproctitle ## setup.py extras atari_py diff --git a/python/setup.py b/python/setup.py index 62fc4b694..5dfdbf033 100644 --- a/python/setup.py +++ b/python/setup.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) # before these files have been created, so we have to move the files # manually. -SUPPORTED_PYTHONS = [(3, 6), (3, 7), (3, 8)] +SUPPORTED_PYTHONS = [(3, 6), (3, 7), (3, 8), (3, 9)] SUPPORTED_BAZEL = (3, 2, 0) ROOT_DIR = os.path.dirname(__file__) @@ -138,11 +138,13 @@ install_requires = [ "grpcio >= 1.28.1", "jsonschema", "msgpack >= 1.0.0, < 2.0.0", - "numpy >= 1.16", + "numpy >= 1.16; python_version < '3.9'", + "numpy >= 1.19.3; python_version >= '3.9'", "protobuf >= 3.8.0", "py-spy >= 0.2.0", "pyyaml", "requests", + "setproctitle", "redis >= 3.5.0", "opencensus", "prometheus_client >= 0.7.1", @@ -270,7 +272,7 @@ def build(build_python, build_java): # that certain flags will not be passed along such as --user or sudo. # TODO(rkn): Fix this. if not os.getenv("SKIP_THIRDPARTY_INSTALL"): - pip_packages = ["psutil", "setproctitle==1.1.10"] + pip_packages = ["psutil"] subprocess.check_call( [ sys.executable, "-m", "pip", "install", "-q", @@ -439,6 +441,12 @@ setuptools.setup( url="https://github.com/ray-project/ray", keywords=("ray distributed parallel machine-learning " "reinforcement-learning deep-learning python"), + classifiers=[ + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + ], packages=setuptools.find_packages(), cmdclass={"build_ext": build_ext}, # The BinaryDistribution argument triggers build_ext.