From 903a2066a7f4836f6b61fef77f2666a86c47c215 Mon Sep 17 00:00:00 2001 From: acxz <17132214+acxz@users.noreply.github.com> Date: Wed, 9 Dec 2020 23:00:36 -0500 Subject: [PATCH 1/8] Add support for Python 3.9 --- ci/travis/test-wheels.sh | 7 ++++--- doc/source/installation.rst | 4 ++++ python/build-wheel-macos.sh | 10 +++++++--- python/build-wheel-manylinux2014.sh | 4 +++- python/setup.py | 8 +++++++- 5 files changed, 25 insertions(+), 8 deletions(-) diff --git a/ci/travis/test-wheels.sh b/ci/travis/test-wheels.sh index f9131acd5..a6e5d2101 100755 --- a/ci/travis/test-wheels.sh +++ b/ci/travis/test-wheels.sh @@ -83,7 +83,7 @@ if [[ "$platform" == "linux" ]]; then # Check that the other wheels are present. NUMBER_OF_WHEELS="$(find "$ROOT_DIR"/../../.whl/ -mindepth 1 -maxdepth 1 -name "*.whl" | wc -l)" - if [[ "$NUMBER_OF_WHEELS" != "3" ]]; then + if [[ "$NUMBER_OF_WHEELS" != "4" ]]; then echo "Wrong number of wheels found." ls -l "$ROOT_DIR/../.whl/" exit 2 @@ -91,10 +91,11 @@ if [[ "$platform" == "linux" ]]; then elif [[ "$platform" == "macosx" ]]; then MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions - PY_WHEEL_VERSIONS=("36" "37" "38") + PY_WHEEL_VERSIONS=("36" "37" "38" "39") PY_MMS=("3.6" "3.7" - "3.8") + "3.8" + "3.9") for ((i=0; i<${#PY_MMS[@]}; ++i)); do PY_MM="${PY_MMS[i]}" diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 445db0239..08b6450b5 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -49,19 +49,23 @@ instead of the ones above: =================== =================== ====================== Linux MacOS Windows (experimental) =================== =================== ====================== +`Linux Python 3.9`_ `MacOS Python 3.9`_ `Windows Python 3.9`_ `Linux Python 3.8`_ `MacOS Python 3.8`_ `Windows Python 3.8`_ `Linux Python 3.7`_ `MacOS Python 3.7`_ `Windows Python 3.7`_ `Linux Python 3.6`_ `MacOS Python 3.6`_ `Windows Python 3.6`_ =================== =================== ====================== +.. _`Linux Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp39-cp39-manylinux2014_x86_64.whl .. _`Linux Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-manylinux2014_x86_64.whl .. _`Linux Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl .. _`Linux Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-manylinux2014_x86_64.whl +.. _`MacOS Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp39-cp39-macosx_10_13_x86_64.whl .. _`MacOS Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-macosx_10_13_x86_64.whl .. _`MacOS Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-macosx_10_13_intel.whl .. _`MacOS Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-macosx_10_13_intel.whl +.. _`Windows Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp39-cp39-win_amd64.whl .. _`Windows Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp38-cp38-win_amd64.whl .. _`Windows Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp37-cp37m-win_amd64.whl .. _`Windows Python 3.6`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.1.0.dev0-cp36-cp36m-win_amd64.whl diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh index 60b35ed04..2ca4fd381 100755 --- a/python/build-wheel-macos.sh +++ b/python/build-wheel-macos.sh @@ -16,17 +16,21 @@ DOWNLOAD_DIR=python_downloads NODE_VERSION="14" PY_VERSIONS=("3.6.1" "3.7.0" - "3.8.2") + "3.8.2" + "3.9.0") PY_INSTS=("python-3.6.1-macosx10.6.pkg" "python-3.7.0-macosx10.6.pkg" - "python-3.8.2-macosx10.9.pkg") + "python-3.8.2-macosx10.9.pkg" + "python-3.9.0-macosx10.9.pkg") PY_MMS=("3.6" "3.7" - "3.8") + "3.8" + "3.9") # The minimum supported numpy version is 1.14, see # https://issues.apache.org/jira/browse/ARROW-3141 NUMPY_VERSIONS=("1.14.5" + "1.14.5" "1.14.5" "1.14.5") diff --git a/python/build-wheel-manylinux2014.sh b/python/build-wheel-manylinux2014.sh index 490ec256e..00d7cb0e8 100755 --- a/python/build-wheel-manylinux2014.sh +++ b/python/build-wheel-manylinux2014.sh @@ -14,11 +14,13 @@ chmod +x /usr/bin/nproc NODE_VERSION="14" PYTHONS=("cp36-cp36m" "cp37-cp37m" - "cp38-cp38") + "cp38-cp38" + "cp39-cp39") # The minimum supported numpy version is 1.14, see # https://issues.apache.org/jira/browse/ARROW-3141 NUMPY_VERSIONS=("1.14.5" + "1.14.5" "1.14.5" "1.14.5") diff --git a/python/setup.py b/python/setup.py index 2c82ad1d9..9a23aebb9 100644 --- a/python/setup.py +++ b/python/setup.py @@ -27,7 +27,7 @@ logger = logging.getLogger(__name__) # before these files have been created, so we have to move the files # manually. -SUPPORTED_PYTHONS = [(3, 6), (3, 7), (3, 8)] +SUPPORTED_PYTHONS = [(3, 6), (3, 7), (3, 8), (3, 9)] SUPPORTED_BAZEL = (3, 2, 0) ROOT_DIR = os.path.dirname(__file__) @@ -440,6 +440,12 @@ setuptools.setup( url="https://github.com/ray-project/ray", keywords=("ray distributed parallel machine-learning " "reinforcement-learning deep-learning python"), + classifiers=[ + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + ], packages=setuptools.find_packages(), cmdclass={"build_ext": build_ext}, # The BinaryDistribution argument triggers build_ext. From b07e5b9a120ce56e5015f30d13251d1a90af2718 Mon Sep 17 00:00:00 2001 From: acxz <17132214+acxz@users.noreply.github.com> Date: Wed, 9 Dec 2020 23:03:01 -0500 Subject: [PATCH 2/8] increase numpy version for py39 --- python/build-wheel-macos.sh | 10 ++++------ python/build-wheel-manylinux2014.sh | 10 ++++------ python/setup.py | 2 +- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh index 2ca4fd381..38355d407 100755 --- a/python/build-wheel-macos.sh +++ b/python/build-wheel-macos.sh @@ -27,12 +27,10 @@ PY_MMS=("3.6" "3.8" "3.9") -# The minimum supported numpy version is 1.14, see -# https://issues.apache.org/jira/browse/ARROW-3141 -NUMPY_VERSIONS=("1.14.5" - "1.14.5" - "1.14.5" - "1.14.5") +NUMPY_VERSIONS=("1.19.3" + "1.19.3" + "1.19.3" + "1.19.3") ./ci/travis/install-bazel.sh diff --git a/python/build-wheel-manylinux2014.sh b/python/build-wheel-manylinux2014.sh index 00d7cb0e8..68d02ed80 100755 --- a/python/build-wheel-manylinux2014.sh +++ b/python/build-wheel-manylinux2014.sh @@ -17,12 +17,10 @@ PYTHONS=("cp36-cp36m" "cp38-cp38" "cp39-cp39") -# The minimum supported numpy version is 1.14, see -# https://issues.apache.org/jira/browse/ARROW-3141 -NUMPY_VERSIONS=("1.14.5" - "1.14.5" - "1.14.5" - "1.14.5") +NUMPY_VERSIONS=("1.19.3" + "1.19.3" + "1.19.3" + "1.19.3") yum -y install unzip zip sudo yum -y install java-1.8.0-openjdk java-1.8.0-openjdk-devel xz diff --git a/python/setup.py b/python/setup.py index 9a23aebb9..27d256f5c 100644 --- a/python/setup.py +++ b/python/setup.py @@ -139,7 +139,7 @@ install_requires = [ "grpcio >= 1.28.1", "jsonschema", "msgpack >= 1.0.0, < 2.0.0", - "numpy >= 1.16", + "numpy >= 1.19.3", "protobuf >= 3.8.0", "py-spy >= 0.2.0", "pyyaml", From bd866d926d54f333d1cfb76bb98ea957a23f767f Mon Sep 17 00:00:00 2001 From: acxz <17132214+acxz@users.noreply.github.com> Date: Fri, 11 Dec 2020 22:07:24 -0500 Subject: [PATCH 3/8] debug sys.path for setproctitle --- python/ray/_raylet.pyx | 1 + 1 file changed, 1 insertion(+) diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index abea24000..6feaab7de 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -17,6 +17,7 @@ import os import pickle import sys import _thread +print(sys.path) import setproctitle from libc.stdint cimport ( From 2b3893830595c2b2d130f95409d57018fca2bd46 Mon Sep 17 00:00:00 2001 From: acxz <17132214+acxz@users.noreply.github.com> Date: Wed, 16 Dec 2020 11:06:33 -0500 Subject: [PATCH 4/8] remove extra newline --- doc/source/installation.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 40813d86a..151a4dbc4 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -55,7 +55,6 @@ instead of the ones above: `Linux Python 3.6`_ `MacOS Python 3.6`_ `Windows Python 3.6`_ =================== =================== ====================== - .. _`Linux Python 3.9`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp39-cp39-manylinux2014_x86_64.whl .. _`Linux Python 3.8`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp38-cp38-manylinux2014_x86_64.whl .. _`Linux Python 3.7`: https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-1.2.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl From c8d14eb3c5f2db71098c8fea56b7e4cd962d3490 Mon Sep 17 00:00:00 2001 From: acxz <17132214+acxz@users.noreply.github.com> Date: Wed, 16 Dec 2020 22:42:31 -0500 Subject: [PATCH 5/8] update setproctitle to use with py39 --- ci/travis/install-dependencies.sh | 2 +- python/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh index 7bc73e967..5d39c1c8b 100755 --- a/ci/travis/install-dependencies.sh +++ b/ci/travis/install-dependencies.sh @@ -333,7 +333,7 @@ install_dependencies() { install_node fi - CC=gcc pip install psutil setproctitle==1.1.10 --target="${WORKSPACE_DIR}/python/ray/thirdparty_files" + CC=gcc pip install psutil setproctitle --target="${WORKSPACE_DIR}/python/ray/thirdparty_files" } install_dependencies "$@" diff --git a/python/setup.py b/python/setup.py index 8be61e9dd..3687903b9 100644 --- a/python/setup.py +++ b/python/setup.py @@ -270,7 +270,7 @@ def build(build_python, build_java): # that certain flags will not be passed along such as --user or sudo. # TODO(rkn): Fix this. if not os.getenv("SKIP_THIRDPARTY_INSTALL"): - pip_packages = ["psutil", "setproctitle==1.1.10"] + pip_packages = ["psutil", "setproctitle"] subprocess.check_call( [ sys.executable, "-m", "pip", "install", "-q", From 020ad98f6f63a7b61e645d390b00bdb04c37e47c Mon Sep 17 00:00:00 2001 From: acxz <17132214+acxz@users.noreply.github.com> Date: Thu, 17 Dec 2020 00:36:12 -0500 Subject: [PATCH 6/8] install setproctitle from pypi instead of building from source --- ci/travis/check_import_order.py | 5 ++--- ci/travis/format.sh | 2 +- ci/travis/install-dependencies.sh | 2 +- doc/source/installation.rst | 29 ----------------------------- python/ray/__init__.py | 2 +- python/ray/tests/test_actor.py | 5 +---- python/ray/tests/test_advanced_3.py | 2 +- python/requirements.txt | 1 + python/setup.py | 3 ++- 9 files changed, 10 insertions(+), 41 deletions(-) diff --git a/ci/travis/check_import_order.py b/ci/travis/check_import_order.py index bf349ca89..26536fbfd 100644 --- a/ci/travis/check_import_order.py +++ b/ci/travis/check_import_order.py @@ -1,6 +1,6 @@ """ This script ensures python files conform to ray's import ordering rules. -In particular, we make sure psutil and setproctitle is imported _after_ +In particular, we make sure psutil is imported _after_ importing ray due to our bundling of the two libraries. Usage: @@ -22,7 +22,6 @@ def check_import(file): check_to_lines = { "import ray": -1, "import psutil": -1, - "import setproctitle": -1 } with io.open(file, "r", encoding="utf-8") as f: @@ -41,7 +40,7 @@ def check_import(file): line) and check_to_lines[check] == -1: check_to_lines[check] = i - for import_lib in ["import psutil", "import setproctitle"]: + for import_lib in ["import psutil"]: if check_to_lines[import_lib] != -1: import_psutil_line = check_to_lines[import_lib] import_ray_line = check_to_lines["import ray"] diff --git a/ci/travis/format.sh b/ci/travis/format.sh index 7a6b45751..bd0f04a0a 100755 --- a/ci/travis/format.sh +++ b/ci/travis/format.sh @@ -291,7 +291,7 @@ else fi # Ensure import ordering -# Make sure that for every import psutil; import setproctitle +# Make sure that for every import psutil # There's a import ray above it. PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE:-python} diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh index 5d39c1c8b..4d5b67e2b 100755 --- a/ci/travis/install-dependencies.sh +++ b/ci/travis/install-dependencies.sh @@ -333,7 +333,7 @@ install_dependencies() { install_node fi - CC=gcc pip install psutil setproctitle --target="${WORKSPACE_DIR}/python/ray/thirdparty_files" + CC=gcc pip install psutil --target="${WORKSPACE_DIR}/python/ray/thirdparty_files" } install_dependencies "$@" diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 151a4dbc4..890db304e 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -340,32 +340,3 @@ that you've cloned the git repository. .. code-block:: bash python -m pytest -v python/ray/tests/test_mini.py - -Troubleshooting ---------------- - -If importing Ray (``python3 -c "import ray"``) in your development clone results -in this error: - -.. code-block:: python - - Traceback (most recent call last): - File "", line 1, in - File ".../ray/python/ray/__init__.py", line 63, in - import ray._raylet # noqa: E402 - File "python/ray/_raylet.pyx", line 98, in init ray._raylet - import ray.memory_monitor as memory_monitor - File ".../ray/python/ray/memory_monitor.py", line 9, in - import psutil # noqa E402 - File ".../ray/python/ray/thirdparty_files/psutil/__init__.py", line 159, in - from . import _psosx as _psplatform - File ".../ray/python/ray/thirdparty_files/psutil/_psosx.py", line 15, in - from . import _psutil_osx as cext - ImportError: cannot import name '_psutil_osx' from partially initialized module 'psutil' (most likely due to a circular import) (.../ray/python/ray/thirdparty_files/psutil/__init__.py) - -Then you should run the following commands: - -.. code-block:: bash - - rm -rf python/ray/thirdparty_files/ - python3 -m pip install setproctitle diff --git a/python/ray/__init__.py b/python/ray/__init__.py index 848ea5910..e1f4af708 100644 --- a/python/ray/__init__.py +++ b/python/ray/__init__.py @@ -37,7 +37,7 @@ pickle5_path = os.path.join( os.path.abspath(os.path.dirname(__file__)), "pickle5_files") sys.path.insert(0, pickle5_path) -# Importing psutil & setproctitle. Must be before ray._raylet is initialized. +# Importing psutil. Must be before ray._raylet is initialized. thirdparty_files = os.path.join( os.path.abspath(os.path.dirname(__file__)), "thirdparty_files") sys.path.insert(0, thirdparty_files) diff --git a/python/ray/tests/test_actor.py b/python/ray/tests/test_actor.py index 05d53c1b3..3824463e1 100644 --- a/python/ray/tests/test_actor.py +++ b/python/ray/tests/test_actor.py @@ -10,15 +10,12 @@ except ImportError: import sys import tempfile import datetime +import setproctitle import ray import ray.test_utils import ray.cluster_utils -# NOTE: We have to import setproctitle after ray because we bundle setproctitle -# with ray. -import setproctitle - def test_caching_actors(shutdown_only): # Test defining actors before ray.init() has been called. diff --git a/python/ray/tests/test_advanced_3.py b/python/ray/tests/test_advanced_3.py index 7f1e8e639..3d6334e1f 100644 --- a/python/ray/tests/test_advanced_3.py +++ b/python/ray/tests/test_advanced_3.py @@ -10,6 +10,7 @@ import time import numpy as np import pickle import pytest +import setproctitle import ray import ray.ray_constants as ray_constants @@ -17,7 +18,6 @@ import ray.util.accelerators import ray.cluster_utils import ray.test_utils from ray import resource_spec -import setproctitle import subprocess from ray.test_utils import (check_call_ray, RayTestTimeoutException, diff --git a/python/requirements.txt b/python/requirements.txt index 28c387fde..426cd5b89 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -23,6 +23,7 @@ py-spy >= 0.2.0 pyyaml redis >= 3.5.0 requests +setproctitle ## setup.py extras atari_py diff --git a/python/setup.py b/python/setup.py index 3687903b9..03047d73b 100644 --- a/python/setup.py +++ b/python/setup.py @@ -143,6 +143,7 @@ install_requires = [ "py-spy >= 0.2.0", "pyyaml", "requests", + "setproctitle", "redis >= 3.5.0", "opencensus", "prometheus_client >= 0.7.1", @@ -270,7 +271,7 @@ def build(build_python, build_java): # that certain flags will not be passed along such as --user or sudo. # TODO(rkn): Fix this. if not os.getenv("SKIP_THIRDPARTY_INSTALL"): - pip_packages = ["psutil", "setproctitle"] + pip_packages = ["psutil"] subprocess.check_call( [ sys.executable, "-m", "pip", "install", "-q", From 84b689f32f6066294ab1c9be4ec55c9c1feca01c Mon Sep 17 00:00:00 2001 From: Akash Patel <17132214+acxz@users.noreply.github.com> Date: Thu, 17 Dec 2020 09:53:32 -0500 Subject: [PATCH 7/8] remove debug statement --- python/ray/_raylet.pyx | 1 - 1 file changed, 1 deletion(-) diff --git a/python/ray/_raylet.pyx b/python/ray/_raylet.pyx index 8a4d2ce6c..47215149a 100644 --- a/python/ray/_raylet.pyx +++ b/python/ray/_raylet.pyx @@ -17,7 +17,6 @@ import os import pickle import sys import _thread -print(sys.path) import setproctitle from libc.stdint cimport ( From 4c462d8107f8f6af010647b4324aac97f7b7d80b Mon Sep 17 00:00:00 2001 From: Akash Patel <17132214+acxz@users.noreply.github.com> Date: Fri, 18 Dec 2020 17:13:48 -0500 Subject: [PATCH 8/8] relax numpy req Co-authored-by: h-vetinari --- python/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/setup.py b/python/setup.py index 03047d73b..5dfdbf033 100644 --- a/python/setup.py +++ b/python/setup.py @@ -138,7 +138,8 @@ install_requires = [ "grpcio >= 1.28.1", "jsonschema", "msgpack >= 1.0.0, < 2.0.0", - "numpy >= 1.19.3", + "numpy >= 1.16; python_version < '3.9'", + "numpy >= 1.19.3; python_version >= '3.9'", "protobuf >= 3.8.0", "py-spy >= 0.2.0", "pyyaml",