From 15dbd88c5d9a5a2e68742aea4d83b73e896e89b1 Mon Sep 17 00:00:00 2001 From: mehrdadn Date: Wed, 1 Apr 2020 10:03:23 -0700 Subject: [PATCH] Python 3.8 compatibility (#7754) --- .bazelrc | 3 + .travis.yml | 6 +- bazel/ray_deps_setup.bzl | 4 +- build.sh | 2 +- ci/travis/test-wheels.sh | 96 ++++++++++++++-------- python/build-wheel-macos.sh | 12 ++- python/build-wheel-manylinux1.sh | 6 +- python/ray/cloudpickle/cloudpickle_fast.py | 2 +- python/setup.py | 2 +- 9 files changed, 87 insertions(+), 46 deletions(-) diff --git a/.bazelrc b/.bazelrc index a36363fae..c8c524b55 100644 --- a/.bazelrc +++ b/.bazelrc @@ -21,6 +21,9 @@ build --per_file_copt="-\\.(asm|S)$,\\.pb\\.cc$@-w" build --per_file_copt="-\\.(asm|S)$,external/.*@-w" # Ignore this warning since it's impractical to fix in the relevant headers build --per_file_copt="-\\.(asm|S)$,.*/ray/raylet/reconstruction_policy_test\\.cc@-Wno-inconsistent-missing-override" +# Ignore this warning since CPython and Cython have issue removing deprecated tp_print on MacOS +build --per_file_copt="-\\.(asm|S)$,.*/_raylet\\.cpp@-Wno-deprecated-declarations" +build --per_file_copt="-\\.(asm|S)$,.*/_streaming\\.cpp@-Wno-deprecated-declarations" # Ignore minor warnings for host tools, which we generally can't control build --host_copt="-Wno-builtin-macro-redefined" build --host_copt="-Wno-inconsistent-missing-override" diff --git a/.travis.yml b/.travis.yml index 27e1bb2f8..bbe50b343 100644 --- a/.travis.yml +++ b/.travis.yml @@ -148,13 +148,13 @@ matrix: # This command should be kept in sync with ray/python/README-building-wheels.md, # except the `$MOUNT_BAZEL_CACHE` part. - - ./ci/suppress_output docker run -e TRAVIS_COMMIT=$TRAVIS_COMMIT --rm -w /ray -v `pwd`:/ray $MOUNT_BAZEL_CACHE -ti rayproject/arrow_linux_x86_64_base:ARROW-5631 /ray/python/build-wheel-manylinux1.sh + - ./ci/suppress_output docker run --rm -w /ray -v `pwd`:/ray $MOUNT_BAZEL_CACHE -e TRAVIS_COMMIT=$TRAVIS_COMMIT -ti rayproject/arrow_linux_x86_64_base:python-3.8.0 /ray/python/build-wheel-manylinux1.sh script: - if [ $RAY_CI_LINUX_WHEELS_AFFECTED != "1" ]; then exit; fi # Explicitly sleep 60 seconds for logs to go through - - ./ci/travis/test-wheels.sh || cat /tmp/ray/session_latest/logs/* || (sleep 60 && false) + - ./ci/travis/test-wheels.sh || { cat /tmp/ray/session_latest/logs/* && sleep 60 && false; } cache: false # Build MacOS wheels. @@ -172,7 +172,7 @@ matrix: - if [ $RAY_CI_MACOS_WHEELS_AFFECTED != "1" ]; then exit; fi # Explicitly sleep 60 seconds for logs to go through - - ./ci/travis/test-wheels.sh || cat /tmp/ray/session_latest/logs/* || (sleep 60 && false) + - ./ci/travis/test-wheels.sh || { cat /tmp/ray/session_latest/logs/* && sleep 60 && false; } # RLlib: Learning tests (from rllib/tuned_examples/regression_tests/*.yaml). - os: linux diff --git a/bazel/ray_deps_setup.bzl b/bazel/ray_deps_setup.bzl index 4211a19d9..eb796be87 100644 --- a/bazel/ray_deps_setup.bzl +++ b/bazel/ray_deps_setup.bzl @@ -178,8 +178,8 @@ def ray_deps_setup(): auto_http_archive( name = "cython", build_file = True, - url = "https://github.com/cython/cython/archive/49414dbc7ddc2ca2979d6dbe1e44714b10d72e7e.tar.gz", - sha256 = "0b697ac90d1e46842c7cbbf5f4a1bde5b7b41037c611167417115337e3756eaa", + url = "https://github.com/cython/cython/archive/26cb654dcf4ed1b1858daf16b39fd13406b1ac64.tar.gz", + sha256 = "d21e155ac9a455831f81608bb06620e4a1d75012a630faf11f4c25ad10cfc9bb", ) auto_http_archive( diff --git a/build.sh b/build.sh index 592e472fd..88f078785 100755 --- a/build.sh +++ b/build.sh @@ -6,7 +6,7 @@ set -x set -e # As the supported Python versions change, edit this array: -SUPPORTED_PYTHONS=( "3.5" "3.6" "3.7" ) +SUPPORTED_PYTHONS=( "3.5" "3.6" "3.7" "3.8" ) ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE:-$0}")"; pwd) diff --git a/ci/travis/test-wheels.sh b/ci/travis/test-wheels.sh index bb714fd53..cd3276fd2 100755 --- a/ci/travis/test-wheels.sh +++ b/ci/travis/test-wheels.sh @@ -21,41 +21,72 @@ else exit 1 fi -TEST_SCRIPT="$TRAVIS_BUILD_DIR/python/ray/tests/test_microbenchmarks.py" +TEST_DIR="$TRAVIS_BUILD_DIR/python/ray/tests" +TEST_SCRIPTS=("$TEST_DIR/test_microbenchmarks.py" "$TEST_DIR/test_basic.py") UI_TEST_SCRIPT="$TRAVIS_BUILD_DIR/python/ray/tests/test_webui.py" +function retry { + local n=1 + local max=3 + + while true; do + "$@" && break || { + if [[ $n -lt $max ]]; then + ((n++)) + echo "Command failed. Attempt $n/$max:" + else + echo "The command has failed after $n attempts." + exit 1 + fi + } + done +} + if [[ "$platform" == "linux" ]]; then - # Now test Python 3.6. - # Install miniconda. - wget --quiet https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O miniconda3.sh + PY_WHEEL_VERSIONS=("36" "37" "38") + PY_MMS=("3.6.9" + "3.7.6" + "3.8.2") + wget --quiet "https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh" -O miniconda3.sh bash miniconda3.sh -b -p "$HOME/miniconda3" + export PATH="$HOME/miniconda3/bin:$PATH" - PYTHON_EXE=$HOME/miniconda3/bin/python - PIP_CMD=$HOME/miniconda3/bin/pip + for ((i=0; i<${#PY_MMS[@]}; ++i)); do + PY_MM="${PY_MMS[i]}" + PY_WHEEL_VERSION="${PY_WHEEL_VERSIONS[i]}" - # Find the right wheel by grepping for the Python version. - PYTHON_WHEEL=$(find "$ROOT_DIR/../../.whl" -type f -maxdepth 1 -print | grep -m1 '36') + conda install -y python="${PY_MM}" - # Install the wheel. - $PIP_CMD install -q "$PYTHON_WHEEL" + PYTHON_EXE="$HOME/miniconda3/bin/python" + PIP_CMD="$HOME/miniconda3/bin/pip" - # Check that ray.__commit__ was set properly. - $PYTHON_EXE -u -c "import ray; print(ray.__commit__)" | grep $TRAVIS_COMMIT || (echo "ray.__commit__ not set properly!" && exit 1) + # Find the right wheel by grepping for the Python version. + PYTHON_WHEEL=$(find "$ROOT_DIR/../../.whl" -type f -maxdepth 1 -print | grep -m1 "$PY_WHEEL_VERSION") - # Install the dependencies to run the tests. - $PIP_CMD install -q aiohttp google grpcio pytest requests + # Install the wheel. + "$PIP_CMD" install -q "$PYTHON_WHEEL" - # Run a simple test script to make sure that the wheel works. - INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)") - $PYTHON_EXE "$TEST_SCRIPT" + # Check that ray.__commit__ was set properly. + "$PYTHON_EXE" -u -c "import ray; print(ray.__commit__)" | grep "$TRAVIS_COMMIT" || (echo "ray.__commit__ not set properly!" && exit 1) - # Run the UI test to make sure that the packaged UI works. - $PYTHON_EXE "$UI_TEST_SCRIPT" + # Install the dependencies to run the tests. + "$PIP_CMD" install -q aiohttp google grpcio pytest requests + + # Run a simple test script to make sure that the wheel works. + INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)") + + for SCRIPT in "${TEST_SCRIPTS[@]}"; do + retry "$PYTHON_EXE" "$SCRIPT" + done + + # Run the UI test to make sure that the packaged UI works. + retry "$PYTHON_EXE" "$UI_TEST_SCRIPT" + done # Check that the other wheels are present. NUMBER_OF_WHEELS=$(ls -1q "$ROOT_DIR"/../../.whl/*.whl | wc -l) - if [[ "$NUMBER_OF_WHEELS" != "3" ]]; then + if [[ "$NUMBER_OF_WHEELS" != "4" ]]; then echo "Wrong number of wheels found." ls -l "$ROOT_DIR/../.whl/" exit 2 @@ -63,37 +94,38 @@ if [[ "$platform" == "linux" ]]; then elif [[ "$platform" == "macosx" ]]; then MACPYTHON_PY_PREFIX=/Library/Frameworks/Python.framework/Versions + PY_WHEEL_VERSIONS=("35" "36" "37" "38") PY_MMS=("3.5" "3.6" - "3.7") - # This array is just used to find the right wheel. - PY_WHEEL_VERSIONS=("35" - "36" - "37") + "3.7" + "3.8") for ((i=0; i<${#PY_MMS[@]}; ++i)); do - PY_MM=${PY_MMS[i]} - PY_WHEEL_VERSION=${PY_WHEEL_VERSIONS[i]} + PY_MM="${PY_MMS[i]}" - PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM + PY_WHEEL_VERSION="${PY_WHEEL_VERSIONS[i]}" + + PYTHON_EXE="$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM" PIP_CMD="$(dirname "$PYTHON_EXE")/pip$PY_MM" # Find the appropriate wheel by grepping for the Python version. PYTHON_WHEEL=$(find "$ROOT_DIR/../../.whl" -type f -maxdepth 1 -print | grep -m1 "$PY_WHEEL_VERSION") # Install the wheel. - $PIP_CMD install -q "$PYTHON_WHEEL" + "$PIP_CMD" install -q "$PYTHON_WHEEL" # Install the dependencies to run the tests. - $PIP_CMD install -q aiohttp google grpcio pytest requests + "$PIP_CMD" install -q aiohttp google grpcio pytest requests # Run a simple test script to make sure that the wheel works. INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)") - $PYTHON_EXE "$TEST_SCRIPT" + for SCRIPT in "${TEST_SCRIPTS[@]}"; do + retry "$PYTHON_EXE" "$SCRIPT" + done if (( $(echo "$PY_MM >= 3.0" | bc) )); then # Run the UI test to make sure that the packaged UI works. - $PYTHON_EXE "$UI_TEST_SCRIPT" + retry "$PYTHON_EXE" "$UI_TEST_SCRIPT" fi done diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh index 7960866b7..17ef828ee 100755 --- a/python/build-wheel-macos.sh +++ b/python/build-wheel-macos.sh @@ -15,17 +15,21 @@ DOWNLOAD_DIR=python_downloads PY_VERSIONS=("3.5.3" "3.6.1" - "3.7.0") + "3.7.0" + "3.8.2") PY_INSTS=("python-3.5.3-macosx10.6.pkg" "python-3.6.1-macosx10.6.pkg" - "python-3.7.0-macosx10.6.pkg") + "python-3.7.0-macosx10.6.pkg" + "python-3.8.2-macosx10.9.pkg") PY_MMS=("3.5" "3.6" - "3.7") + "3.7" + "3.8") # The minimum supported numpy version is 1.14, see # https://issues.apache.org/jira/browse/ARROW-3141 NUMPY_VERSIONS=("1.14.5" + "1.14.5" "1.14.5" "1.14.5") @@ -76,7 +80,7 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do $PIP_CMD install -q setuptools_scm==3.1.0 # Fix the numpy version because this will be the oldest numpy version we can # support. - $PIP_CMD install -q numpy==$NUMPY_VERSION cython==0.29.0 + $PIP_CMD install -q numpy==$NUMPY_VERSION cython==0.29.15 # Install wheel to avoid the error "invalid command 'bdist_wheel'". $PIP_CMD install -q wheel # Set the commit SHA in __init__.py. diff --git a/python/build-wheel-manylinux1.sh b/python/build-wheel-manylinux1.sh index ea4926c77..3579a04af 100755 --- a/python/build-wheel-manylinux1.sh +++ b/python/build-wheel-manylinux1.sh @@ -13,11 +13,13 @@ chmod +x /usr/bin/nproc PYTHONS=("cp35-cp35m" "cp36-cp36m" - "cp37-cp37m") + "cp37-cp37m" + "cp38-cp38") # The minimum supported numpy version is 1.14, see # https://issues.apache.org/jira/browse/ARROW-3141 NUMPY_VERSIONS=("1.14.5" + "1.14.5" "1.14.5" "1.14.5") @@ -55,7 +57,7 @@ for ((i=0; i<${#PYTHONS[@]}; ++i)); do pushd python # Fix the numpy version because this will be the oldest numpy version we can # support. - /opt/python/${PYTHON}/bin/pip install -q numpy==${NUMPY_VERSION} cython==0.29.0 + /opt/python/${PYTHON}/bin/pip install -q numpy==${NUMPY_VERSION} cython==0.29.15 # Set the commit SHA in __init__.py. if [ -n "$TRAVIS_COMMIT" ]; then sed -i.bak "s/{{RAY_COMMIT_SHA}}/$TRAVIS_COMMIT/g" ray/__init__.py && rm ray/__init__.py.bak diff --git a/python/ray/cloudpickle/cloudpickle_fast.py b/python/ray/cloudpickle/cloudpickle_fast.py index ccfee6c7a..65d142e3c 100644 --- a/python/ray/cloudpickle/cloudpickle_fast.py +++ b/python/ray/cloudpickle/cloudpickle_fast.py @@ -417,7 +417,7 @@ def _numpy_frombuffer(buffer, dtype, shape, order): array = _frombuffer(buffer, dtype, shape, order) # Unfortunately, numpy does not follow the standard, so we still # have to set the readonly flag for it here. - array.setflags(write=not buffer.readonly) + array.setflags(write=isinstance(buffer, bytearray) or not buffer.readonly) return array diff --git a/python/setup.py b/python/setup.py index ba3c4800c..1013832e0 100644 --- a/python/setup.py +++ b/python/setup.py @@ -191,7 +191,7 @@ setup( # The BinaryDistribution argument triggers build_ext. distclass=BinaryDistribution, install_requires=requires, - setup_requires=["cython >= 0.29"], + setup_requires=["cython >= 0.29.14"], extras_require=extras, entry_points={ "console_scripts": [