From 1cfadf032e7068f942cde76f47d6528942320f3a Mon Sep 17 00:00:00 2001 From: Robert Nishihara Date: Sat, 21 Sep 2019 18:03:10 -0700 Subject: [PATCH] Properly test Python wheels in Travis. (#5749) --- .travis.yml | 2 +- ci/travis/install-dependencies.sh | 3 +-- ci/travis/test-wheels.sh | 39 +++++++++++++++++++------------ python/build-wheel-macos.sh | 2 ++ python/build-wheel-manylinux1.sh | 7 ++++++ 5 files changed, 35 insertions(+), 18 deletions(-) diff --git a/.travis.yml b/.travis.yml index 650fdace8..ab1562bd0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,7 +47,7 @@ matrix: - export PATH="$HOME/miniconda/bin:$PATH" - cd doc - pip install -q -r requirements-doc.txt - - pip install yapf==0.23.0 + - pip install -q yapf==0.23.0 - sphinx-build -W -b html -d _build/doctrees source _build/html - cd .. # Run Python linting, ignore dict vs {} (C408), others are defaults diff --git a/ci/travis/install-dependencies.sh b/ci/travis/install-dependencies.sh index b1fa9c2ce..33d296725 100755 --- a/ci/travis/install-dependencies.sh +++ b/ci/travis/install-dependencies.sh @@ -59,8 +59,7 @@ elif [[ "$LINT" == "1" ]]; then bash miniconda.sh -b -p $HOME/miniconda export PATH="$HOME/miniconda/bin:$PATH" # Install Python linting tools. - pip install -q flake8==3.7.7 flake8-comprehensions - pip install flake8-quotes==2.0.0 + pip install -q flake8==3.7.7 flake8-comprehensions flake8-quotes==2.0.0 elif [[ "$LINUX_WHEELS" == "1" ]]; then sudo apt-get install docker sudo usermod -a -G docker travis diff --git a/ci/travis/test-wheels.sh b/ci/travis/test-wheels.sh index 738dc6106..bb325ca14 100755 --- a/ci/travis/test-wheels.sh +++ b/ci/travis/test-wheels.sh @@ -21,44 +21,52 @@ else exit 1 fi -TEST_SCRIPT=$ROOT_DIR/../../python/ray/tests/test_microbenchmarks.py +TEST_SCRIPT="tests/test_microbenchmarks.py" if [[ "$platform" == "linux" ]]; then # First test Python 2.7. # Install miniconda. wget https://repo.continuum.io/miniconda/Miniconda2-4.5.4-Linux-x86_64.sh -O miniconda2.sh - bash miniconda2.sh -b -p $HOME/miniconda2 + bash miniconda2.sh -b -p "$HOME/miniconda2" + + PYTHON_EXE=$HOME/miniconda2/bin/python + PIP_CMD=$HOME/miniconda2/bin/pip # Find the right wheel by grepping for the Python version. - PYTHON_WHEEL=$(find $ROOT_DIR/../../.whl -type f -maxdepth 1 -print | grep -m1 '27') + PYTHON_WHEEL=$(find "$ROOT_DIR/../../.whl" -type f -maxdepth 1 -print | grep -m1 '27') # Install the wheel. - $HOME/miniconda2/bin/pip install $PYTHON_WHEEL + $PIP_CMD install -q "$PYTHON_WHEEL" # Run a simple test script to make sure that the wheel works. - $HOME/miniconda2/bin/python $TEST_SCRIPT + INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)") + $PYTHON_EXE -m pytest -v "$INSTALLED_RAY_DIRECTORY/$TEST_SCRIPT" # Now test Python 3.6. # Install miniconda. wget https://repo.continuum.io/miniconda/Miniconda3-4.5.4-Linux-x86_64.sh -O miniconda3.sh - bash miniconda3.sh -b -p $HOME/miniconda3 + bash miniconda3.sh -b -p "$HOME/miniconda3" + + PYTHON_EXE=$HOME/miniconda3/bin/python + PIP_CMD=$HOME/miniconda3/bin/pip # Find the right wheel by grepping for the Python version. - PYTHON_WHEEL=$(find $ROOT_DIR/../../.whl -type f -maxdepth 1 -print | grep -m1 '36') + PYTHON_WHEEL=$(find "$ROOT_DIR/../../.whl" -type f -maxdepth 1 -print | grep -m1 '36') # Install the wheel. - $HOME/miniconda3/bin/pip install $PYTHON_WHEEL + $PIP_CMD install -q "$PYTHON_WHEEL" # Run a simple test script to make sure that the wheel works. - $HOME/miniconda3/bin/python $TEST_SCRIPT + INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)") + $PYTHON_EXE -m pytest -v "$INSTALLED_RAY_DIRECTORY/$TEST_SCRIPT" # Check that the other wheels are present. - NUMBER_OF_WHEELS=$(ls -1q $ROOT_DIR/../../.whl/*.whl | wc -l) + NUMBER_OF_WHEELS=$(ls -1q "$ROOT_DIR"/../../.whl/*.whl | wc -l) if [[ "$NUMBER_OF_WHEELS" != "4" ]]; then echo "Wrong number of wheels found." - ls -l $ROOT_DIR/../.whl/ + ls -l "$ROOT_DIR/../.whl/" exit 2 fi @@ -79,16 +87,17 @@ elif [[ "$platform" == "macosx" ]]; then PY_WHEEL_VERSION=${PY_WHEEL_VERSIONS[i]} PYTHON_EXE=$MACPYTHON_PY_PREFIX/$PY_MM/bin/python$PY_MM - PIP_CMD="$(dirname $PYTHON_EXE)/pip$PY_MM" + PIP_CMD="$(dirname "$PYTHON_EXE")/pip$PY_MM" # Find the appropriate wheel by grepping for the Python version. - PYTHON_WHEEL=$(find $ROOT_DIR/../../.whl -type f -maxdepth 1 -print | grep -m1 "$PY_WHEEL_VERSION") + PYTHON_WHEEL=$(find "$ROOT_DIR/../../.whl" -type f -maxdepth 1 -print | grep -m1 "$PY_WHEEL_VERSION") # Install the wheel. - $PIP_CMD install $PYTHON_WHEEL + $PIP_CMD install -q "$PYTHON_WHEEL" # Run a simple test script to make sure that the wheel works. - $PYTHON_EXE $TEST_SCRIPT + INSTALLED_RAY_DIRECTORY=$(dirname "$($PYTHON_EXE -u -c "import ray; print(ray.__file__)" | tail -n1)") + $PYTHON_EXE -m pytest -v "$INSTALLED_RAY_DIRECTORY/$TEST_SCRIPT" done else echo "Unrecognized environment." diff --git a/python/build-wheel-macos.sh b/python/build-wheel-macos.sh index 2fa4ab57b..8675245a0 100755 --- a/python/build-wheel-macos.sh +++ b/python/build-wheel-macos.sh @@ -79,3 +79,5 @@ for ((i=0; i<${#PY_VERSIONS[@]}; ++i)); do mv dist/*.whl ../.whl/ popd done + +git clean -f -f -x -d -e .whl -e $DOWNLOAD_DIR diff --git a/python/build-wheel-manylinux1.sh b/python/build-wheel-manylinux1.sh index 332f79e4c..c73846285 100755 --- a/python/build-wheel-manylinux1.sh +++ b/python/build-wheel-manylinux1.sh @@ -1,5 +1,10 @@ #!/bin/bash +set -x + +# Cause the script to exit if a single command fails. +set -e + cat << EOF > "/usr/bin/nproc" #!/bin/bash echo 10 @@ -48,6 +53,8 @@ for ((i=0; i<${#PYTHONS[@]}; ++i)); do popd done +git clean -f -f -x -d -e .whl + # Rename the wheels so that they can be uploaded to PyPI. TODO(rkn): This is a # hack, we should use auditwheel instead. pushd .whl