From b9a02e0957600a196dc8245ae1446b1998b9b4d7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 13 Dec 2014 16:33:27 -0600 Subject: [PATCH 1/9] Add a script to create env variables and travis folding helpers Implement travis folding helpers Add a script to create env variables and travis folding helpers Implement travis folding helpers Source all of the shell scripts Remove since we are sourcing Fix syntax of travis fold commands Put the travis folds in the individual files Put the min test and flake in their own files Reinstate top level venv activation Remove unused script commands Fix travis_fold syntax --- .travis.yml | 23 +++++---------- ...tall_optional.sh => travis_install_all.sh} | 22 +++++++++++++++ ...{travis_setup.sh => travis_install_min.sh} | 4 +++ tools/{travis_test.sh => travis_test_all.sh} | 28 +++++++------------ tools/travis_test_min.sh | 10 +++++++ 5 files changed, 53 insertions(+), 34 deletions(-) rename tools/{travis_install_optional.sh => travis_install_all.sh} (74%) rename tools/{travis_setup.sh => travis_install_min.sh} (91%) rename tools/{travis_test.sh => travis_test_all.sh} (56%) create mode 100755 tools/travis_test_min.sh diff --git a/.travis.yml b/.travis.yml index 353f01bd..956f592b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,36 +25,27 @@ python: before_install: - export DISPLAY=:99.0 - sh -e /etc/init.d/xvfb start + - sudo apt-get update - export PYTHONWARNINGS="all" - export WHEELHOUSE="--no-index --find-links=http://travis-wheels.scikit-image.org/" - export TEST_ARGS="--exe --ignore-files=^_test -v --with-doctest --ignore-files=^setup.py$" - - export PIP_DEFAULT_TIMEOUT=60 - - sudo apt-get update - - travis_retry tools/travis_setup.sh + - travis_retry tools/travis_install_min.sh - source ~/venv/bin/activate - - which python - - python --version + - which python; python --version + - python check_bento_build.py - - - tools/header.py "Dependency versions" - tools/build_versions.py - install: - python setup.py build_ext --inplace - python setup.py install script: - - tools/header.py "Run all tests with minimum dependencies" - - nosetests $TEST_ARGS skimage - - - tools/header.py "Pep8 and Flake tests" - - flake8 --exit-zero --exclude=test_*,six.py skimage doc/examples viewer_examples - - - travis_retry tools/travis_install_optional.sh - - tools/travis_test.sh + - travis_retry tools/travis_test_min.sh + - travis_retry tools/travis_install_all.sh + - travis_retry tools/travis_test_all.sh after_success: - coveralls diff --git a/tools/travis_install_optional.sh b/tools/travis_install_all.sh similarity index 74% rename from tools/travis_install_optional.sh rename to tools/travis_install_all.sh index db8d3f32..1ef82e7a 100755 --- a/tools/travis_install_optional.sh +++ b/tools/travis_install_all.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -ex +echo -en "travis_fold:start:install.all\r" + tools/header.py "Install optional dependencies" # Install Qt and then update the Matplotlib settings @@ -48,3 +50,23 @@ if [[ $TRAVIS_PYTHON_VERSION == 2.* ]]; then fi pip install -q tifffile + + +if [[ $TRAVIS_PYTHON_VERSION == 2.7* ]]; then + MPL_QT_API=PyQt4 + MPL_DIR=$HOME/.matplotlib + export QT_API=pyqt + +else + MPL_QT_API=PySide + MPL_DIR=$HOME/.config/matplotlib + export QT_API=pyside +fi + +# Matplotlib settings - do not show figures during doc examples +mkdir -p $MPL_DIR +touch $MPL_DIR/matplotlibrc +echo 'backend : Template' > $MPL_DIR/matplotlibrc + + +echo -n "travis_fold:end:install.all\r" diff --git a/tools/travis_setup.sh b/tools/travis_install_min.sh similarity index 91% rename from tools/travis_setup.sh rename to tools/travis_install_min.sh index 7eee56cf..ed6e046b 100755 --- a/tools/travis_setup.sh +++ b/tools/travis_install_min.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -ex +echo -en "travis_fold:start:install.all\r" + # on Python 2.7, use the system versions of numpy, scipy, and matplotlib # and the minimum version of cython and networkx if [[ $TRAVIS_PYTHON_VERSION == 2.7* ]]; then @@ -29,3 +31,5 @@ pip install $WHEELHOUSE -r requirements.txt # clean up disk space sudo apt-get clean sudo rm -r /tmp/* + +echo -en "travis_fold:end:install.all\r" diff --git a/tools/travis_test.sh b/tools/travis_test_all.sh similarity index 56% rename from tools/travis_test.sh rename to tools/travis_test_all.sh index ea9f08c2..d4eef099 100755 --- a/tools/travis_test.sh +++ b/tools/travis_test_all.sh @@ -1,24 +1,8 @@ #!/usr/bin/env bash set -ex -if [[ $TRAVIS_PYTHON_VERSION == 2.7* ]]; then - MPL_QT_API=PyQt4 - MPL_DIR=$HOME/.matplotlib - export QT_API=pyqt +echo -en "travis_fold:start:doc.examples\r" -else - MPL_QT_API=PySide - MPL_DIR=$HOME/.config/matplotlib - export QT_API=pyside -fi - -# Matplotlib settings - do not show figures during doc examples -mkdir -p $MPL_DIR -touch $MPL_DIR/matplotlibrc -echo 'backend : Template' > $MPL_DIR/matplotlibrc - - -tools/header.py "Run doc examples" for f in doc/examples/*.py; do python "$f" if [ $? -ne 0 ]; then @@ -26,6 +10,7 @@ for f in doc/examples/*.py; do fi done + for f in doc/examples/applications/*.py; do python "$f" if [ $? -ne 0 ]; then @@ -33,15 +18,22 @@ for f in doc/examples/applications/*.py; do fi done +echo -en "travis_fold:end:doc.examples\r" + + +echo -en "travis_fold:start:test.all\r" + # Now configure Matplotlib to use Qt4 echo 'backend: Agg' > $MPL_DIR/matplotlibrc echo 'backend.qt4 : '$MPL_QT_API >> $MPL_DIR/matplotlibrc -tools/header.py "Run tests with all dependencies" +fold_start "Test_all_dependencies" # run tests again with optional dependencies to get more coverage if [[ $TRAVIS_PYTHON_VERSION == 3.3 ]]; then TEST_ARGS="$TEST_ARGS --with-cov --cover-package skimage" fi nosetests $TEST_ARGS +echo -en "travis_fold:end:test.all\r" + diff --git a/tools/travis_test_min.sh b/tools/travis_test_min.sh new file mode 100755 index 00000000..b249f5f5 --- /dev/null +++ b/tools/travis_test_min.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -ex + +echo -en "travis_fold:start:test.min\r" +nosetests $TEST_ARGS skimage +echo -en "travis_fold:end:test.min\r" + +echo -en "travis_fold:start:test.flake8\r" +flake8 --exit-zero --exclude=test_*,six.py skimage doc/examples viewer_examples +echo -en "travis_fold:end:test.flake8\r" From 0b4884a36179e9d29e2f29b6f53270d34bf4c82d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sat, 13 Dec 2014 20:20:24 -0600 Subject: [PATCH 2/9] Switch to a single script to get the folds to work Switch to a single script to get the folds to work Replace the old install_min with this before_install script Reinstate optional installs and add headers move the display setup to the travis_script Use force remove on temp directory Always retry on pip Fix the travis_retry pip alias Source the before_install script and move the alias Create bash functions for folding and export DISPLAY Export fold functions Make repip a function --- .travis.yml | 15 +-- ...nstall_min.sh => travis_before_install.sh} | 28 ++++- tools/travis_install_all.sh | 72 ----------- tools/travis_script.sh | 118 ++++++++++++++++++ tools/travis_test_all.sh | 39 ------ tools/travis_test_min.sh | 10 -- 6 files changed, 142 insertions(+), 140 deletions(-) rename tools/{travis_install_min.sh => travis_before_install.sh} (65%) delete mode 100755 tools/travis_install_all.sh create mode 100755 tools/travis_script.sh delete mode 100755 tools/travis_test_all.sh delete mode 100755 tools/travis_test_min.sh diff --git a/.travis.yml b/.travis.yml index 956f592b..a46b7171 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,18 +23,10 @@ python: - 3.4 before_install: - - export DISPLAY=:99.0 - - sh -e /etc/init.d/xvfb start - sudo apt-get update - - export PYTHONWARNINGS="all" - - export WHEELHOUSE="--no-index --find-links=http://travis-wheels.scikit-image.org/" - - export TEST_ARGS="--exe --ignore-files=^_test -v --with-doctest --ignore-files=^setup.py$" + - source tools/travis_before_install.sh - - travis_retry tools/travis_install_min.sh - - - source ~/venv/bin/activate - which python; python --version - - python check_bento_build.py - tools/build_versions.py @@ -42,10 +34,7 @@ install: - python setup.py build_ext --inplace - python setup.py install -script: - - travis_retry tools/travis_test_min.sh - - travis_retry tools/travis_install_all.sh - - travis_retry tools/travis_test_all.sh +script: tools/travis_script.sh after_success: - coveralls diff --git a/tools/travis_install_min.sh b/tools/travis_before_install.sh similarity index 65% rename from tools/travis_install_min.sh rename to tools/travis_before_install.sh index ed6e046b..fc6df38e 100755 --- a/tools/travis_install_min.sh +++ b/tools/travis_before_install.sh @@ -1,7 +1,12 @@ #!/usr/bin/env bash -set -ex -echo -en "travis_fold:start:install.all\r" +export WHEELHOUSE="--no-index --find-links=http://travis-wheels.scikit-image.org/" + + +repip () { + travis_retry pip $@ +} + # on Python 2.7, use the system versions of numpy, scipy, and matplotlib # and the minimum version of cython and networkx @@ -15,7 +20,7 @@ else fi source ~/venv/bin/activate -pip install wheel flake8 coveralls nose +repip install wheel flake8 coveralls nose # install system tk for matplotlib sudo apt-get install python-tk @@ -26,10 +31,21 @@ if [[ $TRAVIS_PYTHON_VERSION == 2.7* ]]; then sed -i 's/matplotlib>=*.*.*/matplotlib==1.3.1/g' requirements.txt fi -pip install $WHEELHOUSE -r requirements.txt +repip install $WHEELHOUSE -r requirements.txt # clean up disk space sudo apt-get clean -sudo rm -r /tmp/* +sudo rm -rf /tmp/* -echo -en "travis_fold:end:install.all\r" + +fold_start () { + echo -en "travis_fold:start:$1\r" +} + +fold_end () { + echo -en "travis_fold:end:$1\r" +} + +export -f fold_start +export -f fold_end +export -f repip diff --git a/tools/travis_install_all.sh b/tools/travis_install_all.sh deleted file mode 100755 index 1ef82e7a..00000000 --- a/tools/travis_install_all.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -set -ex - -echo -en "travis_fold:start:install.all\r" - -tools/header.py "Install optional dependencies" - -# Install Qt and then update the Matplotlib settings -if [[ $TRAVIS_PYTHON_VERSION == 2.7* ]]; then - sudo apt-get install -q python-qt4 - - # http://stackoverflow.com/a/9716100 - LIBS=( PyQt4 sip.so ) - - VAR=( $(which -a python$TRAVIS_PYTHON_VERSION) ) - - GET_PYTHON_LIB_CMD="from distutils.sysconfig import get_python_lib; print (get_python_lib())" - LIB_VIRTUALENV_PATH=$(python -c "$GET_PYTHON_LIB_CMD") - LIB_SYSTEM_PATH=$(${VAR[-1]} -c "$GET_PYTHON_LIB_CMD") - - for LIB in ${LIBS[@]} - do - sudo ln -sf $LIB_SYSTEM_PATH/$LIB $LIB_VIRTUALENV_PATH/$LIB - done - -else - sudo apt-get install -q libqt4-dev - pip install -q PySide $WHEELHOUSE - python ~/venv/bin/pyside_postinstall.py -install -fi - -# imread does NOT support py3.2 -if [[ $TRAVIS_PYTHON_VERSION != 3.2 ]]; then - sudo apt-get install -q libtiff4-dev libwebp-dev libpng12-dev xcftools - pip install -q imread -fi - -# Install SimpleITK from wheelhouse if available (not 3.2 or 3.4) -if [[ $TRAVIS_PYTHON_VERSION =~ 3\.[24] ]]; then - echo "SimpleITK unavailable on $TRAVIS_PYTHON_VERSION" -else - pip install -q SimpleITK $WHEELHOUSE -fi - -sudo apt-get install -q libfreeimage3 -pip install -q astropy $WHEELHOUSE - -if [[ $TRAVIS_PYTHON_VERSION == 2.* ]]; then - pip install -q pyamg -fi - -pip install -q tifffile - - -if [[ $TRAVIS_PYTHON_VERSION == 2.7* ]]; then - MPL_QT_API=PyQt4 - MPL_DIR=$HOME/.matplotlib - export QT_API=pyqt - -else - MPL_QT_API=PySide - MPL_DIR=$HOME/.config/matplotlib - export QT_API=pyside -fi - -# Matplotlib settings - do not show figures during doc examples -mkdir -p $MPL_DIR -touch $MPL_DIR/matplotlibrc -echo 'backend : Template' > $MPL_DIR/matplotlibrc - - -echo -n "travis_fold:end:install.all\r" diff --git a/tools/travis_script.sh b/tools/travis_script.sh new file mode 100755 index 00000000..fea6bbbc --- /dev/null +++ b/tools/travis_script.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +set -ex + +fold_start "script.setup" +sh -e /etc/init.d/xvfb start +export DISPLAY=:99.0 +PYTHONWARNINGS="all" +TEST_ARGS="--exe --ignore-files=^_test -v --with-doctest --ignore-files=^setup.py$" +fold_end "script.setup" + + +fold_start "test.min" +tools/header.py "Test with min requirements" + +nosetests $TEST_ARGS skimage +fold_end "test.min" + + +fold_start "test.flake8" +tools/header.py "Flake8 test" + +flake8 --exit-zero --exclude=test_*,six.py skimage doc/examples viewer_examples +fold_end "test.flake8" + + + +fold_start "install.all" +tools/header.py "Install optional dependencies" + +# Install Qt and then update the Matplotlib settings +if [[ $TRAVIS_PYTHON_VERSION == 2.7* ]]; then + sudo apt-get install -q python-qt4 + + # http://stackoverflow.com/a/9716100 + LIBS=( PyQt4 sip.so ) + + VAR=( $(which -a python$TRAVIS_PYTHON_VERSION) ) + + GET_PYTHON_LIB_CMD="from distutils.sysconfig import get_python_lib; print (get_python_lib())" + LIB_VIRTUALENV_PATH=$(python -c "$GET_PYTHON_LIB_CMD") + LIB_SYSTEM_PATH=$(${VAR[-1]} -c "$GET_PYTHON_LIB_CMD") + + for LIB in ${LIBS[@]} + do + sudo ln -sf $LIB_SYSTEM_PATH/$LIB $LIB_VIRTUALENV_PATH/$LIB + done + +else + sudo apt-get install -q libqt4-dev + repip install -q PySide $WHEELHOUSE + python ~/venv/bin/pyside_postinstall.py -install +fi + +# imread does NOT support py3.2 +if [[ $TRAVIS_PYTHON_VERSION != 3.2 ]]; then + sudo apt-get install -q libtiff4-dev libwebp-dev libpng12-dev xcftools + repip install -q imread +fi + +# Install SimpleITK from wheelhouse if available (not 3.2 or 3.4) +if [[ $TRAVIS_PYTHON_VERSION =~ 3\.[24] ]]; then + echo "SimpleITK unavailable on $TRAVIS_PYTHON_VERSION" +else + repip install -q SimpleITK $WHEELHOUSE +fi + +sudo apt-get install -q libfreeimage3 +repip install -q astropy $WHEELHOUSE + +if [[ $TRAVIS_PYTHON_VERSION == 2.* ]]; then + repip install -q pyamg +fi + +repip install -q tifffile + +fold_end "install.all" + + +fold_start "doc.examples" +tools/header.py "Run doc examples" + +for f in doc/examples/*.py; do + python "$f" + if [ $? -ne 0 ]; then + exit 1 + fi +done + +fold_end "doc.examples" + + +fold_start "doc.applications" +tools/header.py "Run doc applications" + +for f in doc/examples/applications/*.py; do + python "$f" + if [ $? -ne 0 ]; then + exit 1 + fi +done + +# Now configure Matplotlib to use Qt4 +echo 'backend: Agg' > $MPL_DIR/matplotlibrc +echo 'backend.qt4 : '$MPL_QT_API >> $MPL_DIR/matplotlibrc + +fold_end "doc.applications" + + +fold_start "test.all" +tools/header.py "Test with optional dependencies" + +# run tests again with optional dependencies to get more coverage +if [[ $TRAVIS_PYTHON_VERSION == 3.3 ]]; then + TEST_ARGS="$TEST_ARGS --with-cov --cover-package skimage" +fi +nosetests $TEST_ARGS + +fold_end "test.all" diff --git a/tools/travis_test_all.sh b/tools/travis_test_all.sh deleted file mode 100755 index d4eef099..00000000 --- a/tools/travis_test_all.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -set -ex - -echo -en "travis_fold:start:doc.examples\r" - -for f in doc/examples/*.py; do - python "$f" - if [ $? -ne 0 ]; then - exit 1 - fi -done - - -for f in doc/examples/applications/*.py; do - python "$f" - if [ $? -ne 0 ]; then - exit 1 - fi -done - -echo -en "travis_fold:end:doc.examples\r" - - -echo -en "travis_fold:start:test.all\r" - -# Now configure Matplotlib to use Qt4 -echo 'backend: Agg' > $MPL_DIR/matplotlibrc -echo 'backend.qt4 : '$MPL_QT_API >> $MPL_DIR/matplotlibrc - - -fold_start "Test_all_dependencies" -# run tests again with optional dependencies to get more coverage -if [[ $TRAVIS_PYTHON_VERSION == 3.3 ]]; then - TEST_ARGS="$TEST_ARGS --with-cov --cover-package skimage" -fi -nosetests $TEST_ARGS - -echo -en "travis_fold:end:test.all\r" - diff --git a/tools/travis_test_min.sh b/tools/travis_test_min.sh deleted file mode 100755 index b249f5f5..00000000 --- a/tools/travis_test_min.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash -set -ex - -echo -en "travis_fold:start:test.min\r" -nosetests $TEST_ARGS skimage -echo -en "travis_fold:end:test.min\r" - -echo -en "travis_fold:start:test.flake8\r" -flake8 --exit-zero --exclude=test_*,six.py skimage doc/examples viewer_examples -echo -en "travis_fold:end:test.flake8\r" From 91332bf3485be6a00f61c37f7de551d2d551e8a1 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 14 Dec 2014 06:20:23 -0600 Subject: [PATCH 3/9] Use our own retry command since travis_retry is not exported Use our own retry command since travis_retry is not exported Reinstate the MPL settings Fix the order of the MPL settings Fix the MPL versions Streamline the section headers Adjust min MPL requirement for the Ubuntu version Force pip to not install matplotlib and scipy --- requirements.txt | 2 +- tools/travis_before_install.sh | 37 ++++++++++++++------ tools/travis_script.sh | 62 ++++++++++++++++++---------------- 3 files changed, 61 insertions(+), 40 deletions(-) diff --git a/requirements.txt b/requirements.txt index 189b627b..a879d652 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ cython>=0.19.2 -matplotlib>=1.1.0 +matplotlib>=1.0 numpy>=1.6.1 scipy>=0.9 six>=1.3 diff --git a/tools/travis_before_install.sh b/tools/travis_before_install.sh index fc6df38e..03828fd2 100755 --- a/tools/travis_before_install.sh +++ b/tools/travis_before_install.sh @@ -1,10 +1,24 @@ #!/usr/bin/env bash +set -ex export WHEELHOUSE="--no-index --find-links=http://travis-wheels.scikit-image.org/" -repip () { - travis_retry pip $@ +retry () { + # https://gist.github.com/fungusakafungus/1026804 + local retry_max=3 + local count=$retry_max + while [ $count -gt 0 ]; do + "$@" && break + count=$(($count - 1)) + sleep 1 + done + + [ $count -eq 0 ] && { + echo "Retry failed [$retry_max]: $@" >&2 + return 1 + } + return 0 } @@ -15,37 +29,40 @@ if [[ $TRAVIS_PYTHON_VERSION == 2.7* ]]; then sudo apt-get install python-scipy python-matplotlib sed -i 's/cython>=/cython==/g' requirements.txt sed -i 's/networkx>=/networkx==/g' requirements.txt + sed -i '/matplotlib/d' requirements.txt + sed -i '/scipy/d' requirements.txt else virtualenv -p python --system-site-packages ~/venv fi source ~/venv/bin/activate -repip install wheel flake8 coveralls nose +retry pip install wheel flake8 coveralls nose # install system tk for matplotlib sudo apt-get install python-tk # on Python 3.2, use matplotlib 1.3.1 -if [[ $TRAVIS_PYTHON_VERSION == 2.7* ]]; then +if [[ $TRAVIS_PYTHON_VERSION == 3.2 ]]; then sed -i 's/matplotlib>=*.*.*/matplotlib==1.3.1/g' requirements.txt fi -repip install $WHEELHOUSE -r requirements.txt +retry pip install $WHEELHOUSE -r requirements.txt # clean up disk space sudo apt-get clean sudo rm -rf /tmp/* -fold_start () { +section () { echo -en "travis_fold:start:$1\r" + tools/header.py $1 } -fold_end () { +section_end () { echo -en "travis_fold:end:$1\r" } -export -f fold_start -export -f fold_end -export -f repip +export -f section +export -f section_end +export -f retry diff --git a/tools/travis_script.sh b/tools/travis_script.sh index fea6bbbc..ffd21f26 100755 --- a/tools/travis_script.sh +++ b/tools/travis_script.sh @@ -1,31 +1,25 @@ #!/usr/bin/env bash set -ex -fold_start "script.setup" +section "Script Setup" sh -e /etc/init.d/xvfb start export DISPLAY=:99.0 PYTHONWARNINGS="all" TEST_ARGS="--exe --ignore-files=^_test -v --with-doctest --ignore-files=^setup.py$" -fold_end "script.setup" +section_end "Script Setup" -fold_start "test.min" -tools/header.py "Test with min requirements" - +section "Test with min requirements" nosetests $TEST_ARGS skimage -fold_end "test.min" +section_end "Test with min requirements" -fold_start "test.flake8" -tools/header.py "Flake8 test" - +section "Flake8 test" flake8 --exit-zero --exclude=test_*,six.py skimage doc/examples viewer_examples -fold_end "test.flake8" +section_end "Flake8 test" - -fold_start "install.all" -tools/header.py "Install optional dependencies" +section "Install optional dependencies" # Install Qt and then update the Matplotlib settings if [[ $TRAVIS_PYTHON_VERSION == 2.7* ]]; then @@ -47,37 +41,42 @@ if [[ $TRAVIS_PYTHON_VERSION == 2.7* ]]; then else sudo apt-get install -q libqt4-dev - repip install -q PySide $WHEELHOUSE + retry pip install -q PySide $WHEELHOUSE python ~/venv/bin/pyside_postinstall.py -install fi # imread does NOT support py3.2 if [[ $TRAVIS_PYTHON_VERSION != 3.2 ]]; then sudo apt-get install -q libtiff4-dev libwebp-dev libpng12-dev xcftools - repip install -q imread + retry pip install -q imread fi # Install SimpleITK from wheelhouse if available (not 3.2 or 3.4) if [[ $TRAVIS_PYTHON_VERSION =~ 3\.[24] ]]; then echo "SimpleITK unavailable on $TRAVIS_PYTHON_VERSION" else - repip install -q SimpleITK $WHEELHOUSE + retry pip install -q SimpleITK $WHEELHOUSE fi sudo apt-get install -q libfreeimage3 -repip install -q astropy $WHEELHOUSE +retry pip install -q astropy $WHEELHOUSE if [[ $TRAVIS_PYTHON_VERSION == 2.* ]]; then - repip install -q pyamg + retry pip install -q pyamg fi -repip install -q tifffile +retry pip install -q tifffile -fold_end "install.all" +section_end "Install optional dependencies" -fold_start "doc.examples" -tools/header.py "Run doc examples" +section "Run doc examples" + +# Matplotlib settings - do not show figures during doc examples +MPL_DIR=$HOME/.config/matplotlib +mkdir -p $MPL_DIR +touch $MPL_DIR/matplotlibrc +echo 'backend : Template' > $MPL_DIR/matplotlibrc for f in doc/examples/*.py; do python "$f" @@ -86,11 +85,10 @@ for f in doc/examples/*.py; do fi done -fold_end "doc.examples" +section_end "Run doc examples" -fold_start "doc.applications" -tools/header.py "Run doc applications" +section "Run doc applications" for f in doc/examples/applications/*.py; do python "$f" @@ -100,14 +98,20 @@ for f in doc/examples/applications/*.py; do done # Now configure Matplotlib to use Qt4 +if [[ $TRAVIS_PYTHON_VERSION == 2.7* ]]; then + MPL_QT_API=PyQt4 + export QT_API=pyqt +else + MPL_QT_API=PySide + export QT_API=pyside +fi echo 'backend: Agg' > $MPL_DIR/matplotlibrc echo 'backend.qt4 : '$MPL_QT_API >> $MPL_DIR/matplotlibrc -fold_end "doc.applications" +section_end "Run doc applications" -fold_start "test.all" -tools/header.py "Test with optional dependencies" +section "Test with optional dependencies" # run tests again with optional dependencies to get more coverage if [[ $TRAVIS_PYTHON_VERSION == 3.3 ]]; then @@ -115,4 +119,4 @@ if [[ $TRAVIS_PYTHON_VERSION == 3.3 ]]; then fi nosetests $TEST_ARGS -fold_end "test.all" +section_end "Test with optional dependencies" From fd6f7fe9fd71188824159daa599caea1bfdac43b Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 14 Dec 2014 08:17:19 -0600 Subject: [PATCH 4/9] Use the system python for 2.7 --- tools/travis_before_install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/travis_before_install.sh b/tools/travis_before_install.sh index 03828fd2..2f71faaa 100755 --- a/tools/travis_before_install.sh +++ b/tools/travis_before_install.sh @@ -25,7 +25,7 @@ retry () { # on Python 2.7, use the system versions of numpy, scipy, and matplotlib # and the minimum version of cython and networkx if [[ $TRAVIS_PYTHON_VERSION == 2.7* ]]; then - virtualenv -p python --system-site-packages ~/venv + virtualenv --system-site-packages ~/venv sudo apt-get install python-scipy python-matplotlib sed -i 's/cython>=/cython==/g' requirements.txt sed -i 's/networkx>=/networkx==/g' requirements.txt From 68547f0a8a5f90d6ae5e4aeb05482983c9ff69b0 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 14 Dec 2014 08:20:01 -0600 Subject: [PATCH 5/9] Do not explicitly remove packages from requirements.txt --- requirements.txt | 2 +- tools/travis_before_install.sh | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index a879d652..189b627b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ cython>=0.19.2 -matplotlib>=1.0 +matplotlib>=1.1.0 numpy>=1.6.1 scipy>=0.9 six>=1.3 diff --git a/tools/travis_before_install.sh b/tools/travis_before_install.sh index 2f71faaa..8dad03f7 100755 --- a/tools/travis_before_install.sh +++ b/tools/travis_before_install.sh @@ -29,8 +29,6 @@ if [[ $TRAVIS_PYTHON_VERSION == 2.7* ]]; then sudo apt-get install python-scipy python-matplotlib sed -i 's/cython>=/cython==/g' requirements.txt sed -i 's/networkx>=/networkx==/g' requirements.txt - sed -i '/matplotlib/d' requirements.txt - sed -i '/scipy/d' requirements.txt else virtualenv -p python --system-site-packages ~/venv fi From 12248e18e7c52a278e8a9a821df97eab66636c6c Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 14 Dec 2014 08:20:43 -0600 Subject: [PATCH 6/9] Fix usage of set command Fix usage of set command Fix set command again Fix mpl directory for python 2.7 --- tools/travis_before_install.sh | 2 ++ tools/travis_script.sh | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/tools/travis_before_install.sh b/tools/travis_before_install.sh index 8dad03f7..d1a03ab8 100755 --- a/tools/travis_before_install.sh +++ b/tools/travis_before_install.sh @@ -64,3 +64,5 @@ section_end () { export -f section export -f section_end export -f retry + +set +ex diff --git a/tools/travis_script.sh b/tools/travis_script.sh index ffd21f26..6e40e70c 100755 --- a/tools/travis_script.sh +++ b/tools/travis_script.sh @@ -73,11 +73,17 @@ section_end "Install optional dependencies" section "Run doc examples" # Matplotlib settings - do not show figures during doc examples -MPL_DIR=$HOME/.config/matplotlib +if [[ $TRAVIS_PYTHON_VERSION == 2.7* ]]; then + MPL_DIR=$HOME/.matplotlib +else + MPL_DIR=$HOME/.config/matplotlib +fi + mkdir -p $MPL_DIR touch $MPL_DIR/matplotlibrc echo 'backend : Template' > $MPL_DIR/matplotlibrc + for f in doc/examples/*.py; do python "$f" if [ $? -ne 0 ]; then From 78974d93dc9e7dfb3ebf35dd77576e022debe855 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 14 Dec 2014 09:16:03 -0600 Subject: [PATCH 7/9] Fix section titles to avoid spaces --- tools/header.py | 1 + tools/travis_script.sh | 28 ++++++++++++++-------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/tools/header.py b/tools/header.py index d7a4c50b..01e2069f 100755 --- a/tools/header.py +++ b/tools/header.py @@ -9,6 +9,7 @@ print('*' * screen_width) if len(sys.argv) > 1: header = ' '.join(sys.argv[1:]) + header = header.replace('.', ' ') print('*', header.center(screen_width - 4), '*') print('*' * screen_width) diff --git a/tools/travis_script.sh b/tools/travis_script.sh index 6e40e70c..fad0d2a6 100755 --- a/tools/travis_script.sh +++ b/tools/travis_script.sh @@ -1,25 +1,25 @@ #!/usr/bin/env bash set -ex -section "Script Setup" +section "Script.Setup" sh -e /etc/init.d/xvfb start export DISPLAY=:99.0 PYTHONWARNINGS="all" TEST_ARGS="--exe --ignore-files=^_test -v --with-doctest --ignore-files=^setup.py$" -section_end "Script Setup" +section_end "Script.Setup" -section "Test with min requirements" +section "Test.with.min.requirements" nosetests $TEST_ARGS skimage -section_end "Test with min requirements" +section_end "Test.with.min.requirements" -section "Flake8 test" +section "Flake8.test" flake8 --exit-zero --exclude=test_*,six.py skimage doc/examples viewer_examples -section_end "Flake8 test" +section_end "Flake8.test" -section "Install optional dependencies" +section "Install.optional.dependencies" # Install Qt and then update the Matplotlib settings if [[ $TRAVIS_PYTHON_VERSION == 2.7* ]]; then @@ -67,10 +67,10 @@ fi retry pip install -q tifffile -section_end "Install optional dependencies" +section_end "Install.optional.dependencies" -section "Run doc examples" +section "Run.doc.examples" # Matplotlib settings - do not show figures during doc examples if [[ $TRAVIS_PYTHON_VERSION == 2.7* ]]; then @@ -91,10 +91,10 @@ for f in doc/examples/*.py; do fi done -section_end "Run doc examples" +section_end "Run.doc.examples" -section "Run doc applications" +section "Run.doc.applications" for f in doc/examples/applications/*.py; do python "$f" @@ -114,10 +114,10 @@ fi echo 'backend: Agg' > $MPL_DIR/matplotlibrc echo 'backend.qt4 : '$MPL_QT_API >> $MPL_DIR/matplotlibrc -section_end "Run doc applications" +section_end "Run.doc.applications" -section "Test with optional dependencies" +section "Test.with.optional.dependencies" # run tests again with optional dependencies to get more coverage if [[ $TRAVIS_PYTHON_VERSION == 3.3 ]]; then @@ -125,4 +125,4 @@ if [[ $TRAVIS_PYTHON_VERSION == 3.3 ]]; then fi nosetests $TEST_ARGS -section_end "Test with optional dependencies" +section_end "Test.with.optional.dependencies" From a8d480a1c82004f0c2eb315103e613cb3a5e118e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 14 Dec 2014 09:46:22 -0600 Subject: [PATCH 8/9] Add a coveralls repo token --- tools/travis_before_install.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/travis_before_install.sh b/tools/travis_before_install.sh index d1a03ab8..55046243 100755 --- a/tools/travis_before_install.sh +++ b/tools/travis_before_install.sh @@ -2,6 +2,7 @@ set -ex export WHEELHOUSE="--no-index --find-links=http://travis-wheels.scikit-image.org/" +export COVERALLS_REPO_TOKEN=7LdFN9232ZbSY3oaXHbQIzLazrSf6w2pQ retry () { From 6c8385f679ffb62b185f168139c297ea98e5c44e Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Sun, 14 Dec 2014 12:19:38 -0600 Subject: [PATCH 9/9] Update travis_notes and move to tools folder --- {doc => tools}/travis_notes.txt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) rename {doc => tools}/travis_notes.txt (56%) diff --git a/doc/travis_notes.txt b/tools/travis_notes.txt similarity index 56% rename from doc/travis_notes.txt rename to tools/travis_notes.txt index 37f5d4a1..84271821 100644 --- a/doc/travis_notes.txt +++ b/tools/travis_notes.txt @@ -1,5 +1,6 @@ -- Use http://yaml-online-parser.appspot.com/ to make sure it is valid yaml. +- Use http://yaml-online-parser.appspot.com/ to make sure +the yaml file is valid. http://lint.travis-ci.org/ is recommended elsewhere but does not give helpful error reports. - Make sure all of your "-" lines start on the same column @@ -13,9 +14,11 @@ set -ex ``` -- Use `travis_retry` before a command to have it try several times before -failing (useful for installing from third party sources). Note that this -command is not available within a bash script. +- Use the `retry` bash function from `before_install.sh` before a command to have it try 3 times before +failing (useful for installing from third party sources). +- Use the `section` function to start a folded section of the script. +Section names must have underscores or dots instead of spaces and must be +accompanied by a corresponding `section_end` call. - Feel free to cancel a build rather than waiting for it to go to completion if you have made a change to that branch. - A VM with 64bit Ubuntu 12.04 is a huge help for debugging.