From 78a0eb6b98897e7c56787e5cfffd2d6686d64db8 Mon Sep 17 00:00:00 2001 From: Jonathan Helmus Date: Tue, 8 Jul 2014 23:00:05 -0500 Subject: [PATCH] Implement appveyor builds Implement Appveyor builds initial test Updated appveyor.yml New nosetest call With conda update conda with pillow without pillow TST: Change case sensitivive JPG extension Revert "TST: Change case sensitivive JPG extension" This reverts commit 2deed7cc63736f7c6f0387bd37df4c6643c32847. Trying with Python 2.7 Trying with version env Trying again with variables Testing on all Python versions don't allow failures Allow failure Do not actually use tests Ignore failing tests Removed Python 2.6 test Testing only Python 2.6 Testing out more sklearn like AppVeyor CI Added install to pip Add artifacts Enabled all permutations Disable 2.6, add PIL Python 2.6, 2.7 only with PIL Testing with variable dependencies Allow failure with IF ELSE With Python 3.4 Scikit-learn like appveyor CI Fixing paths Undo path fix path fix single line path fix single line 2 Using Miniconda More path fixes New wheelhouse link Added pillow to requirements.txt Added networkx to requirements.txt Add testing of 32/64-bit Python 2.7 and 3.4 to matrix Debugging Cython compile Retry with all 4 builds Updated install.ps1 file Updated based on latest python-appveyor-demo Debugging pip install Specify numpy 1.8.1 until whl is uploaded to rackspace Use skimage-wide requirements.txt file Minor comment change to trigger build Install wheel and then install from WHEELHOUSE Install six from pip Install networkx from pip Install pyparsing from pip Install pytz from pip Try using just find-links Install the binary dependencies first, then the rest Add pillow to the install list Fix appveyor.yml syntax Fix requirements.txt syntax Fix requirements.txt syntax again Fix appveyor call to initial install Fix appveyor call to initial install again Fix appveyor call to initial install yet again Install wheel Install wheel first Install wheel and nose in the appveyor requirements.txt Fix Python3 version to match python ftp site Only use cleanup decorator if available Add debug info to multiimage test More debugging information Fix handling of path separators on Windows Add another warning guard Fix warning handling for non-windows Do not use TkAgg as it may be causing alloc error Clean up echo command Allow for unclosed file warning Fix spacing in echo command --- appveyor.yml | 68 +++++++ skimage/io/collection.py | 3 +- skimage/io/tests/test_multi_image.py | 4 +- skimage/io/tests/test_pil.py | 2 +- skimage/measure/tests/test_regionprops.py | 6 +- .../segmentation/tests/test_random_walker.py | 2 +- skimage/viewer/tests/test_tools.py | 6 +- tools/appveyor/install.ps1 | 180 ++++++++++++++++++ tools/appveyor/requirements.txt | 17 ++ tools/appveyor/run_with_env.cmd | 47 +++++ 10 files changed, 326 insertions(+), 9 deletions(-) create mode 100644 appveyor.yml create mode 100644 tools/appveyor/install.ps1 create mode 100644 tools/appveyor/requirements.txt create mode 100644 tools/appveyor/run_with_env.cmd diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 00000000..df1ffd3e --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,68 @@ +# AppVeyor.com is a Continuous Integration service to build and run tests under +# Windows + +environment: + global: + # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the + # /E:ON and /V:ON options are not enabled in the batch script intepreter + # See: http://stackoverflow.com/a/13751649/163740 + CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\tools\\appveyor\\run_with_env.cmd" + + matrix: + - PYTHON: "C:\\Python27_32" + PYTHON_VERSION: "2.7" + PYTHON_ARCH: "32" + + - PYTHON: "C:\\Python27_64" + PYTHON_VERSION: "2.7" + PYTHON_ARCH: "64" + + - PYTHON: "C:\\Python34_32" + PYTHON_VERSION: "3.4.2" + PYTHON_ARCH: "32" + + - PYTHON: "C:\\Python24_64" + PYTHON_VERSION: "3.4.2" + PYTHON_ARCH: "64" + +install: + # Install Python (from the official .msi of http://python.org) and pip when + # not already installed. + - "powershell ./tools/appveyor/install.ps1" + - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" + + # Check that we have the expected version and architecture for Python + - "python --version" + - "python -c \"import struct; print(struct.calcsize('P') * 8)\"" + + # Install the build and runtime dependencies of the project. + - "%CMD_IN_ENV% pip install -v %WHEELHOUSE% -r tools/appveyor/requirements.txt" + - "%CMD_IN_ENV% pip install -v -r requirements.txt" + - "%CMD_IN_ENV% python setup.py bdist_wheel bdist_wininst" + - ps: "ls dist" + + # Install the generated wheel package to test it + - "pip install --pre --no-index --find-links dist/ scikit-image" + + + +# Not a .NET project, we build scikit-image in the install step instead +build: false + +test_script: + # Change to a non-source folder to make sure we run the tests on the + # installed library. + - "cd C:\\" + + # Use the Agg backend in Matplotlib + - echo backend:Agg > matplotlibrc + + # Run unit tests with nose + - "python -c \"import nose; nose.main()\" -v -s skimage" + +artifacts: + # Archive the generated wheel package in the ci.appveyor.com build report. + - path: dist\* + +#on_success: +# - TODO: upload the content of dist/*.whl to a public wheelhouse diff --git a/skimage/io/collection.py b/skimage/io/collection.py index dccd88c8..49d6aa25 100644 --- a/skimage/io/collection.py +++ b/skimage/io/collection.py @@ -156,7 +156,8 @@ class ImageCollection(object): **load_func_kwargs): """Load and manage a collection of images.""" if isinstance(load_pattern, six.string_types): - load_pattern = load_pattern.split(os.pathsep) + load_pattern = load_pattern.replace(os.pathsep, ':') + load_pattern = load_pattern.split(':') self._files = [] for pattern in load_pattern: self._files.extend(glob(pattern)) diff --git a/skimage/io/tests/test_multi_image.py b/skimage/io/tests/test_multi_image.py index 76a71ec2..04a51ec6 100644 --- a/skimage/io/tests/test_multi_image.py +++ b/skimage/io/tests/test_multi_image.py @@ -29,12 +29,12 @@ class TestMultiImage(): assert len(self.imgs[2]) == len(self.imgs[3]) == 24 assert len(self.imgs[4]) == 2 assert len(self.imgs[5]) == 24 - assert len(self.imgs[6]) == 26 + assert len(self.imgs[6]) == 26, len(self.imgs[6]) def test_slicing(self): img = self.imgs[-1] assert type(img[:]) is ImageCollection - assert len(img[:]) == 26 + assert len(img[:]) == 26, len(img[:]) assert len(img[:1]) == 1 assert len(img[1:]) == 25 assert_allclose(img[0], img[:1][0]) diff --git a/skimage/io/tests/test_pil.py b/skimage/io/tests/test_pil.py index 780c66e0..fc28ce89 100644 --- a/skimage/io/tests/test_pil.py +++ b/skimage/io/tests/test_pil.py @@ -147,7 +147,7 @@ def test_imsave_filelike(): s = BytesIO() # save to file-like object - with expected_warnings(['precision loss']): + with expected_warnings(['precision loss|unclosed file']): imsave(s, image) # read from file-like object diff --git a/skimage/measure/tests/test_regionprops.py b/skimage/measure/tests/test_regionprops.py index 5a14aba5..3ac735c7 100644 --- a/skimage/measure/tests/test_regionprops.py +++ b/skimage/measure/tests/test_regionprops.py @@ -126,13 +126,13 @@ def test_equiv_diameter(): def test_euler_number(): - with expected_warnings(['`background`']): + with expected_warnings(['`background`|CObject type']): en = regionprops(SAMPLE)[0].euler_number assert en == 0 SAMPLE_mod = SAMPLE.copy() SAMPLE_mod[7, -3] = 0 - with expected_warnings(['`background`']): + with expected_warnings(['`background`|CObject type']): en = regionprops(SAMPLE_mod)[0].euler_number assert en == -1 @@ -372,7 +372,7 @@ def test_equals(): r2 = regions[0] r3 = regions[1] - with expected_warnings(['`background`']): + with expected_warnings(['`background`|CObject type']): assert_equal(r1 == r2, True, "Same regionprops are not equal") assert_equal(r1 != r3, True, "Different regionprops are equal") diff --git a/skimage/segmentation/tests/test_random_walker.py b/skimage/segmentation/tests/test_random_walker.py index f2207fde..a3a45c8d 100644 --- a/skimage/segmentation/tests/test_random_walker.py +++ b/skimage/segmentation/tests/test_random_walker.py @@ -162,7 +162,7 @@ def test_3d_inactive(): old_labels = np.copy(labels) labels[5:25, 26:29, 26:29] = -1 after_labels = np.copy(labels) - with expected_warnings(['"cg" mode']): + with expected_warnings(['"cg" mode|CObject type']): labels = random_walker(data, labels, mode='cg') assert (labels.reshape(data.shape)[13:17, 13:17, 13:17] == 2).all() assert data.shape == labels.shape diff --git a/skimage/viewer/tests/test_tools.py b/skimage/viewer/tests/test_tools.py index aae44783..b79a40e2 100644 --- a/skimage/viewer/tests/test_tools.py +++ b/skimage/viewer/tests/test_tools.py @@ -8,7 +8,11 @@ from skimage.viewer import ImageViewer, has_qt from skimage.viewer.canvastools import ( LineTool, ThickLineTool, RectangleTool, PaintTool) from skimage.viewer.canvastools.base import CanvasToolBase -from matplotlib.testing.decorators import cleanup +try: + from matplotlib.testing.decorators import cleanup +except ImportError: + def cleanup(func): + return func def get_end_points(image): diff --git a/tools/appveyor/install.ps1 b/tools/appveyor/install.ps1 new file mode 100644 index 00000000..0f165d8b --- /dev/null +++ b/tools/appveyor/install.ps1 @@ -0,0 +1,180 @@ +# Sample script to install Python and pip under Windows +# Authors: Olivier Grisel, Jonathan Helmus and Kyle Kastner +# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ + +$MINICONDA_URL = "http://repo.continuum.io/miniconda/" +$BASE_URL = "https://www.python.org/ftp/python/" +$GET_PIP_URL = "https://bootstrap.pypa.io/get-pip.py" +$GET_PIP_PATH = "C:\get-pip.py" + + +function DownloadPython ($python_version, $platform_suffix) { + $webclient = New-Object System.Net.WebClient + $filename = "python-" + $python_version + $platform_suffix + ".msi" + $url = $BASE_URL + $python_version + "/" + $filename + + $basedir = $pwd.Path + "\" + $filepath = $basedir + $filename + if (Test-Path $filename) { + Write-Host "Reusing" $filepath + return $filepath + } + + # Download and retry up to 3 times in case of network transient errors. + Write-Host "Downloading" $filename "from" $url + $retry_attempts = 2 + for($i=0; $i -lt $retry_attempts; $i++){ + try { + $webclient.DownloadFile($url, $filepath) + break + } + Catch [Exception]{ + Start-Sleep 1 + } + } + if (Test-Path $filepath) { + Write-Host "File saved at" $filepath + } else { + # Retry once to get the error message if any at the last try + $webclient.DownloadFile($url, $filepath) + } + return $filepath +} + + +function InstallPython ($python_version, $architecture, $python_home) { + Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home + if (Test-Path $python_home) { + Write-Host $python_home "already exists, skipping." + return $false + } + if ($architecture -eq "32") { + $platform_suffix = "" + } else { + $platform_suffix = ".amd64" + } + $msipath = DownloadPython $python_version $platform_suffix + Write-Host "Installing" $msipath "to" $python_home + $install_log = $python_home + ".log" + $install_args = "/qn /log $install_log /i $msipath TARGETDIR=$python_home" + $uninstall_args = "/qn /x $msipath" + RunCommand "msiexec.exe" $install_args + if (-not(Test-Path $python_home)) { + Write-Host "Python seems to be installed else-where, reinstalling." + RunCommand "msiexec.exe" $uninstall_args + RunCommand "msiexec.exe" $install_args + } + if (Test-Path $python_home) { + Write-Host "Python $python_version ($architecture) installation complete" + } else { + Write-Host "Failed to install Python in $python_home" + Get-Content -Path $install_log + Exit 1 + } +} + +function RunCommand ($command, $command_args) { + Write-Host $command $command_args + Start-Process -FilePath $command -ArgumentList $command_args -Wait -Passthru +} + + +function InstallPip ($python_home) { + $pip_path = $python_home + "\Scripts\pip.exe" + $python_path = $python_home + "\python.exe" + if (-not(Test-Path $pip_path)) { + Write-Host "Installing pip..." + $webclient = New-Object System.Net.WebClient + $webclient.DownloadFile($GET_PIP_URL, $GET_PIP_PATH) + Write-Host "Executing:" $python_path $GET_PIP_PATH + Start-Process -FilePath "$python_path" -ArgumentList "$GET_PIP_PATH" -Wait -Passthru + } else { + Write-Host "pip already installed." + } +} + + +function DownloadMiniconda ($python_version, $platform_suffix) { + $webclient = New-Object System.Net.WebClient + if ($python_version -eq "3.4") { + $filename = "Miniconda3-3.5.5-Windows-" + $platform_suffix + ".exe" + } else { + $filename = "Miniconda-3.5.5-Windows-" + $platform_suffix + ".exe" + } + $url = $MINICONDA_URL + $filename + + $basedir = $pwd.Path + "\" + $filepath = $basedir + $filename + if (Test-Path $filename) { + Write-Host "Reusing" $filepath + return $filepath + } + + # Download and retry up to 3 times in case of network transient errors. + Write-Host "Downloading" $filename "from" $url + $retry_attempts = 2 + for($i=0; $i -lt $retry_attempts; $i++){ + try { + $webclient.DownloadFile($url, $filepath) + break + } + Catch [Exception]{ + Start-Sleep 1 + } + } + if (Test-Path $filepath) { + Write-Host "File saved at" $filepath + } else { + # Retry once to get the error message if any at the last try + $webclient.DownloadFile($url, $filepath) + } + return $filepath +} + + +function InstallMiniconda ($python_version, $architecture, $python_home) { + Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home + if (Test-Path $python_home) { + Write-Host $python_home "already exists, skipping." + return $false + } + if ($architecture -eq "32") { + $platform_suffix = "x86" + } else { + $platform_suffix = "x86_64" + } + $filepath = DownloadMiniconda $python_version $platform_suffix + Write-Host "Installing" $filepath "to" $python_home + $install_log = $python_home + ".log" + $args = "/S /D=$python_home" + Write-Host $filepath $args + Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru + if (Test-Path $python_home) { + Write-Host "Python $python_version ($architecture) installation complete" + } else { + Write-Host "Failed to install Python in $python_home" + Get-Content -Path $install_log + Exit 1 + } +} + + +function InstallMinicondaPip ($python_home) { + $pip_path = $python_home + "\Scripts\pip.exe" + $conda_path = $python_home + "\Scripts\conda.exe" + if (-not(Test-Path $pip_path)) { + Write-Host "Installing pip..." + $args = "install --yes pip" + Write-Host $conda_path $args + Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru + } else { + Write-Host "pip already installed." + } +} + +function main () { + InstallPython $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON + InstallPip $env:PYTHON +} + +main diff --git a/tools/appveyor/requirements.txt b/tools/appveyor/requirements.txt new file mode 100644 index 00000000..9cc4022f --- /dev/null +++ b/tools/appveyor/requirements.txt @@ -0,0 +1,17 @@ +# Fetch numpy and scipy wheels from the sklearn rackspace wheelhouse. +# Those wheels were generated by @ogrisel by calling `wheel convert` on +# the binaries from http://www.lfd.uci.edu/~gohlke/pythonlibs/ +# This is a temporary solution. As soon as numpy, scipy and cython provide +# official wheels for windows we can delete this --find-links line. +--find-links http://28daf2247a33ed269873-7b1aad3fab3cc330e1fd9d109892382a.r6.cf2.rackcdn.com/appveyor_pip_index.html + +# fix the versions of binary packages to force the use of the whl +# of the rackspace folder instead of trying to install from PyPI +# wheels are preferred for a given version +numpy==1.8.1 +scipy==0.14.0 +cython==0.20.2 +matplotlib==1.4.2 +pillow==2.6.1 +wheel +nose diff --git a/tools/appveyor/run_with_env.cmd b/tools/appveyor/run_with_env.cmd new file mode 100644 index 00000000..0c70d633 --- /dev/null +++ b/tools/appveyor/run_with_env.cmd @@ -0,0 +1,47 @@ +:: To build extensions for 64 bit Python 3, we need to configure environment +:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) +:: +:: To build extensions for 64 bit Python 2, we need to configure environment +:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) +:: +:: 32 bit builds do not require specific environment configurations. +:: +:: Note: this script needs to be run with the /E:ON and /V:ON flags for the +:: cmd interpreter, at least for (SDK v7.0) +:: +:: More details at: +:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows +:: http://stackoverflow.com/a/13751649/163740 +:: +:: Author: Olivier Grisel +:: License: BSD 3 clause +@ECHO OFF + +SET COMMAND_TO_RUN=%* +SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows + +SET MAJOR_PYTHON_VERSION="%PYTHON_VERSION:~0,1%" +IF %MAJOR_PYTHON_VERSION% == "2" ( + SET WINDOWS_SDK_VERSION="v7.0" +) ELSE IF %MAJOR_PYTHON_VERSION% == "3" ( + SET WINDOWS_SDK_VERSION="v7.1" +) ELSE ( + ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" + EXIT 1 +) + +IF "%PYTHON_ARCH%"=="64" ( + ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture + SET DISTUTILS_USE_SDK=1 + SET MSSdk=1 + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 +) ELSE ( + ECHO Using default MSVC build environment for 32 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 +)