From 96046fc12d3ba6a6fa9cb1c302222da3612d15fc Mon Sep 17 00:00:00 2001 From: Joe Jevnik Date: Wed, 3 Feb 2016 16:52:23 -0500 Subject: [PATCH 01/51] BLD: appveyor.yml --- appveyor.yml | 59 +++++++++++++++++++++++++++ etc/install.ps1 | 96 ++++++++++++++++++++++++++++++++++++++++++++ etc/run_with_env.cmd | 95 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 250 insertions(+) create mode 100644 appveyor.yml create mode 100644 etc/install.ps1 create mode 100644 etc/run_with_env.cmd diff --git a/appveyor.yml b/appveyor.yml new file mode 100644 index 00000000..49ebf38d --- /dev/null +++ b/appveyor.yml @@ -0,0 +1,59 @@ +matrix: + fast_finish: true + +environment: + global: + # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the + # /E:ON and /V:ON options are not enabled in the batch script intepreter + # See: http://stackoverflow.com/a/13751649/163740 + CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\etc\\run_with_env.cmd" + + matrix: + - PYTHON: "C:\\Python34_64" + PYTHON_VERSION: "3.4" + PYTHON_ARCH: "64" + PANDAS_VERSION: "0.16.1" + NUMPY_VERSION: "1.9.2" + SCIPY_VERSION: "0.15.1" + + - PYTHON: "C:\\Python27_64" + PYTHON_VERSION: "2.7" + PYTHON_ARCH: "64" + PANDAS_VERSION: "0.16.1" + NUMPY_VERSION: "1.9.2" + SCIPY_VERSION: "0.15.1" + + +# We always use a 64-bit machine, but can build x86 distributions +# with the PYTHON_ARCH variable (which is used by CMD_IN_ENV). +platform: + - x64 + +# all our python builds have to happen in tests_script... +build: false + +init: + - "ECHO %PYTHON_VERSION% %PYTHON%" + +install: + # this installs the appropriate Miniconda (Py2/Py3, 32/64 bit), + - powershell .\etc\install.ps1 + - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% + + - cmd: conda create -n testenv --yes pip python=%PYTHON_VERSION% + - cmd: activate testenv + - cmd: conda install --yes -c quantopian numpy=%NUMPY_VERSION% pandas=%PANDAS_VERSION% scipy=%SCIPY_VERSION% Cython=0.22.1 ta-lib=0.4.8 + # remove these packages so our requirements can downgrade them + - cmd: conda remove --yes six pytz + - cmd: pip install --upgrade pip coverage coveralls + - cmd: pip install -r etc/requirements.txt + - cmd: pip install -r etc/requirements_dev.txt + # this uses git requirements right now + - cmd: pip install -r etc/requirements_blaze.txt + - cmd: pip install -e . + +test_script: + - cd \ + - pip freeze | sort + - flake8 zipline tests + - nosetests --with-timer --exclude=^test_examples --with-coverage --cover-package=zipline --timer-top-n=15 diff --git a/etc/install.ps1 b/etc/install.ps1 new file mode 100644 index 00000000..c964973c --- /dev/null +++ b/etc/install.ps1 @@ -0,0 +1,96 @@ +# Sample script to install Miniconda under Windows +# Authors: Olivier Grisel, Jonathan Helmus and Kyle Kastner, Robert McGibbon +# License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ + +$MINICONDA_URL = "http://repo.continuum.io/miniconda/" + + +function DownloadMiniconda ($python_version, $platform_suffix) { + $webclient = New-Object System.Net.WebClient + if ($python_version -match "3.4") { + $filename = "Miniconda3-latest-Windows-" + $platform_suffix + ".exe" + } else { + $filename = "Miniconda-latest-Windows-" + $platform_suffix + ".exe" + } + $url = $MINICONDA_URL + $filename + + $basedir = $pwd.Path + "\" + $filepath = $basedir + $filename + if (Test-Path $filename) { + Write-Host "Reusing" $filepath + return $filepath + } + + # Download and retry up to 3 times in case of network transient errors. + Write-Host "Downloading" $filename "from" $url + $retry_attempts = 2 + for($i=0; $i -lt $retry_attempts; $i++){ + try { + $webclient.DownloadFile($url, $filepath) + break + } + Catch [Exception]{ + Start-Sleep 1 + } + } + if (Test-Path $filepath) { + Write-Host "File saved at" $filepath + } else { + # Retry once to get the error message if any at the last try + $webclient.DownloadFile($url, $filepath) + } + return $filepath +} + + +function InstallMiniconda ($python_version, $architecture, $python_home) { + Write-Host "Installing Python" $python_version "for" $architecture "bit architecture to" $python_home + if (Test-Path $python_home) { + Write-Host $python_home "already exists, skipping." + return $false + } + if ($architecture -match "32") { + $platform_suffix = "x86" + } else { + $platform_suffix = "x86_64" + } + + $filepath = DownloadMiniconda $python_version $platform_suffix + Write-Host "Installing" $filepath "to" $python_home + $install_log = $python_home + ".log" + $args = "/S /D=$python_home" + Write-Host $filepath $args + Start-Process -FilePath $filepath -ArgumentList $args -Wait -Passthru + if (Test-Path $python_home) { + Write-Host "Python $python_version ($architecture) installation complete" + } else { + Write-Host "Failed to install Python in $python_home" + Get-Content -Path $install_log + Exit 1 + } +} + + +function InstallCondaPackages ($python_home, $spec) { + $conda_path = $python_home + "\Scripts\conda.exe" + $args = "install --yes " + $spec + Write-Host ("conda " + $args) + Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru +} + +function UpdateConda ($python_home) { + $conda_path = $python_home + "\Scripts\conda.exe" + Write-Host "Updating conda..." + $args = "update --yes conda" + Write-Host $conda_path $args + Start-Process -FilePath "$conda_path" -ArgumentList $args -Wait -Passthru +} + + +function main () { + InstallMiniconda $env:PYTHON_VERSION $env:PYTHON_ARCH $env:PYTHON + UpdateConda $env:PYTHON + InstallCondaPackages $env:PYTHON "conda-build jinja2 anaconda-client" +} + +main diff --git a/etc/run_with_env.cmd b/etc/run_with_env.cmd new file mode 100644 index 00000000..848f4608 --- /dev/null +++ b/etc/run_with_env.cmd @@ -0,0 +1,95 @@ +:: EXPECTED ENV VARS: PYTHON_ARCH (either x86 or x64) +:: CONDA_PY (either 27, 33, 35 etc. - only major version is extracted) +:: +:: +:: To build extensions for 64 bit Python 3, we need to configure environment +:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) +:: +:: To build extensions for 64 bit Python 2, we need to configure environment +:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) +:: +:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific +:: environment configurations. +:: +:: Note: this script needs to be run with the /E:ON and /V:ON flags for the +:: cmd interpreter, at least for (SDK v7.0) +:: +:: More details at: +:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows +:: http://stackoverflow.com/a/13751649/163740 +:: +:: Author: Phil Elson +:: Original Author: Olivier Grisel (https://github.com/ogrisel/python-appveyor-demo) +:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ +:: +:: Notes about batch files for Python people: +:: +:: Quotes in values are literally part of the values: +:: SET FOO="bar" +:: FOO is now five characters long: " b a r " +:: If you don't want quotes, don't include them on the right-hand side. +:: +:: The CALL lines at the end of this file look redundant, but if you move them +:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y +:: case, I don't know why. +:: originally from https://github.com/pelson/Obvious-CI/blob/master/scripts/obvci_appveyor_python_build_env.cmd +@ECHO OFF + +SET COMMAND_TO_RUN=%* +SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows + +:: Extract the major and minor versions, and allow for the minor version to be +:: more than 9. This requires the version number to have two dots in it. +SET MAJOR_PYTHON_VERSION=%CONDA_PY:~0,1% + +IF "%CONDA_PY:~2,1%" == "" ( + :: CONDA_PY style, such as 27, 34 etc. + SET MINOR_PYTHON_VERSION=%CONDA_PY:~1,1% +) ELSE ( + IF "%CONDA_PY:~3,1%" == "." ( + SET MINOR_PYTHON_VERSION=%CONDA_PY:~2,1% + ) ELSE ( + SET MINOR_PYTHON_VERSION=%CONDA_PY:~2,2% + ) +) + +:: Based on the Python version, determine what SDK version to use, and whether +:: to set the SDK for 64-bit. +IF %MAJOR_PYTHON_VERSION% == 2 ( + SET WINDOWS_SDK_VERSION="v7.0" + SET SET_SDK_64=Y +) ELSE ( + IF %MAJOR_PYTHON_VERSION% == 3 ( + SET WINDOWS_SDK_VERSION="v7.1" + IF %MINOR_PYTHON_VERSION% LEQ 4 ( + SET SET_SDK_64=Y + ) ELSE ( + SET SET_SDK_64=N + ) + ) ELSE ( + ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" + EXIT /B 1 + ) +) + +IF "%PYTHON_ARCH%"=="64" ( + IF %SET_SDK_64% == Y ( + ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture + SET DISTUTILS_USE_SDK=1 + SET MSSdk=1 + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT /B 1 + ) ELSE ( + ECHO Using default MSVC build environment for 64 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT /B 1 + ) +) ELSE ( + ECHO Using default MSVC build environment for 32 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT /B 1 +) From b7f4a2f589880e7dd241c3dc7097a811fee5e95c Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 29 Feb 2016 21:38:34 -0500 Subject: [PATCH 02/51] BLD: Provide env vars required by run_with_env.cmd --- appveyor.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 49ebf38d..9411674a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,6 +14,8 @@ environment: PYTHON_ARCH: "64" PANDAS_VERSION: "0.16.1" NUMPY_VERSION: "1.9.2" + CONDA_PY: "34" + CONDA_NPY: "19" SCIPY_VERSION: "0.15.1" - PYTHON: "C:\\Python27_64" @@ -21,6 +23,8 @@ environment: PYTHON_ARCH: "64" PANDAS_VERSION: "0.16.1" NUMPY_VERSION: "1.9.2" + CONDA_PY: "27" + CONDA_NPY: "19" SCIPY_VERSION: "0.15.1" From 15f9502a1080cfd4cba68e515decaff13a821d7c Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 29 Feb 2016 21:39:31 -0500 Subject: [PATCH 03/51] BLD: Updates to match travis BLD: Combine create and install --- appveyor.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 9411674a..7eb27155 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -44,11 +44,8 @@ install: - powershell .\etc\install.ps1 - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% - - cmd: conda create -n testenv --yes pip python=%PYTHON_VERSION% + - cmd: conda create -n testenv --yes -c quantopian pip numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 - cmd: activate testenv - - cmd: conda install --yes -c quantopian numpy=%NUMPY_VERSION% pandas=%PANDAS_VERSION% scipy=%SCIPY_VERSION% Cython=0.22.1 ta-lib=0.4.8 - # remove these packages so our requirements can downgrade them - - cmd: conda remove --yes six pytz - cmd: pip install --upgrade pip coverage coveralls - cmd: pip install -r etc/requirements.txt - cmd: pip install -r etc/requirements_dev.txt @@ -59,5 +56,5 @@ install: test_script: - cd \ - pip freeze | sort + - nosetests tests/ - flake8 zipline tests - - nosetests --with-timer --exclude=^test_examples --with-coverage --cover-package=zipline --timer-top-n=15 From de65650fd2bf31edc104e8785c3d301efc903a41 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 29 Feb 2016 21:39:42 -0500 Subject: [PATCH 04/51] BLD: Build ta-lib first --- appveyor.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 7eb27155..75c5fd29 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -44,6 +44,10 @@ install: - powershell .\etc\install.ps1 - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% + - conda info -a + - conda install conda-build + - cmd: '%CMD_IN_ENV% conda build conda\ta-lib -q' + - cmd: conda create -n testenv --yes -c quantopian pip numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 - cmd: activate testenv - cmd: pip install --upgrade pip coverage coveralls From 8563e5fabbe707a5a4e3ef93ddc61470b03297eb Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 1 Mar 2016 17:03:34 -0500 Subject: [PATCH 05/51] BLD: Windows ta-lib build --- conda/ta-lib/bld.bat | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/conda/ta-lib/bld.bat b/conda/ta-lib/bld.bat index 7c933e4f..6ef9c91a 100644 --- a/conda/ta-lib/bld.bat +++ b/conda/ta-lib/bld.bat @@ -1,2 +1,35 @@ -%SYS_PYTHON% setup.py build --compiler msvc -%SYS_PYTHON% setup.py install --prefix=%PREFIX% +rmdir /s /q "C:\ta-lib\" +powershell -Command "(New-Object Net.WebClient).DownloadFile('http://prdownloads.sourceforge.net/ta-lib/ta-lib-0.4.0-msvc.zip', 'ta-lib-0.4.0-msvc.zip')" +IF %ERRORLEVEL% == 1; exit 1 +powershell -Command "Add-Type -AssemblyName System.IO.Compression.FileSystem;[System.IO.Compression.ZipFile]::ExtractToDirectory('ta-lib-0.4.0-msvc.zip', 'C:\')" +IF %ERRORLEVEL% == 1; exit 1 +pushd C:\ta-lib\c\ +pushd make\cdd\win32\msvc +nmake +IF %ERRORLEVEL% == 1; exit 1 +popd +pushd make\cdr\win32\msvc +nmake +IF %ERRORLEVEL% == 1; exit 1 +popd +pushd make\cmd\win32\msvc +nmake +IF %ERRORLEVEL% == 1; exit 1 +popd +pushd make\cmr\win32\msvc +nmake +IF %ERRORLEVEL% == 1; exit 1 +popd +pushd make\csd\win32\msvc +nmake +IF %ERRORLEVEL% == 1; exit 1 +popd +pushd make\csr\win32\msvc +nmake +IF %ERRORLEVEL% == 1; exit 1 +popd +popd +del ta-lib-0.4.0-msvc.zip + +python setup.py build --compiler msvc +python setup.py install --prefix=%PREFIX% From fbb29ba91a8db18f8c0f6960023a6ee0339e8a02 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 3 Mar 2016 09:33:55 -0500 Subject: [PATCH 06/51] BLD: Have travis build zipline conda package Also include talib requirements when pip installing Format numpy version for conda build Include talib req in appveyor install too --- .travis.yml | 5 ++++- appveyor.yml | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4faf2244..111b1183 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,16 +24,19 @@ before_install: - sed -i "s/pandas==.*/pandas==$PANDAS_VERSION/" etc/requirements.txt - sed -i "s/scipy==.*/scipy==$SCIPY_VERSION/" etc/requirements.txt install: + - conda install conda-build --yes - conda create -n testenv --yes -c quantopian pip python=$TRAVIS_PYTHON_VERSION numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION libgfortran=1.0 ta-lib=0.4.9 - source activate testenv - pip install --upgrade pip coverage coveralls --cache-dir=$CACHE_DIR - pip install -r etc/requirements.txt --cache-dir=$CACHE_DIR - pip install -r etc/requirements_dev.txt --cache-dir=$CACHE_DIR - pip install -r etc/requirements_blaze.txt --cache-dir=$CACHE_DIR # this uses git requirements right now - - pip install -e . --cache-dir=$CACHE_DIR + - pip install -e .[talib] --cache-dir=$CACHE_DIR before_script: - pip freeze | sort script: + - IFS='.' read -r -a conda_npy_version <<< "$NUMPY_VERSION" + - conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy="${conda_npy_version[0]}.${conda_npy_version[1]}" # accepts only major.minor - nosetests tests/ - flake8 zipline tests after_success: diff --git a/appveyor.yml b/appveyor.yml index 75c5fd29..51bf1bee 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -48,14 +48,14 @@ install: - conda install conda-build - cmd: '%CMD_IN_ENV% conda build conda\ta-lib -q' - - cmd: conda create -n testenv --yes -c quantopian pip numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 + - cmd: conda create -n testenv --yes --use-local pip numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 - cmd: activate testenv - cmd: pip install --upgrade pip coverage coveralls - cmd: pip install -r etc/requirements.txt - cmd: pip install -r etc/requirements_dev.txt # this uses git requirements right now - cmd: pip install -r etc/requirements_blaze.txt - - cmd: pip install -e . + - cmd: pip install -e .[talib] test_script: - cd \ From c753adc3494fba78a11a49f97b7ea63676fe7cda Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 3 Mar 2016 09:33:42 -0500 Subject: [PATCH 07/51] BLD: Eventual matrix --- appveyor.yml | 72 +++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 63 insertions(+), 9 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 51bf1bee..a916e07d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,21 +11,75 @@ environment: matrix: - PYTHON: "C:\\Python34_64" PYTHON_VERSION: "3.4" - PYTHON_ARCH: "64" + PYTHON_ARCH: "32" PANDAS_VERSION: "0.16.1" NUMPY_VERSION: "1.9.2" CONDA_PY: "34" CONDA_NPY: "19" SCIPY_VERSION: "0.15.1" - - PYTHON: "C:\\Python27_64" - PYTHON_VERSION: "2.7" - PYTHON_ARCH: "64" - PANDAS_VERSION: "0.16.1" - NUMPY_VERSION: "1.9.2" - CONDA_PY: "27" - CONDA_NPY: "19" - SCIPY_VERSION: "0.15.1" +# - PYTHON: "C:\\Python27_64" +# PYTHON_VERSION: "2.7" +# PYTHON_ARCH: "32" +# PANDAS_VERSION: "0.16.1" +# NUMPY_VERSION: "1.9.2" +# CONDA_PY: "27" +# CONDA_NPY: "19" +# SCIPY_VERSION: "0.15.1" +# +# - PYTHON: "C:\\Python34_64" +# PYTHON_VERSION: "3.4" +# PYTHON_ARCH: "32" +# PANDAS_VERSION: "0.17.1" +# NUMPY_VERSION: "1.10.2" +# CONDA_PY: "34" +# CONDA_NPY: "19" +# SCIPY_VERSION: "0.16.1" +# +# - PYTHON: "C:\\Python27_64" +# PYTHON_VERSION: "2.7" +# PYTHON_ARCH: "32" +# PANDAS_VERSION: "0.17.1" +# NUMPY_VERSION: "1.10.2" +# CONDA_PY: "27" +# CONDA_NPY: "19" +# SCIPY_VERSION: "0.16.1" +# +# - PYTHON: "C:\\Python34_64" +# PYTHON_VERSION: "3.4" +# PYTHON_ARCH: "32" +# PANDAS_VERSION: "0.16.1" +# NUMPY_VERSION: "1.9.2" +# CONDA_PY: "64" +# CONDA_NPY: "19" +# SCIPY_VERSION: "0.15.1" +# +# - PYTHON: "C:\\Python27_64" +# PYTHON_VERSION: "2.7" +# PYTHON_ARCH: "32" +# PANDAS_VERSION: "0.16.1" +# NUMPY_VERSION: "1.9.2" +# CONDA_PY: "64" +# CONDA_NPY: "19" +# SCIPY_VERSION: "0.15.1" +# +# - PYTHON: "C:\\Python34_64" +# PYTHON_VERSION: "3.4" +# PYTHON_ARCH: "32" +# PANDAS_VERSION: "0.17.1" +# NUMPY_VERSION: "1.10.2" +# CONDA_PY: "64" +# CONDA_NPY: "19" +# SCIPY_VERSION: "0.16.1" +# +# - PYTHON: "C:\\Python27_64" +# PYTHON_VERSION: "2.7" +# PYTHON_ARCH: "64" +# PANDAS_VERSION: "0.17.1" +# NUMPY_VERSION: "1.10.2" +# CONDA_PY: "27" +# CONDA_NPY: "19" +# SCIPY_VERSION: "0.16.1" # We always use a 64-bit machine, but can build x86 distributions From 8a5c8f5519e2ccf206541d138c7cb3cab858651d Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 3 Mar 2016 10:51:12 -0500 Subject: [PATCH 08/51] BLD: unshallow the travis git checkout so conda build can clone it --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 111b1183..206efe8c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,6 +25,7 @@ before_install: - sed -i "s/scipy==.*/scipy==$SCIPY_VERSION/" etc/requirements.txt install: - conda install conda-build --yes + - conda create -n testenv --yes -c quantopian pip python=$TRAVIS_PYTHON_VERSION numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION libgfortran=1.0 ta-lib=0.4.9 - source activate testenv - pip install --upgrade pip coverage coveralls --cache-dir=$CACHE_DIR @@ -35,8 +36,11 @@ install: before_script: - pip freeze | sort script: + # unshallow the clone so the conda build can clone it. + - git fetch --unshallow - IFS='.' read -r -a conda_npy_version <<< "$NUMPY_VERSION" - conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy="${conda_npy_version[0]}.${conda_npy_version[1]}" # accepts only major.minor + - nosetests tests/ - flake8 zipline tests after_success: From 70cb317610d684c9014f860b6a3170071e02b338 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 3 Mar 2016 11:23:09 -0500 Subject: [PATCH 09/51] BLD: Calculate travis cache dir --- .travis.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 206efe8c..7b2a61fa 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,13 +4,13 @@ matrix: fast_finish: true include: - python: 2.7 - env: PANDAS_VERSION=0.16.1 NUMPY_VERSION=1.9.2 SCIPY_VERSION=0.15.1 CACHE_DIR=$HOME/.cache/.pip/pip_np19 + env: PANDAS_VERSION=0.16.1 NUMPY_VERSION=1.9.2 SCIPY_VERSION=0.15.1 - python: 2.7 - env: PANDAS_VERSION=0.17.1 NUMPY_VERSION=1.10.2 SCIPY_VERSION=0.16.1 CACHE_DIR=$HOME/.cache/.pip/pip_np110 + env: PANDAS_VERSION=0.17.1 NUMPY_VERSION=1.10.2 SCIPY_VERSION=0.16.1 - python: 3.4 - env: PANDAS_VERSION=0.16.1 NUMPY_VERSION=1.9.2 SCIPY_VERSION=0.15.1 CACHE_DIR=$HOME/.cache/.pip/pip_np19 + env: PANDAS_VERSION=0.16.1 NUMPY_VERSION=1.9.2 SCIPY_VERSION=0.15.1 - python: 3.4 - env: PANDAS_VERSION=0.17.1 NUMPY_VERSION=1.10.2 SCIPY_VERSION=0.16.1 CACHE_DIR=$HOME/.cache/.pip/pip_np110 + env: PANDAS_VERSION=0.17.1 NUMPY_VERSION=1.10.2 SCIPY_VERSION=0.16.1 cache: directories: - $HOME/.cache/.pip/ @@ -28,6 +28,9 @@ install: - conda create -n testenv --yes -c quantopian pip python=$TRAVIS_PYTHON_VERSION numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION libgfortran=1.0 ta-lib=0.4.9 - source activate testenv + - IFS='.' read -r -a NPY_VERSION_ARR <<< "$NUMPY_VERSION" + - NPY_VERSION_MAJ_MIN=${NPY_VERSION_ARR[0]}${NPY_VERSION_ARR[1]} + - CACHE_DIR="$HOME/.cache/.pip/pip_np""$NPY_VERSION_MAJ_MIN" - pip install --upgrade pip coverage coveralls --cache-dir=$CACHE_DIR - pip install -r etc/requirements.txt --cache-dir=$CACHE_DIR - pip install -r etc/requirements_dev.txt --cache-dir=$CACHE_DIR @@ -38,8 +41,7 @@ before_script: script: # unshallow the clone so the conda build can clone it. - git fetch --unshallow - - IFS='.' read -r -a conda_npy_version <<< "$NUMPY_VERSION" - - conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy="${conda_npy_version[0]}.${conda_npy_version[1]}" # accepts only major.minor + - conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN - nosetests tests/ - flake8 zipline tests From 67b1e6b436c4ff4001ae6950b21ccfc7b6d30df5 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 3 Mar 2016 11:23:31 -0500 Subject: [PATCH 10/51] BLD: Stay in build dir to find tests --- appveyor.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index a916e07d..90387aed 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -110,9 +110,8 @@ install: # this uses git requirements right now - cmd: pip install -r etc/requirements_blaze.txt - cmd: pip install -e .[talib] + - cmd: pip freeze | sort test_script: - - cd \ - - pip freeze | sort - nosetests tests/ - flake8 zipline tests From d5f01c2ac4c071f4f4b80457b02f32952a70fd89 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 3 Mar 2016 11:29:29 -0500 Subject: [PATCH 11/51] BLD: Add pip cache to appveyor cache --- appveyor.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 90387aed..40f1c398 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -85,7 +85,10 @@ environment: # We always use a 64-bit machine, but can build x86 distributions # with the PYTHON_ARCH variable (which is used by CMD_IN_ENV). platform: - - x64 + - x64 + +cache: + - '%LOCALAPPDATA%\pip\Cache' # all our python builds have to happen in tests_script... build: false From 65cb8ecc26045a382012bf2190e0190ab8287c26 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 3 Mar 2016 13:43:57 -0500 Subject: [PATCH 12/51] BLD: Build other conda recipes. Build zipline recipe on appveyor. --- .travis.yml | 4 ++++ appveyor.yml | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7b2a61fa..510550dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,6 +39,10 @@ install: before_script: - pip freeze | sort script: + - conda build conda/bcolz -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NUMPY_VERSION + - conda build conda/cyordereddict -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NUMPY_VERSION + - conda build conda/logbook -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NUMPY_VERSION + - conda build conda/ta-lib -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NUMPY_VERSION # unshallow the clone so the conda build can clone it. - git fetch --unshallow - conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN diff --git a/appveyor.yml b/appveyor.yml index 40f1c398..2a86d222 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -103,7 +103,11 @@ install: - conda info -a - conda install conda-build - - cmd: '%CMD_IN_ENV% conda build conda\ta-lib -q' + - cmd: '%CMD_IN_ENV% conda build conda\bcolz -q --python=%PYTHON_VERSION% --numpy=%NUMPY_VERSION%' + - cmd: '%CMD_IN_ENV% conda build conda\cyordereddict -q --python=%PYTHON_VERSION% --numpy=%NUMPY_VERSION%' + - cmd: '%CMD_IN_ENV% conda build conda\logbook -q --python=%PYTHON_VERSION% --numpy=%NUMPY_VERSION%' + - cmd: '%CMD_IN_ENV% conda build conda\ta-lib -q --python=%PYTHON_VERSION% --numpy=%NUMPY_VERSION%' + - cmd: '%CMD_IN_ENV% conda build conda\zipline --python=%PYTHON_VERSION% --numpy=%NUMPY_VERSION%' - cmd: conda create -n testenv --yes --use-local pip numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 - cmd: activate testenv From cb5c9b4afd3078fc581bfd35d72ac6b844ec913c Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 3 Mar 2016 15:30:28 -0500 Subject: [PATCH 13/51] BLD: Use only major.minor numpy version when building packages --- .travis.yml | 8 ++++---- appveyor.yml | 12 +++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 510550dc..d0124f1d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,10 +39,10 @@ install: before_script: - pip freeze | sort script: - - conda build conda/bcolz -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NUMPY_VERSION - - conda build conda/cyordereddict -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NUMPY_VERSION - - conda build conda/logbook -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NUMPY_VERSION - - conda build conda/ta-lib -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NUMPY_VERSION + - conda build conda/bcolz -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN + - conda build conda/cyordereddict -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN + - conda build conda/logbook -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN + - conda build conda/ta-lib -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN # unshallow the clone so the conda build can clone it. - git fetch --unshallow - conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN diff --git a/appveyor.yml b/appveyor.yml index 2a86d222..f561bbde 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -103,11 +103,13 @@ install: - conda info -a - conda install conda-build - - cmd: '%CMD_IN_ENV% conda build conda\bcolz -q --python=%PYTHON_VERSION% --numpy=%NUMPY_VERSION%' - - cmd: '%CMD_IN_ENV% conda build conda\cyordereddict -q --python=%PYTHON_VERSION% --numpy=%NUMPY_VERSION%' - - cmd: '%CMD_IN_ENV% conda build conda\logbook -q --python=%PYTHON_VERSION% --numpy=%NUMPY_VERSION%' - - cmd: '%CMD_IN_ENV% conda build conda\ta-lib -q --python=%PYTHON_VERSION% --numpy=%NUMPY_VERSION%' - - cmd: '%CMD_IN_ENV% conda build conda\zipline --python=%PYTHON_VERSION% --numpy=%NUMPY_VERSION%' + - ps: $NPY_VERSION_ARR=$env:NUMPY_VERSION -split '.', 0, 'simplematch' + - ps: $env:NPY_VERSION_MAJ_MIN=$NPY_VERSION_ARR[0..1] -join "" + - cmd: '%CMD_IN_ENV% conda build conda/bcolz -q --python=%PYTHON_VERSION% --numpy=%NPY_VERSION_MAJ_MIN%' + - cmd: '%CMD_IN_ENV% conda build conda/cyordereddict -q --python=%PYTHON_VERSION% --numpy=%NPY_VERSION_MAJ_MIN%' + - cmd: '%CMD_IN_ENV% conda build conda/logbook -q --python=%PYTHON_VERSION% --numpy=%NPY_VERSION_MAJ_MIN%' + - cmd: '%CMD_IN_ENV% conda build conda/ta-lib -q --python=%PYTHON_VERSION% --numpy=%NPY_VERSION_MAJ_MIN%' + - cmd: '%CMD_IN_ENV% conda build conda/zipline --python=%PYTHON_VERSION% --numpy=%NPY_VERSION_MAJ_MIN%' - cmd: conda create -n testenv --yes --use-local pip numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 - cmd: activate testenv From bf6047f80d1e44720d563fc5a448f762746c1b2b Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 3 Mar 2016 16:15:30 -0500 Subject: [PATCH 14/51] BLD: Peg the conda-build version --- .travis.yml | 2 +- appveyor.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d0124f1d..a0aeb1be 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,7 @@ before_install: - sed -i "s/pandas==.*/pandas==$PANDAS_VERSION/" etc/requirements.txt - sed -i "s/scipy==.*/scipy==$SCIPY_VERSION/" etc/requirements.txt install: - - conda install conda-build --yes + - conda install conda-build=1.19.2 --yes - conda create -n testenv --yes -c quantopian pip python=$TRAVIS_PYTHON_VERSION numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION libgfortran=1.0 ta-lib=0.4.9 - source activate testenv diff --git a/appveyor.yml b/appveyor.yml index f561bbde..2ee0e797 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -102,7 +102,7 @@ install: - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% - conda info -a - - conda install conda-build + - conda install conda-build=1.19.2 --yes - ps: $NPY_VERSION_ARR=$env:NUMPY_VERSION -split '.', 0, 'simplematch' - ps: $env:NPY_VERSION_MAJ_MIN=$NPY_VERSION_ARR[0..1] -join "" - cmd: '%CMD_IN_ENV% conda build conda/bcolz -q --python=%PYTHON_VERSION% --numpy=%NPY_VERSION_MAJ_MIN%' From e6d91e28581bca9872cba755236566e978268539 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 3 Mar 2016 22:23:35 -0500 Subject: [PATCH 15/51] BLD: Remove cygwin's git --- appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 2ee0e797..98f14281 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -97,6 +97,8 @@ init: - "ECHO %PYTHON_VERSION% %PYTHON%" install: + # Get cygwin's git out of our PATH. See https://github.com/omnia-md/conda-dev-recipes/pull/16/files#diff-180360612c6b8c4ed830919bbb4dd459 + - "del C:\\cygwin\\bin\\git.exe" # this installs the appropriate Miniconda (Py2/Py3, 32/64 bit), - powershell .\etc\install.ps1 - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% From f11f5adc6fe906a9aab2cd7adb963231e310c448 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Sat, 5 Mar 2016 12:06:29 -0500 Subject: [PATCH 16/51] BLD: Don't download dependencies on windows conda build --- conda/zipline/bld.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/zipline/bld.bat b/conda/zipline/bld.bat index 87b1481d..ebb1aa4d 100644 --- a/conda/zipline/bld.bat +++ b/conda/zipline/bld.bat @@ -1,4 +1,4 @@ -"%PYTHON%" setup.py install +"%PYTHON%" setup.py install --single-version-externally-managed --record=record.txt if errorlevel 1 exit 1 :: Add more build steps here, if they are necessary. From 0ac379eb749e1f4dc8bebf0e0a9eeb9a8f5577c9 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Sat, 5 Mar 2016 12:52:49 -0500 Subject: [PATCH 17/51] BLD: Disallowed to uninstall pip on appveyor --- appveyor.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 98f14281..35e10c37 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -115,7 +115,6 @@ install: - cmd: conda create -n testenv --yes --use-local pip numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 - cmd: activate testenv - - cmd: pip install --upgrade pip coverage coveralls - cmd: pip install -r etc/requirements.txt - cmd: pip install -r etc/requirements_dev.txt # this uses git requirements right now From d873038a7ec81aa70d7c1aaedca9d06043e172be Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Sat, 5 Mar 2016 16:22:42 -0500 Subject: [PATCH 18/51] BUG: Specify int64 instead of system int to handle 32bit python --- tests/utils/test_preprocess.py | 2 +- zipline/data/_adjustments.pyx | 3 ++- zipline/data/us_equity_pricing.py | 5 +++-- zipline/pipeline/loaders/synthetic.py | 3 ++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/utils/test_preprocess.py b/tests/utils/test_preprocess.py index 4f425b16..2626cedb 100644 --- a/tests/utils/test_preprocess.py +++ b/tests/utils/test_preprocess.py @@ -280,7 +280,7 @@ class PreprocessTestCase(TestCase): self.assertIs(c_ret, good_c) with self.assertRaises(TypeError) as e: - foo(good_a, arange(3), good_c) + foo(good_a, arange(3, dtype='int64'), good_c) expected_message = ( "{qualname}() expected a value with dtype 'datetime64[ns]'" diff --git a/zipline/data/_adjustments.pyx b/zipline/data/_adjustments.pyx index b53474f2..5fded76b 100644 --- a/zipline/data/_adjustments.pyx +++ b/zipline/data/_adjustments.pyx @@ -18,6 +18,7 @@ from cpython cimport ( ) from numpy import ( + int64, uint32, zeros, ) @@ -208,7 +209,7 @@ cpdef load_adjustments_from_sqlite(object adjustments_db, # sqlite3.Connection dict col_adjustments cdef ndarray[int64_t, ndim=1] _dates_seconds = \ - dates.values.astype('datetime64[s]').view(int) + dates.values.astype('datetime64[s]').view(int64) # Pre-populate date index cache. for i, dt in enumerate(_dates_seconds): diff --git a/zipline/data/us_equity_pricing.py b/zipline/data/us_equity_pricing.py index 6b15f013..8d508b9e 100644 --- a/zipline/data/us_equity_pricing.py +++ b/zipline/data/us_equity_pricing.py @@ -302,8 +302,9 @@ class DailyBarWriterFromCSVs(BcolzDailyBarWriter): return array.astype(uint32) elif colname == 'day': nanos_per_second = (1000 * 1000 * 1000) - self.check_uint_safe(arrmax.view(int) / nanos_per_second, colname) - return (array.view(int) / nanos_per_second).astype(uint32) + self.check_uint_safe(arrmax.view(int64) / nanos_per_second, + colname) + return (array.view(int64) / nanos_per_second).astype(uint32) @staticmethod def check_uint_safe(value, colname): diff --git a/zipline/pipeline/loaders/synthetic.py b/zipline/pipeline/loaders/synthetic.py index 1d983a53..a212b2c0 100644 --- a/zipline/pipeline/loaders/synthetic.py +++ b/zipline/pipeline/loaders/synthetic.py @@ -173,7 +173,8 @@ class SeededRandomLoader(PrecomputedLoader): """ Return uniformly-distributed integers between 0 and 100. """ - return self.state.random_integers(low=0, high=100, size=shape) + return (self.state.random_integers(low=0, high=100, size=shape) + .astype('int64')) # default is system int def _datetime_values(self, shape): """ From 863e7c39ee6fee6a4c67dd1ca972f83cfec17a66 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Sat, 5 Mar 2016 17:16:39 -0500 Subject: [PATCH 19/51] BLD: Try 64bit python --- appveyor.yml | 66 +--------------------------------------------------- 1 file changed, 1 insertion(+), 65 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 35e10c37..ff9481ee 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -11,77 +11,13 @@ environment: matrix: - PYTHON: "C:\\Python34_64" PYTHON_VERSION: "3.4" - PYTHON_ARCH: "32" + PYTHON_ARCH: "64" PANDAS_VERSION: "0.16.1" NUMPY_VERSION: "1.9.2" CONDA_PY: "34" CONDA_NPY: "19" SCIPY_VERSION: "0.15.1" -# - PYTHON: "C:\\Python27_64" -# PYTHON_VERSION: "2.7" -# PYTHON_ARCH: "32" -# PANDAS_VERSION: "0.16.1" -# NUMPY_VERSION: "1.9.2" -# CONDA_PY: "27" -# CONDA_NPY: "19" -# SCIPY_VERSION: "0.15.1" -# -# - PYTHON: "C:\\Python34_64" -# PYTHON_VERSION: "3.4" -# PYTHON_ARCH: "32" -# PANDAS_VERSION: "0.17.1" -# NUMPY_VERSION: "1.10.2" -# CONDA_PY: "34" -# CONDA_NPY: "19" -# SCIPY_VERSION: "0.16.1" -# -# - PYTHON: "C:\\Python27_64" -# PYTHON_VERSION: "2.7" -# PYTHON_ARCH: "32" -# PANDAS_VERSION: "0.17.1" -# NUMPY_VERSION: "1.10.2" -# CONDA_PY: "27" -# CONDA_NPY: "19" -# SCIPY_VERSION: "0.16.1" -# -# - PYTHON: "C:\\Python34_64" -# PYTHON_VERSION: "3.4" -# PYTHON_ARCH: "32" -# PANDAS_VERSION: "0.16.1" -# NUMPY_VERSION: "1.9.2" -# CONDA_PY: "64" -# CONDA_NPY: "19" -# SCIPY_VERSION: "0.15.1" -# -# - PYTHON: "C:\\Python27_64" -# PYTHON_VERSION: "2.7" -# PYTHON_ARCH: "32" -# PANDAS_VERSION: "0.16.1" -# NUMPY_VERSION: "1.9.2" -# CONDA_PY: "64" -# CONDA_NPY: "19" -# SCIPY_VERSION: "0.15.1" -# -# - PYTHON: "C:\\Python34_64" -# PYTHON_VERSION: "3.4" -# PYTHON_ARCH: "32" -# PANDAS_VERSION: "0.17.1" -# NUMPY_VERSION: "1.10.2" -# CONDA_PY: "64" -# CONDA_NPY: "19" -# SCIPY_VERSION: "0.16.1" -# -# - PYTHON: "C:\\Python27_64" -# PYTHON_VERSION: "2.7" -# PYTHON_ARCH: "64" -# PANDAS_VERSION: "0.17.1" -# NUMPY_VERSION: "1.10.2" -# CONDA_PY: "27" -# CONDA_NPY: "19" -# SCIPY_VERSION: "0.16.1" - - # We always use a 64-bit machine, but can build x86 distributions # with the PYTHON_ARCH variable (which is used by CMD_IN_ENV). platform: From f89cbbdde27acf9a00486692aa92ba875ba9f715 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Sun, 6 Mar 2016 09:38:27 -0500 Subject: [PATCH 20/51] BLD: Include commit hash in conda build string when not at exact tag --- conda/zipline/meta.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/conda/zipline/meta.yaml b/conda/zipline/meta.yaml index ebb82b5c..bc86cec4 100644 --- a/conda/zipline/meta.yaml +++ b/conda/zipline/meta.yaml @@ -2,17 +2,20 @@ package: name: zipline - version: {{ environ.get('GIT_DESCRIBE_TAG', '')}} + version: {{ GIT_DESCRIBE_TAG|replace('v', '') }} build: - number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }} + number: {{ GIT_DESCRIBE_NUMBER|int }} + string: np{{ NPY_VER|replace('.', '') }}py{{ PY_VER|replace('.', '') }}_{{ ( + GIT_BUILD_STR if GIT_DESCRIBE_NUMBER|int != 0 else '0' + ) }} source: git_url: ../../ requirements: build: - - python + - python {{ PY_VER }}* {% for req in data.get('build_requires', []) -%} - {{req}} {% endfor %} From e4b39e68d05522169328c84c4b6c78ed8618db04 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Sun, 6 Mar 2016 11:34:45 -0500 Subject: [PATCH 21/51] BLD: Echo more params --- appveyor.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index ff9481ee..b606d007 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -30,7 +30,8 @@ cache: build: false init: - - "ECHO %PYTHON_VERSION% %PYTHON%" + - "ECHO %PYTHON_VERSION% %PYTHON_ARCH% %PYTHON%" + - "ECHO %NUMPY_VERSION%" install: # Get cygwin's git out of our PATH. See https://github.com/omnia-md/conda-dev-recipes/pull/16/files#diff-180360612c6b8c4ed830919bbb4dd459 From 8a94af3f973ffde1de4c43a145b7f69825ebb073 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Sun, 6 Mar 2016 15:37:34 -0500 Subject: [PATCH 22/51] BLD: Need to specify python version --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index b606d007..c557fcc6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -50,7 +50,7 @@ install: - cmd: '%CMD_IN_ENV% conda build conda/ta-lib -q --python=%PYTHON_VERSION% --numpy=%NPY_VERSION_MAJ_MIN%' - cmd: '%CMD_IN_ENV% conda build conda/zipline --python=%PYTHON_VERSION% --numpy=%NPY_VERSION_MAJ_MIN%' - - cmd: conda create -n testenv --yes --use-local pip numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 + - cmd: conda create -n testenv --yes --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 - cmd: activate testenv - cmd: pip install -r etc/requirements.txt - cmd: pip install -r etc/requirements_dev.txt From cbc1316a7a0607ae1df7fef1669de032639710c9 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Sun, 6 Mar 2016 15:43:20 -0500 Subject: [PATCH 23/51] MAINT: Moved scripts to appveyor dir --- appveyor.yml | 4 ++-- {etc => ci/appveyor}/install.ps1 | 0 {etc => ci/appveyor}/run_with_env.cmd | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename {etc => ci/appveyor}/install.ps1 (100%) rename {etc => ci/appveyor}/run_with_env.cmd (100%) diff --git a/appveyor.yml b/appveyor.yml index c557fcc6..7b1e88ce 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -6,7 +6,7 @@ environment: # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the # /E:ON and /V:ON options are not enabled in the batch script intepreter # See: http://stackoverflow.com/a/13751649/163740 - CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\etc\\run_with_env.cmd" + CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci\\appveyor\\run_with_env.cmd" matrix: - PYTHON: "C:\\Python34_64" @@ -37,7 +37,7 @@ install: # Get cygwin's git out of our PATH. See https://github.com/omnia-md/conda-dev-recipes/pull/16/files#diff-180360612c6b8c4ed830919bbb4dd459 - "del C:\\cygwin\\bin\\git.exe" # this installs the appropriate Miniconda (Py2/Py3, 32/64 bit), - - powershell .\etc\install.ps1 + - powershell .\ci\appveyor\install.ps1 - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% - conda info -a diff --git a/etc/install.ps1 b/ci/appveyor/install.ps1 similarity index 100% rename from etc/install.ps1 rename to ci/appveyor/install.ps1 diff --git a/etc/run_with_env.cmd b/ci/appveyor/run_with_env.cmd similarity index 100% rename from etc/run_with_env.cmd rename to ci/appveyor/run_with_env.cmd From e53fc9abceef4249af0bb9fa079cbffae5e6a54e Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Sun, 6 Mar 2016 09:20:30 -0500 Subject: [PATCH 24/51] BLD: Use the bcolz we built Fixing VC usage --- appveyor.yml | 5 ++++- ci/appveyor/vcvars64.bat | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 ci/appveyor/vcvars64.bat diff --git a/appveyor.yml b/appveyor.yml index 7b1e88ce..346e2ee3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -42,6 +42,9 @@ install: - conda info -a - conda install conda-build=1.19.2 --yes + # https://blog.ionelmc.ro/2014/12/21/compiling-python-extensions-on-windows/ for 64bit C compilation + - ps: copy .\ci\appveyor\vcvars64.bat "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64" + - ps: $NPY_VERSION_ARR=$env:NUMPY_VERSION -split '.', 0, 'simplematch' - ps: $env:NPY_VERSION_MAJ_MIN=$NPY_VERSION_ARR[0..1] -join "" - cmd: '%CMD_IN_ENV% conda build conda/bcolz -q --python=%PYTHON_VERSION% --numpy=%NPY_VERSION_MAJ_MIN%' @@ -50,7 +53,7 @@ install: - cmd: '%CMD_IN_ENV% conda build conda/ta-lib -q --python=%PYTHON_VERSION% --numpy=%NPY_VERSION_MAJ_MIN%' - cmd: '%CMD_IN_ENV% conda build conda/zipline --python=%PYTHON_VERSION% --numpy=%NPY_VERSION_MAJ_MIN%' - - cmd: conda create -n testenv --yes --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 + - cmd: conda create -n testenv --yes --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 bcolz=0.12.1 - cmd: activate testenv - cmd: pip install -r etc/requirements.txt - cmd: pip install -r etc/requirements_dev.txt diff --git a/ci/appveyor/vcvars64.bat b/ci/appveyor/vcvars64.bat new file mode 100644 index 00000000..ef77b9d3 --- /dev/null +++ b/ci/appveyor/vcvars64.bat @@ -0,0 +1 @@ +CALL "C:\Program Files\Microsoft SDKs\Windows\v7.1\Bin\SetEnv.cmd" /x64 From 2d9846ef1ca26e6e5d28284782cf3542c8be191a Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Sun, 6 Mar 2016 17:23:05 -0500 Subject: [PATCH 25/51] BLD: Add more to build matrix Windows has no numpy 1.10.2 built, so bumping to 1.10.4 --- .travis.yml | 4 ++-- appveyor.yml | 55 ++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 51 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index a0aeb1be..fbc09158 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,11 +6,11 @@ matrix: - python: 2.7 env: PANDAS_VERSION=0.16.1 NUMPY_VERSION=1.9.2 SCIPY_VERSION=0.15.1 - python: 2.7 - env: PANDAS_VERSION=0.17.1 NUMPY_VERSION=1.10.2 SCIPY_VERSION=0.16.1 + env: PANDAS_VERSION=0.17.1 NUMPY_VERSION=1.10.4 SCIPY_VERSION=0.16.1 - python: 3.4 env: PANDAS_VERSION=0.16.1 NUMPY_VERSION=1.9.2 SCIPY_VERSION=0.15.1 - python: 3.4 - env: PANDAS_VERSION=0.17.1 NUMPY_VERSION=1.10.2 SCIPY_VERSION=0.16.1 + env: PANDAS_VERSION=0.17.1 NUMPY_VERSION=1.10.4 SCIPY_VERSION=0.16.1 cache: directories: - $HOME/.cache/.pip/ diff --git a/appveyor.yml b/appveyor.yml index 346e2ee3..508d6d23 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,5 +1,5 @@ -matrix: - fast_finish: true +#matrix: +# fast_finish: true environment: global: @@ -9,15 +9,58 @@ environment: CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci\\appveyor\\run_with_env.cmd" matrix: - - PYTHON: "C:\\Python34_64" - PYTHON_VERSION: "3.4" + - PYTHON: "C:\\Python27_64" + PYTHON_VERSION: "2.7" PYTHON_ARCH: "64" PANDAS_VERSION: "0.16.1" NUMPY_VERSION: "1.9.2" - CONDA_PY: "34" - CONDA_NPY: "19" SCIPY_VERSION: "0.15.1" + - PYTHON_VERSION: "2.7" + PYTHON_ARCH: "32" + PANDAS_VERSION: "0.16.1" + NUMPY_VERSION: "1.9.2" + SCIPY_VERSION: "0.15.1" + + - PYTHON_VERSION: "3.4" + PYTHON_ARCH: "64" + PANDAS_VERSION: "0.16.1" + NUMPY_VERSION: "1.9.2" + SCIPY_VERSION: "0.15.1" + + - PYTHON_VERSION: "3.4" + PYTHON_ARCH: "32" + PANDAS_VERSION: "0.16.1" + NUMPY_VERSION: "1.9.2" + SCIPY_VERSION: "0.15.1" + + - PYTHON_VERSION: "2.7" + PYTHON_ARCH: "64" + PANDAS_VERSION: "0.17.1" + NUMPY_VERSION: "1.10.4" + SCIPY_VERSION: "0.16.1" + + - PYTHON: "C:\\Python27_64" + PYTHON_VERSION: "2.7" + PYTHON_ARCH: "32" + PANDAS_VERSION: "0.17.1" + NUMPY_VERSION: "1.10.4" + SCIPY_VERSION: "0.16.1" + + - PYTHON: "C:\\Python34_64" + PYTHON_VERSION: "3.4" + PYTHON_ARCH: "64" + PANDAS_VERSION: "0.17.1" + NUMPY_VERSION: "1.10.4" + SCIPY_VERSION: "0.16.1" + + - PYTHON: "C:\\Python34_64" + PYTHON_VERSION: "3.4" + PYTHON_ARCH: "32" + PANDAS_VERSION: "0.17.1" + NUMPY_VERSION: "1.10.4" + SCIPY_VERSION: "0.16.1" + # We always use a 64-bit machine, but can build x86 distributions # with the PYTHON_ARCH variable (which is used by CMD_IN_ENV). platform: From 8386da903065d69bde385f6e2a34da3ffd6a2346 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Sun, 6 Mar 2016 23:01:00 -0500 Subject: [PATCH 26/51] BLD: Update requirements with matrix vals, like we do travis --- appveyor.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 508d6d23..94c045f3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -82,6 +82,9 @@ install: # this installs the appropriate Miniconda (Py2/Py3, 32/64 bit), - powershell .\ci\appveyor\install.ps1 - SET PATH=%PYTHON%;%PYTHON%\Scripts;%PATH% + - sed -i "s/numpy==.*/numpy==%NUMPY_VERSION%/" etc/requirements.txt + - sed -i "s/pandas==.*/pandas==%PANDAS_VERSION%/" etc/requirements.txt + - sed -i "s/scipy==.*/scipy==%SCIPY_VERSION%/" etc/requirements.txt - conda info -a - conda install conda-build=1.19.2 --yes From 335f73e1e9b9e4d15b5c0e2c661aba6883b73e8b Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 7 Mar 2016 09:43:26 -0500 Subject: [PATCH 27/51] TST: Windows fixups --- tests/pipeline/test_adjusted_array.py | 2 +- tests/pipeline/test_numerical_expression.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/pipeline/test_adjusted_array.py b/tests/pipeline/test_adjusted_array.py index a5fab747..a2a8cde0 100644 --- a/tests/pipeline/test_adjusted_array.py +++ b/tests/pipeline/test_adjusted_array.py @@ -365,7 +365,7 @@ class AdjustedArrayTestCase(TestCase): frame[0, 0] = 5.0 def test_bad_input(self): - msg = "Mask shape \(2, 3\) != data shape \(5, 5\)" + msg = "Mask shape \(2L?, 3L?\) != data shape \(5L?, 5L?\)" data = arange(25).reshape(5, 5) bad_mask = array([[0, 1, 1], [0, 0, 1]], dtype=bool) diff --git a/tests/pipeline/test_numerical_expression.py b/tests/pipeline/test_numerical_expression.py index 040ed124..9922004d 100644 --- a/tests/pipeline/test_numerical_expression.py +++ b/tests/pipeline/test_numerical_expression.py @@ -34,7 +34,7 @@ from zipline.pipeline.expression import ( NumericalExpression, NUMEXPR_MATH_FUNCS, ) -from zipline.testing import check_arrays +from zipline.testing import check_allclose from zipline.utils.numpy_utils import datetime64ns_dtype, float64_dtype @@ -91,7 +91,7 @@ class NumericalExpressionTestCase(TestCase): self.mask.columns, self.mask.values, ) - check_arrays(result, expected) + check_allclose(result, expected) def check_constant_output(self, expr, expected): self.assertFalse(isnan(expected)) From dd8175b1d9e4f347034d8b43ed4d327df9657400 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 7 Mar 2016 09:43:46 -0500 Subject: [PATCH 28/51] TST: Forward arguments to numpy --- zipline/testing/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zipline/testing/core.py b/zipline/testing/core.py index ea26bceb..15d1a703 100644 --- a/zipline/testing/core.py +++ b/zipline/testing/core.py @@ -568,7 +568,8 @@ def check_allclose(actual, """ if type(actual) != type(desired): raise AssertionError("%s != %s" % (type(actual), type(desired))) - return assert_allclose(actual, desired, err_msg=err_msg, verbose=True) + return assert_allclose(actual, desired, rtol=rtol, atol=atol, + err_msg=err_msg, verbose=verbose) def check_arrays(x, y, err_msg='', verbose=True): From b3801c5bb62ce615ef1e1f6eb985dd1358ef4e56 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 7 Mar 2016 10:20:31 -0500 Subject: [PATCH 29/51] BLD: Cancel previously queued appveyor builds for a PR --- appveyor.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 94c045f3..eef471a8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -77,6 +77,16 @@ init: - "ECHO %NUMPY_VERSION%" install: + # If there is a newer build queued for the same PR, cancel this one. + # The AppVeyor 'rollout builds' option is supposed to serve the same + # purpose but it is problematic because it tends to cancel builds pushed + # directly to master instead of just PR builds (or the converse). + # credits: JuliaLang developers. + - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` + https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` + Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` + throw "There are newer queued builds for this pull request, failing early." } + # Get cygwin's git out of our PATH. See https://github.com/omnia-md/conda-dev-recipes/pull/16/files#diff-180360612c6b8c4ed830919bbb4dd459 - "del C:\\cygwin\\bin\\git.exe" # this installs the appropriate Miniconda (Py2/Py3, 32/64 bit), From c49e77723b0b31d7c5febd9f7ecc43768636b1bf Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 7 Mar 2016 10:24:33 -0500 Subject: [PATCH 30/51] BLD: run_with_env.cmd requires CONDA_PY --- appveyor.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index eef471a8..1b2f3991 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -102,12 +102,14 @@ install: - ps: copy .\ci\appveyor\vcvars64.bat "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64" - ps: $NPY_VERSION_ARR=$env:NUMPY_VERSION -split '.', 0, 'simplematch' - - ps: $env:NPY_VERSION_MAJ_MIN=$NPY_VERSION_ARR[0..1] -join "" - - cmd: '%CMD_IN_ENV% conda build conda/bcolz -q --python=%PYTHON_VERSION% --numpy=%NPY_VERSION_MAJ_MIN%' - - cmd: '%CMD_IN_ENV% conda build conda/cyordereddict -q --python=%PYTHON_VERSION% --numpy=%NPY_VERSION_MAJ_MIN%' - - cmd: '%CMD_IN_ENV% conda build conda/logbook -q --python=%PYTHON_VERSION% --numpy=%NPY_VERSION_MAJ_MIN%' - - cmd: '%CMD_IN_ENV% conda build conda/ta-lib -q --python=%PYTHON_VERSION% --numpy=%NPY_VERSION_MAJ_MIN%' - - cmd: '%CMD_IN_ENV% conda build conda/zipline --python=%PYTHON_VERSION% --numpy=%NPY_VERSION_MAJ_MIN%' + - ps: $env:CONDA_NPY=$NPY_VERSION_ARR[0..1] -join "" + - ps: $PY_VERSION_ARR=$env:PYTHON_VERSION -split '.', 0, 'simplematch' + - ps: $env:CONDA_PY=$PY_VERSION_ARR[0..1] -join "" + - cmd: '%CMD_IN_ENV% conda build conda/bcolz --python=%CONDA_PY% --numpy=%CONDA_NPY%' + - cmd: '%CMD_IN_ENV% conda build conda/cyordereddict --python=%CONDA_PY% --numpy=%CONDA_NPY%' + - cmd: '%CMD_IN_ENV% conda build conda/logbook --python=%CONDA_PY% --numpy=%CONDA_NPY%' + - cmd: '%CMD_IN_ENV% conda build conda/ta-lib --python=%CONDA_PY% --numpy=%CONDA_NPY%' + - cmd: '%CMD_IN_ENV% conda build conda/zipline --python=%CONDA_PY% --numpy=%CONDA_NPY%' - cmd: conda create -n testenv --yes --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 bcolz=0.12.1 - cmd: activate testenv From 8215c0f59b10385fcdbdc9b198618dd424c33fa2 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 7 Mar 2016 10:39:01 -0500 Subject: [PATCH 31/51] BLD: Calculate python dir --- appveyor.yml | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 1b2f3991..05ce7a6c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,8 +9,7 @@ environment: CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci\\appveyor\\run_with_env.cmd" matrix: - - PYTHON: "C:\\Python27_64" - PYTHON_VERSION: "2.7" + - PYTHON_VERSION: "2.7" PYTHON_ARCH: "64" PANDAS_VERSION: "0.16.1" NUMPY_VERSION: "1.9.2" @@ -40,22 +39,19 @@ environment: NUMPY_VERSION: "1.10.4" SCIPY_VERSION: "0.16.1" - - PYTHON: "C:\\Python27_64" - PYTHON_VERSION: "2.7" + - PYTHON_VERSION: "2.7" PYTHON_ARCH: "32" PANDAS_VERSION: "0.17.1" NUMPY_VERSION: "1.10.4" SCIPY_VERSION: "0.16.1" - - PYTHON: "C:\\Python34_64" - PYTHON_VERSION: "3.4" + - PYTHON_VERSION: "3.4" PYTHON_ARCH: "64" PANDAS_VERSION: "0.17.1" NUMPY_VERSION: "1.10.4" SCIPY_VERSION: "0.16.1" - - PYTHON: "C:\\Python34_64" - PYTHON_VERSION: "3.4" + - PYTHON_VERSION: "3.4" PYTHON_ARCH: "32" PANDAS_VERSION: "0.17.1" NUMPY_VERSION: "1.10.4" @@ -87,6 +83,11 @@ install: Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` throw "There are newer queued builds for this pull request, failing early." } + - ps: $NPY_VERSION_ARR=$env:NUMPY_VERSION -split '.', 0, 'simplematch' + - ps: $env:CONDA_NPY=$NPY_VERSION_ARR[0..1] -join "" + - ps: $PY_VERSION_ARR=$env:PYTHON_VERSION -split '.', 0, 'simplematch' + - ps: $env:CONDA_PY=$PY_VERSION_ARR[0..1] -join "" + - SET PYTHON=C:\Python%CONDA_PY%_64 # Get cygwin's git out of our PATH. See https://github.com/omnia-md/conda-dev-recipes/pull/16/files#diff-180360612c6b8c4ed830919bbb4dd459 - "del C:\\cygwin\\bin\\git.exe" # this installs the appropriate Miniconda (Py2/Py3, 32/64 bit), @@ -101,10 +102,6 @@ install: # https://blog.ionelmc.ro/2014/12/21/compiling-python-extensions-on-windows/ for 64bit C compilation - ps: copy .\ci\appveyor\vcvars64.bat "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64" - - ps: $NPY_VERSION_ARR=$env:NUMPY_VERSION -split '.', 0, 'simplematch' - - ps: $env:CONDA_NPY=$NPY_VERSION_ARR[0..1] -join "" - - ps: $PY_VERSION_ARR=$env:PYTHON_VERSION -split '.', 0, 'simplematch' - - ps: $env:CONDA_PY=$PY_VERSION_ARR[0..1] -join "" - cmd: '%CMD_IN_ENV% conda build conda/bcolz --python=%CONDA_PY% --numpy=%CONDA_NPY%' - cmd: '%CMD_IN_ENV% conda build conda/cyordereddict --python=%CONDA_PY% --numpy=%CONDA_NPY%' - cmd: '%CMD_IN_ENV% conda build conda/logbook --python=%CONDA_PY% --numpy=%CONDA_NPY%' From a1ad7179206f2a5dba5c6f1045a374c6c4b07ab5 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 7 Mar 2016 11:27:46 -0500 Subject: [PATCH 32/51] TST: Clean up refs to adjustments db, so windows can remove its dir --- tests/pipeline/test_pipeline_algo.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/pipeline/test_pipeline_algo.py b/tests/pipeline/test_pipeline_algo.py index f0aea648..b4f59536 100644 --- a/tests/pipeline/test_pipeline_algo.py +++ b/tests/pipeline/test_pipeline_algo.py @@ -344,10 +344,10 @@ class PipelineAlgorithmTestCase(TestCase): cls.tempdir = tempdir = TempDirectory() tempdir.create() try: - cls.raw_data, cls.bar_reader = cls.create_bar_reader(tempdir) - cls.adj_reader = cls.create_adjustment_reader(tempdir) + cls.raw_data, bar_reader = cls.create_bar_reader(tempdir) + adj_reader = cls.create_adjustment_reader(tempdir) cls.pipeline_loader = USEquityPricingLoader( - cls.bar_reader, cls.adj_reader + bar_reader, adj_reader ) except: cls.tempdir.cleanup() @@ -358,6 +358,7 @@ class PipelineAlgorithmTestCase(TestCase): @classmethod def tearDownClass(cls): + del cls.pipeline_loader del cls.env cls.tempdir.cleanup() From b7f54dc1acb5c9a85bfd146988ca8dd36a19c9e8 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Fri, 11 Mar 2016 10:02:45 -0500 Subject: [PATCH 33/51] BLD: Don't build existing conda packages --- .travis.yml | 10 +++++----- appveyor.yml | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index fbc09158..4ed48f9e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,13 +39,13 @@ install: before_script: - pip freeze | sort script: - - conda build conda/bcolz -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN - - conda build conda/cyordereddict -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN - - conda build conda/logbook -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN - - conda build conda/ta-lib -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN + - conda build conda/bcolz -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --skip-existing -c quantopian + - conda build conda/cyordereddict -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --skip-existing -c quantopian + - conda build conda/logbook -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --skip-existing -c quantopian + - conda build conda/ta-lib -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --skip-existing -c quantopian # unshallow the clone so the conda build can clone it. - git fetch --unshallow - - conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN + - conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN -c quantopian - nosetests tests/ - flake8 zipline tests diff --git a/appveyor.yml b/appveyor.yml index 05ce7a6c..2aa411fc 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -102,11 +102,11 @@ install: # https://blog.ionelmc.ro/2014/12/21/compiling-python-extensions-on-windows/ for 64bit C compilation - ps: copy .\ci\appveyor\vcvars64.bat "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64" - - cmd: '%CMD_IN_ENV% conda build conda/bcolz --python=%CONDA_PY% --numpy=%CONDA_NPY%' - - cmd: '%CMD_IN_ENV% conda build conda/cyordereddict --python=%CONDA_PY% --numpy=%CONDA_NPY%' - - cmd: '%CMD_IN_ENV% conda build conda/logbook --python=%CONDA_PY% --numpy=%CONDA_NPY%' - - cmd: '%CMD_IN_ENV% conda build conda/ta-lib --python=%CONDA_PY% --numpy=%CONDA_NPY%' - - cmd: '%CMD_IN_ENV% conda build conda/zipline --python=%CONDA_PY% --numpy=%CONDA_NPY%' + - cmd: '%CMD_IN_ENV% conda build conda/bcolz --python=%CONDA_PY% --numpy=%CONDA_NPY% --skip-existing -c quantopian' + - cmd: '%CMD_IN_ENV% conda build conda/cyordereddict --python=%CONDA_PY% --numpy=%CONDA_NPY% --skip-existing -c quantopian' + - cmd: '%CMD_IN_ENV% conda build conda/logbook --python=%CONDA_PY% --numpy=%CONDA_NPY% --skip-existing -c quantopian' + - cmd: '%CMD_IN_ENV% conda build conda/ta-lib --python=%CONDA_PY% --numpy=%CONDA_NPY% --skip-existing -c quantopian' + - cmd: '%CMD_IN_ENV% conda build conda/zipline --python=%CONDA_PY% --numpy=%CONDA_NPY% -c quantopian' - cmd: conda create -n testenv --yes --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 bcolz=0.12.1 - cmd: activate testenv From 8aabf69abb894067a4183fda77bf5492f13612f8 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Fri, 11 Mar 2016 15:07:06 -0500 Subject: [PATCH 34/51] BLD: Upload packages to anaconda with "ci" label BLD: Install anaconda-client for upload --- .travis.yml | 37 +++++++++++++++++++++---------------- appveyor.yml | 2 +- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4ed48f9e..da611bd9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,16 @@ language: python sudo: false -matrix: - fast_finish: true - include: - - python: 2.7 - env: PANDAS_VERSION=0.16.1 NUMPY_VERSION=1.9.2 SCIPY_VERSION=0.15.1 - - python: 2.7 - env: PANDAS_VERSION=0.17.1 NUMPY_VERSION=1.10.4 SCIPY_VERSION=0.16.1 - - python: 3.4 - env: PANDAS_VERSION=0.16.1 NUMPY_VERSION=1.9.2 SCIPY_VERSION=0.15.1 - - python: 3.4 - env: PANDAS_VERSION=0.17.1 NUMPY_VERSION=1.10.4 SCIPY_VERSION=0.16.1 +fast_finish: true +python: + - 2.7 + - 3.4 +env: + global: + # ANACONDA_TOKEN + - secure: "HlTfqMSkU4yFTGSHvCg1FJ5gQaHA7SVWtH/h9uHHNArowBNqF0c9nJQcPNydi1szzGE+29ZiUwbj1AMI3CgpevZuowrxwMNlsEXol7uSxMJGczFEfauREEKfX0rEOzHcIRxlN6tC8AEBXnxBNesLbw9XadB+lJ/y7Fkfl28n5oU=" + matrix: + - PANDAS_VERSION=0.16.1 NUMPY_VERSION=1.9.2 SCIPY_VERSION=0.15.1 + - PANDAS_VERSION=0.17.1 NUMPY_VERSION=1.10.4 SCIPY_VERSION=0.16.1 cache: directories: - $HOME/.cache/.pip/ @@ -24,7 +24,7 @@ before_install: - sed -i "s/pandas==.*/pandas==$PANDAS_VERSION/" etc/requirements.txt - sed -i "s/scipy==.*/scipy==$SCIPY_VERSION/" etc/requirements.txt install: - - conda install conda-build=1.19.2 --yes + - conda install conda-build=1.19.2 anaconda-client=1.3.1 --yes - conda create -n testenv --yes -c quantopian pip python=$TRAVIS_PYTHON_VERSION numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION libgfortran=1.0 ta-lib=0.4.9 - source activate testenv @@ -39,13 +39,18 @@ install: before_script: - pip freeze | sort script: - - conda build conda/bcolz -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --skip-existing -c quantopian - - conda build conda/cyordereddict -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --skip-existing -c quantopian - - conda build conda/logbook -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --skip-existing -c quantopian - - conda build conda/ta-lib -q --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --skip-existing -c quantopian + - | + for recipe in $(ls -d conda/*/ | xargs -I {} basename {}); do + if [[ "$recipe" = "zipline" ]]; then continue; fi + + conda build conda/$recipe --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --skip-existing -c quantopian + RECIPE_OUTPUT=$(conda build conda/$recipe --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --output) + if [[ -f "$RECIPE_OUTPUT" && "$TRAVIS_SECURE_ENV_VARS" = "true" ]]; then anaconda -t $ANACONDA_TOKEN upload "$RECIPE_OUTPUT" -u quantopian --label ci; fi + done # unshallow the clone so the conda build can clone it. - git fetch --unshallow - conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN -c quantopian + - if [[ "$TRAVIS_SECURE_ENV_VARS" = "true" ]]; then anaconda -t $ANACONDA_TOKEN upload `conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --output` -u quantopian --label ci; fi - nosetests tests/ - flake8 zipline tests diff --git a/appveyor.yml b/appveyor.yml index 2aa411fc..bf2841a1 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -98,7 +98,7 @@ install: - sed -i "s/scipy==.*/scipy==%SCIPY_VERSION%/" etc/requirements.txt - conda info -a - - conda install conda-build=1.19.2 --yes + - conda install conda-build=1.19.2 anaconda-client=1.3.1 --yes # https://blog.ionelmc.ro/2014/12/21/compiling-python-extensions-on-windows/ for 64bit C compilation - ps: copy .\ci\appveyor\vcvars64.bat "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64" From e3af8219bca3ce0aad21944457942660fc0a3be1 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Sat, 12 Mar 2016 09:56:03 -0500 Subject: [PATCH 35/51] BLD: Run tests first. deactivate conda env so we can find anaconda --- .travis.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index da611bd9..9444e26d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,6 +39,10 @@ install: before_script: - pip freeze | sort script: + - nosetests tests/ + - flake8 zipline tests + # deactive env to get access to anaconda command + - source deactivate - | for recipe in $(ls -d conda/*/ | xargs -I {} basename {}); do if [[ "$recipe" = "zipline" ]]; then continue; fi @@ -51,9 +55,9 @@ script: - git fetch --unshallow - conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN -c quantopian - if [[ "$TRAVIS_SECURE_ENV_VARS" = "true" ]]; then anaconda -t $ANACONDA_TOKEN upload `conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --output` -u quantopian --label ci; fi + # reactivate env (necessary for coveralls) + - source activate testenv - - nosetests tests/ - - flake8 zipline tests after_success: - coveralls From d2b6009dc6aecd90d7d7edcd58a86a48d5e74ec0 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Sun, 13 Mar 2016 21:51:25 -0400 Subject: [PATCH 36/51] BLD: Find path to built conda package --- .travis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9444e26d..c09d46ec 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,8 +53,9 @@ script: done # unshallow the clone so the conda build can clone it. - git fetch --unshallow - - conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN -c quantopian - - if [[ "$TRAVIS_SECURE_ENV_VARS" = "true" ]]; then anaconda -t $ANACONDA_TOKEN upload `conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --output` -u quantopian --label ci; fi + - exec 3>&1; ZP_OUT=$(conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN -c quantopian | tee >(cat - >&3)) + - ZP_OUTPUT=$(echo "$ZP_OUT" | grep "anaconda upload" | awk '{print $NF}') + - if [[ "$TRAVIS_SECURE_ENV_VARS" = "true" ]]; then anaconda -t $ANACONDA_TOKEN upload $ZP_OUTPUT -u quantopian --label ci; fi # reactivate env (necessary for coveralls) - source activate testenv From 2ee66e4c1dcda63c53abaa810ee67a3152fd9a7b Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 14 Mar 2016 08:36:50 -0400 Subject: [PATCH 37/51] BLD: Bumped numexpr version and added conda recipe --- appveyor.yml | 3 ++- conda/numexpr/bld.bat | 8 ++++++++ conda/numexpr/build.sh | 9 +++++++++ conda/numexpr/meta.yaml | 22 ++++++++++++++++++++++ etc/requirements.txt | 2 +- 5 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 conda/numexpr/bld.bat create mode 100644 conda/numexpr/build.sh create mode 100644 conda/numexpr/meta.yaml diff --git a/appveyor.yml b/appveyor.yml index bf2841a1..ab217abe 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -105,10 +105,11 @@ install: - cmd: '%CMD_IN_ENV% conda build conda/bcolz --python=%CONDA_PY% --numpy=%CONDA_NPY% --skip-existing -c quantopian' - cmd: '%CMD_IN_ENV% conda build conda/cyordereddict --python=%CONDA_PY% --numpy=%CONDA_NPY% --skip-existing -c quantopian' - cmd: '%CMD_IN_ENV% conda build conda/logbook --python=%CONDA_PY% --numpy=%CONDA_NPY% --skip-existing -c quantopian' + - cmd: '%CMD_IN_ENV% conda build conda/numexpr --python=%CONDA_PY% --numpy=%CONDA_NPY% --skip-existing -c quantopian' - cmd: '%CMD_IN_ENV% conda build conda/ta-lib --python=%CONDA_PY% --numpy=%CONDA_NPY% --skip-existing -c quantopian' - cmd: '%CMD_IN_ENV% conda build conda/zipline --python=%CONDA_PY% --numpy=%CONDA_NPY% -c quantopian' - - cmd: conda create -n testenv --yes --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 bcolz=0.12.1 + - cmd: conda create -n testenv --yes --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 bcolz=0.12.1 numexpr=2.4.6 - cmd: activate testenv - cmd: pip install -r etc/requirements.txt - cmd: pip install -r etc/requirements_dev.txt diff --git a/conda/numexpr/bld.bat b/conda/numexpr/bld.bat new file mode 100644 index 00000000..87b1481d --- /dev/null +++ b/conda/numexpr/bld.bat @@ -0,0 +1,8 @@ +"%PYTHON%" setup.py install +if errorlevel 1 exit 1 + +:: Add more build steps here, if they are necessary. + +:: See +:: http://docs.continuum.io/conda/build.html +:: for a list of environment variables that are set during the build process. diff --git a/conda/numexpr/build.sh b/conda/numexpr/build.sh new file mode 100644 index 00000000..4d7fc032 --- /dev/null +++ b/conda/numexpr/build.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +$PYTHON setup.py install + +# Add more build steps here, if they are necessary. + +# See +# http://docs.continuum.io/conda/build.html +# for a list of environment variables that are set during the build process. diff --git a/conda/numexpr/meta.yaml b/conda/numexpr/meta.yaml new file mode 100644 index 00000000..f712ab3c --- /dev/null +++ b/conda/numexpr/meta.yaml @@ -0,0 +1,22 @@ +package: + name: numexpr + version: "2.4.6" + +source: + fn: numexpr-2.4.6.tar.gz + url: https://pypi.python.org/packages/source/n/numexpr/numexpr-2.4.6.tar.gz + md5: 17ac6fafc9ea1ce3eb970b9abccb4fbd + +requirements: + build: + - python + - numpy x.x + + run: + - python + - numpy x.x + +test: + # Python imports + imports: + - numexpr diff --git a/etc/requirements.txt b/etc/requirements.txt index cdfbb839..60342568 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -41,7 +41,7 @@ decorator==4.0.0 networkx==1.9.1 # NumericalExpression pipeline terms. -numexpr==2.4.3 +numexpr==2.4.6 # On disk storage format for pipeline data. bcolz==0.12.1 From dab85a81525761c3b9e5e2397325c487fde833d4 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 14 Mar 2016 10:52:37 -0400 Subject: [PATCH 38/51] BLD: Upload anaconda packages from appveyor Updated travis anaconda token --- .travis.yml | 4 ++-- appveyor.yml | 22 +++++++++++++++++----- 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index c09d46ec..559fe9a8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,8 @@ python: - 3.4 env: global: - # ANACONDA_TOKEN - - secure: "HlTfqMSkU4yFTGSHvCg1FJ5gQaHA7SVWtH/h9uHHNArowBNqF0c9nJQcPNydi1szzGE+29ZiUwbj1AMI3CgpevZuowrxwMNlsEXol7uSxMJGczFEfauREEKfX0rEOzHcIRxlN6tC8AEBXnxBNesLbw9XadB+lJ/y7Fkfl28n5oU=" + # ANACONDA_TOKEN with api:write + - secure: "RJJjjQloUjCSkhII93QM+YAsr6YYq7hPFvlbnT07ogn1NeUPsVCyJ97oiZfKtPgdbd24hdQP/CHfB0HgTTES8n996tN3QWc4hZj0e10kFyIlas9qnkrRYRR1jxGShBDXrLdx/tGh8z1qnnCm1fy+fDhAF7Zerouwy4EA2YEzxdE=" matrix: - PANDAS_VERSION=0.16.1 NUMPY_VERSION=1.9.2 SCIPY_VERSION=0.15.1 - PANDAS_VERSION=0.17.1 NUMPY_VERSION=1.10.4 SCIPY_VERSION=0.16.1 diff --git a/appveyor.yml b/appveyor.yml index ab217abe..043817e5 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -8,6 +8,10 @@ environment: # See: http://stackoverflow.com/a/13751649/163740 CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\ci\\appveyor\\run_with_env.cmd" + # with api:write + ANACONDA_TOKEN: + secure: "u3oPlANiKY5g1yMtcK+2MQfb4ViKGEap3TcBtvS7Y1InyeWs80Ko3/PsP7Lp6qwq" + matrix: - PYTHON_VERSION: "2.7" PYTHON_ARCH: "64" @@ -102,11 +106,19 @@ install: # https://blog.ionelmc.ro/2014/12/21/compiling-python-extensions-on-windows/ for 64bit C compilation - ps: copy .\ci\appveyor\vcvars64.bat "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64" - - cmd: '%CMD_IN_ENV% conda build conda/bcolz --python=%CONDA_PY% --numpy=%CONDA_NPY% --skip-existing -c quantopian' - - cmd: '%CMD_IN_ENV% conda build conda/cyordereddict --python=%CONDA_PY% --numpy=%CONDA_NPY% --skip-existing -c quantopian' - - cmd: '%CMD_IN_ENV% conda build conda/logbook --python=%CONDA_PY% --numpy=%CONDA_NPY% --skip-existing -c quantopian' - - cmd: '%CMD_IN_ENV% conda build conda/numexpr --python=%CONDA_PY% --numpy=%CONDA_NPY% --skip-existing -c quantopian' - - cmd: '%CMD_IN_ENV% conda build conda/ta-lib --python=%CONDA_PY% --numpy=%CONDA_NPY% --skip-existing -c quantopian' + - cmd: | + FOR /D %%r IN (conda/*) DO ( + IF NOT "%%r" == "zipline" ( + %CMD_IN_ENV% conda build "conda/%%r" --python=%CONDA_PY% --numpy=%CONDA_NPY% --skip-existing -c quantopian + + FOR /f "tokens=* USEBACKQ" %%F IN (`%CMD_IN_ENV% conda build "conda/%%r" --python=%CONDA_PY% --numpy=%CONDA_NPY% --output`) DO ( + SET RECIPE_OUTPUT=%%F + ) + IF EXIST "%RECIPE_OUTPUT%" ( + %CMD_IN_ENV% anaconda -t %ANACONDA_TOKEN% upload "%RECIPE_OUTPUT%" -u quantopian --label ci + ) + ) + ) - cmd: '%CMD_IN_ENV% conda build conda/zipline --python=%CONDA_PY% --numpy=%CONDA_NPY% -c quantopian' - cmd: conda create -n testenv --yes --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 bcolz=0.12.1 numexpr=2.4.6 From bf3307e768fd0d7a049cdeb2b9c2c33a28b38dc1 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 14 Mar 2016 14:11:15 -0400 Subject: [PATCH 39/51] BLD: Replaced batch script for building dependency packages with powershell. Easier to write. Check $LastExitCode when running non-powershell --- appveyor.yml | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 043817e5..5a656231 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -106,19 +106,17 @@ install: # https://blog.ionelmc.ro/2014/12/21/compiling-python-extensions-on-windows/ for 64bit C compilation - ps: copy .\ci\appveyor\vcvars64.bat "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64" - - cmd: | - FOR /D %%r IN (conda/*) DO ( - IF NOT "%%r" == "zipline" ( - %CMD_IN_ENV% conda build "conda/%%r" --python=%CONDA_PY% --numpy=%CONDA_NPY% --skip-existing -c quantopian - - FOR /f "tokens=* USEBACKQ" %%F IN (`%CMD_IN_ENV% conda build "conda/%%r" --python=%CONDA_PY% --numpy=%CONDA_NPY% --output`) DO ( - SET RECIPE_OUTPUT=%%F - ) - IF EXIST "%RECIPE_OUTPUT%" ( - %CMD_IN_ENV% anaconda -t %ANACONDA_TOKEN% upload "%RECIPE_OUTPUT%" -u quantopian --label ci - ) - ) - ) + - ps: >- + foreach ($recipe in dir .\conda -dir -name) { ` + if ($recipe -eq "zipline") { continue } ` + iex "$env:CMD_IN_ENV conda build conda/$recipe -q --python=$env:CONDA_PY --numpy=$env:CONDA_NPY --skip-existing -c quantopian"; ` + if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } ` + $RECIPE_OUTPUT=conda build conda/$recipe --python=$env:CONDA_PY --numpy=$env:CONDA_NPY --output; ` + if (Test-Path $RECIPE_OUTPUT) { ` + anaconda -t $env:ANACONDA_TOKEN upload "$RECIPE_OUTPUT" -u quantopian --label ci; ` + if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } ` + } ` + } - cmd: '%CMD_IN_ENV% conda build conda/zipline --python=%CONDA_PY% --numpy=%CONDA_NPY% -c quantopian' - cmd: conda create -n testenv --yes --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 bcolz=0.12.1 numexpr=2.4.6 From 8557047d1e68c99823aab497f1e4952c9fc0173b Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 15 Mar 2016 11:00:49 -0400 Subject: [PATCH 40/51] BLD: Add ci channel when deciding whether to build BLD: Include quantopian channels in testenv --- .travis.yml | 4 ++-- appveyor.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 559fe9a8..1b6a765a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,13 +47,13 @@ script: for recipe in $(ls -d conda/*/ | xargs -I {} basename {}); do if [[ "$recipe" = "zipline" ]]; then continue; fi - conda build conda/$recipe --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --skip-existing -c quantopian + conda build conda/$recipe --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --skip-existing -c quantopian -c https://conda.anaconda.org/quantopian/label/ci RECIPE_OUTPUT=$(conda build conda/$recipe --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --output) if [[ -f "$RECIPE_OUTPUT" && "$TRAVIS_SECURE_ENV_VARS" = "true" ]]; then anaconda -t $ANACONDA_TOKEN upload "$RECIPE_OUTPUT" -u quantopian --label ci; fi done # unshallow the clone so the conda build can clone it. - git fetch --unshallow - - exec 3>&1; ZP_OUT=$(conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN -c quantopian | tee >(cat - >&3)) + - exec 3>&1; ZP_OUT=$(conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN -c quantopian -c https://conda.anaconda.org/quantopian/label/ci | tee >(cat - >&3)) - ZP_OUTPUT=$(echo "$ZP_OUT" | grep "anaconda upload" | awk '{print $NF}') - if [[ "$TRAVIS_SECURE_ENV_VARS" = "true" ]]; then anaconda -t $ANACONDA_TOKEN upload $ZP_OUTPUT -u quantopian --label ci; fi # reactivate env (necessary for coveralls) diff --git a/appveyor.yml b/appveyor.yml index 5a656231..0a3347e6 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -109,7 +109,7 @@ install: - ps: >- foreach ($recipe in dir .\conda -dir -name) { ` if ($recipe -eq "zipline") { continue } ` - iex "$env:CMD_IN_ENV conda build conda/$recipe -q --python=$env:CONDA_PY --numpy=$env:CONDA_NPY --skip-existing -c quantopian"; ` + iex "$env:CMD_IN_ENV conda build conda/$recipe -q --python=$env:CONDA_PY --numpy=$env:CONDA_NPY --skip-existing -c quantopian -c https://conda.anaconda.org/quantopian/label/ci"; ` if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } ` $RECIPE_OUTPUT=conda build conda/$recipe --python=$env:CONDA_PY --numpy=$env:CONDA_NPY --output; ` if (Test-Path $RECIPE_OUTPUT) { ` @@ -117,9 +117,9 @@ install: if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } ` } ` } - - cmd: '%CMD_IN_ENV% conda build conda/zipline --python=%CONDA_PY% --numpy=%CONDA_NPY% -c quantopian' + - cmd: '%CMD_IN_ENV% conda build conda/zipline --python=%CONDA_PY% --numpy=%CONDA_NPY% -c quantopian -c https://conda.anaconda.org/quantopian/label/ci' - - cmd: conda create -n testenv --yes --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 bcolz=0.12.1 numexpr=2.4.6 + - cmd: conda create -n testenv --yes --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 bcolz=0.12.1 numexpr=2.4.6 -c quantopian -c https://conda.anaconda.org/quantopian/label/ci - cmd: activate testenv - cmd: pip install -r etc/requirements.txt - cmd: pip install -r etc/requirements_dev.txt From 3369e487ed1eb5b3c6c6d9457550cc1f5f42a45c Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 15 Mar 2016 11:38:53 -0400 Subject: [PATCH 41/51] BLD: Upload zipline package from appveyor Check for secure var existence Send anaconda upload stderr to stdout We're already looking for the exit code, and it's treating stderr as failure. Don't display progress bar on appveyor --- appveyor.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 0a3347e6..adceb4a3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -102,24 +102,30 @@ install: - sed -i "s/scipy==.*/scipy==%SCIPY_VERSION%/" etc/requirements.txt - conda info -a - - conda install conda-build=1.19.2 anaconda-client=1.3.1 --yes + - conda install conda-build=1.19.2 anaconda-client=1.3.1 --yes -q # https://blog.ionelmc.ro/2014/12/21/compiling-python-extensions-on-windows/ for 64bit C compilation - ps: copy .\ci\appveyor\vcvars64.bat "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64" - ps: >- foreach ($recipe in dir .\conda -dir -name) { ` if ($recipe -eq "zipline") { continue } ` - iex "$env:CMD_IN_ENV conda build conda/$recipe -q --python=$env:CONDA_PY --numpy=$env:CONDA_NPY --skip-existing -c quantopian -c https://conda.anaconda.org/quantopian/label/ci"; ` + iex "$env:CMD_IN_ENV conda build conda/$recipe -q --python=$env:CONDA_PY --numpy=$env:CONDA_NPY --skip-existing -c quantopian -c https://conda.anaconda.org/quantopian/label/ci" 2>&1; ` if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } ` $RECIPE_OUTPUT=conda build conda/$recipe --python=$env:CONDA_PY --numpy=$env:CONDA_NPY --output; ` - if (Test-Path $RECIPE_OUTPUT) { ` - anaconda -t $env:ANACONDA_TOKEN upload "$RECIPE_OUTPUT" -u quantopian --label ci; ` + if ((Test-Path $RECIPE_OUTPUT) -and ($env:ANACONDA_TOKEN)) { ` + iex "$env:CMD_IN_ENV anaconda -t $env:ANACONDA_TOKEN upload $RECIPE_OUTPUT -u quantopian --label ci --no-progress" 2>&1; ` if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } ` } ` } - - cmd: '%CMD_IN_ENV% conda build conda/zipline --python=%CONDA_PY% --numpy=%CONDA_NPY% -c quantopian -c https://conda.anaconda.org/quantopian/label/ci' + - ps: iex "$env:CMD_IN_ENV conda build conda/zipline -q --python=$env:CONDA_PY --numpy=$env:CONDA_NPY -c quantopian -c https://conda.anaconda.org/quantopian/label/ci" 2>&1 | tee -variable ZP_OUT + - ps: $ZP_OUTPUT=(($ZP_OUT | sls "anaconda upload") -split ' ')[-1] + - ps: >- + if ((Test-Path $ZP_OUTPUT) -and ($env:ANACONDA_TOKEN)) { ` + iex "$env:CMD_IN_ENV anaconda -t $env:ANACONDA_TOKEN upload $ZP_OUTPUT -u quantopian --label ci --no-progress" 2>&1; ` + if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } ` + } - - cmd: conda create -n testenv --yes --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 bcolz=0.12.1 numexpr=2.4.6 -c quantopian -c https://conda.anaconda.org/quantopian/label/ci + - cmd: conda create -n testenv --yes -q --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 bcolz=0.12.1 numexpr=2.4.6 -c quantopian -c https://conda.anaconda.org/quantopian/label/ci - cmd: activate testenv - cmd: pip install -r etc/requirements.txt - cmd: pip install -r etc/requirements_dev.txt From f3908319c45b9078fcee35658a347e234793f9a8 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 15 Mar 2016 18:23:59 -0400 Subject: [PATCH 42/51] BLD: Have pip use cache subdir --- appveyor.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index adceb4a3..6cdb7e3d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -127,11 +127,12 @@ install: - cmd: conda create -n testenv --yes -q --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 bcolz=0.12.1 numexpr=2.4.6 -c quantopian -c https://conda.anaconda.org/quantopian/label/ci - cmd: activate testenv - - cmd: pip install -r etc/requirements.txt - - cmd: pip install -r etc/requirements_dev.txt + - SET CACHE_DIR=%LOCALAPPDATA%\pip\Cache\pip_np%CONDA_NPY%py%CONDA_PY% + - cmd: pip install -r etc/requirements.txt --cache-dir=%CACHE_DIR% + - cmd: pip install -r etc/requirements_dev.txt --cache-dir=%CACHE_DIR% # this uses git requirements right now - - cmd: pip install -r etc/requirements_blaze.txt - - cmd: pip install -e .[talib] + - cmd: pip install -r etc/requirements_blaze.txt --cache-dir=%CACHE_DIR% + - cmd: pip install -e .[talib] --cache-dir=%CACHE_DIR% - cmd: pip freeze | sort test_script: From f272f1efb627a1d4b2a816456ed44857e7f355c2 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 15 Mar 2016 18:24:51 -0400 Subject: [PATCH 43/51] STY: Don't need to specify cmd --- appveyor.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 6cdb7e3d..3520f6df 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -125,15 +125,15 @@ install: if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } ` } - - cmd: conda create -n testenv --yes -q --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 bcolz=0.12.1 numexpr=2.4.6 -c quantopian -c https://conda.anaconda.org/quantopian/label/ci - - cmd: activate testenv + - conda create -n testenv --yes -q --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 bcolz=0.12.1 numexpr=2.4.6 -c quantopian -c https://conda.anaconda.org/quantopian/label/ci + - activate testenv - SET CACHE_DIR=%LOCALAPPDATA%\pip\Cache\pip_np%CONDA_NPY%py%CONDA_PY% - - cmd: pip install -r etc/requirements.txt --cache-dir=%CACHE_DIR% - - cmd: pip install -r etc/requirements_dev.txt --cache-dir=%CACHE_DIR% + - pip install -r etc/requirements.txt --cache-dir=%CACHE_DIR% + - pip install -r etc/requirements_dev.txt --cache-dir=%CACHE_DIR% # this uses git requirements right now - - cmd: pip install -r etc/requirements_blaze.txt --cache-dir=%CACHE_DIR% - - cmd: pip install -e .[talib] --cache-dir=%CACHE_DIR% - - cmd: pip freeze | sort + - pip install -r etc/requirements_blaze.txt --cache-dir=%CACHE_DIR% + - pip install -e .[talib] --cache-dir=%CACHE_DIR% + - pip freeze | sort test_script: - nosetests tests/ From c843a078ac0da4e34d068036e72132e75d6c93f7 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Wed, 16 Mar 2016 10:40:18 -0400 Subject: [PATCH 44/51] BLD: Test that we can conda install zipline on windows --- appveyor.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 3520f6df..5e171250 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -125,6 +125,9 @@ install: if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } ` } + # test that we can conda install zipline in a new env + - conda create -n installenv --yes -q --use-local python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% zipline -c quantopian -c https://conda.anaconda.org/quantopian/label/ci + - conda create -n testenv --yes -q --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 bcolz=0.12.1 numexpr=2.4.6 -c quantopian -c https://conda.anaconda.org/quantopian/label/ci - activate testenv - SET CACHE_DIR=%LOCALAPPDATA%\pip\Cache\pip_np%CONDA_NPY%py%CONDA_PY% From 7852d6900e2462d4ba1a212912e74db0ebe5f3e7 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 17 Mar 2016 21:19:16 -0400 Subject: [PATCH 45/51] BLD: Moving powershell to python --- .travis.yml | 12 ++++---- appveyor.yml | 20 +------------- ci/make_conda_packages.py | 58 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 24 deletions(-) create mode 100644 ci/make_conda_packages.py diff --git a/.travis.yml b/.travis.yml index 1b6a765a..9eb2b4fd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,8 +29,9 @@ install: - conda create -n testenv --yes -c quantopian pip python=$TRAVIS_PYTHON_VERSION numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION libgfortran=1.0 ta-lib=0.4.9 - source activate testenv - IFS='.' read -r -a NPY_VERSION_ARR <<< "$NUMPY_VERSION" - - NPY_VERSION_MAJ_MIN=${NPY_VERSION_ARR[0]}${NPY_VERSION_ARR[1]} - - CACHE_DIR="$HOME/.cache/.pip/pip_np""$NPY_VERSION_MAJ_MIN" + - CONDA_NPY=${NPY_VERSION_ARR[0]}${NPY_VERSION_ARR[1]} + - CONDA_PY=$TRAVIS_PYTHON_VERSION + - CACHE_DIR="$HOME/.cache/.pip/pip_np""CONDA_NPY" - pip install --upgrade pip coverage coveralls --cache-dir=$CACHE_DIR - pip install -r etc/requirements.txt --cache-dir=$CACHE_DIR - pip install -r etc/requirements_dev.txt --cache-dir=$CACHE_DIR @@ -47,13 +48,14 @@ script: for recipe in $(ls -d conda/*/ | xargs -I {} basename {}); do if [[ "$recipe" = "zipline" ]]; then continue; fi - conda build conda/$recipe --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --skip-existing -c quantopian -c https://conda.anaconda.org/quantopian/label/ci - RECIPE_OUTPUT=$(conda build conda/$recipe --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --output) + conda build conda/$recipe --python=$CONDA_PY --numpy=$CONDA_NPY --skip-existing -c quantopian -c https://conda.anaconda.org/quantopian/label/ci + RECIPE_OUTPUT=$(conda build conda/$recipe --python=$CONDA_PY --numpy=$CONDA_NPY --output) if [[ -f "$RECIPE_OUTPUT" && "$TRAVIS_SECURE_ENV_VARS" = "true" ]]; then anaconda -t $ANACONDA_TOKEN upload "$RECIPE_OUTPUT" -u quantopian --label ci; fi done + # unshallow the clone so the conda build can clone it. - git fetch --unshallow - - exec 3>&1; ZP_OUT=$(conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN -c quantopian -c https://conda.anaconda.org/quantopian/label/ci | tee >(cat - >&3)) + - exec 3>&1; ZP_OUT=$(conda build conda/zipline --python=$CONDA_PY --numpy=$CONDA_NPY -c quantopian -c https://conda.anaconda.org/quantopian/label/ci | tee >(cat - >&3)) - ZP_OUTPUT=$(echo "$ZP_OUT" | grep "anaconda upload" | awk '{print $NF}') - if [[ "$TRAVIS_SECURE_ENV_VARS" = "true" ]]; then anaconda -t $ANACONDA_TOKEN upload $ZP_OUTPUT -u quantopian --label ci; fi # reactivate env (necessary for coveralls) diff --git a/appveyor.yml b/appveyor.yml index 5e171250..c4a79ac7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -105,25 +105,7 @@ install: - conda install conda-build=1.19.2 anaconda-client=1.3.1 --yes -q # https://blog.ionelmc.ro/2014/12/21/compiling-python-extensions-on-windows/ for 64bit C compilation - ps: copy .\ci\appveyor\vcvars64.bat "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64" - - - ps: >- - foreach ($recipe in dir .\conda -dir -name) { ` - if ($recipe -eq "zipline") { continue } ` - iex "$env:CMD_IN_ENV conda build conda/$recipe -q --python=$env:CONDA_PY --numpy=$env:CONDA_NPY --skip-existing -c quantopian -c https://conda.anaconda.org/quantopian/label/ci" 2>&1; ` - if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } ` - $RECIPE_OUTPUT=conda build conda/$recipe --python=$env:CONDA_PY --numpy=$env:CONDA_NPY --output; ` - if ((Test-Path $RECIPE_OUTPUT) -and ($env:ANACONDA_TOKEN)) { ` - iex "$env:CMD_IN_ENV anaconda -t $env:ANACONDA_TOKEN upload $RECIPE_OUTPUT -u quantopian --label ci --no-progress" 2>&1; ` - if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } ` - } ` - } - - ps: iex "$env:CMD_IN_ENV conda build conda/zipline -q --python=$env:CONDA_PY --numpy=$env:CONDA_NPY -c quantopian -c https://conda.anaconda.org/quantopian/label/ci" 2>&1 | tee -variable ZP_OUT - - ps: $ZP_OUTPUT=(($ZP_OUT | sls "anaconda upload") -split ' ')[-1] - - ps: >- - if ((Test-Path $ZP_OUTPUT) -and ($env:ANACONDA_TOKEN)) { ` - iex "$env:CMD_IN_ENV anaconda -t $env:ANACONDA_TOKEN upload $ZP_OUTPUT -u quantopian --label ci --no-progress" 2>&1; ` - if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } ` - } + - "%CMD_IN_ENV% python .\\ci\\make_conda_packages.py" # test that we can conda install zipline in a new env - conda create -n installenv --yes -q --use-local python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% zipline -c quantopian -c https://conda.anaconda.org/quantopian/label/ci diff --git a/ci/make_conda_packages.py b/ci/make_conda_packages.py new file mode 100644 index 00000000..162ef1e7 --- /dev/null +++ b/ci/make_conda_packages.py @@ -0,0 +1,58 @@ +import os +import re +import subprocess + + +def get_immediate_subdirectories(a_dir): + return [name for name in os.listdir(a_dir) + if os.path.isdir(os.path.join(a_dir, name))] + + +def iter_stdout(cmd): + p = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + + try: + for line in iter(p.stdout.readline, b''): + yield line.decode().rstrip() + finally: + retcode = p.wait() + if retcode: + raise subprocess.CalledProcessError(retcode, cmd[0]) + + +PKG_PATH_PATTERN = re.compile(".* anaconda upload (?P.+)$") + + +def main(): + for recipe in get_immediate_subdirectories('conda'): + cmd = ["conda", "build", os.path.join('conda', recipe), + "--python", os.environ['CONDA_PY'], + "--numpy", os.environ['CONDA_NPY'], + "--skip-existing", + "-c", "quantopian", + "-c", "https://conda.anaconda.org/quantopian/label/ci"] + + output = None + + for line in iter_stdout(cmd): + print(line) + + if not output: + match = PKG_PATH_PATTERN.match(line) + if match: + output = match.group('pkg_path') + + if (output and os.path.exists(output) and + os.environ.get('ANACONDA_TOKEN')): + + cmd = ["anaconda", "-t", os.environ['ANACONDA_TOKEN'], + "upload", output, "-u", "quantopian", "--label", "ci"] + + for line in iter_stdout(cmd): + print(line) + + +if __name__ == '__main__': + main() From 42c1ef86b566363e1ab31a6dcab236d6fe3132b2 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 22 Mar 2016 19:25:18 -0400 Subject: [PATCH 46/51] DEV: Move TA-Lib reqs to own file --- etc/requirements_talib.txt | 1 + setup.py | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 etc/requirements_talib.txt diff --git a/etc/requirements_talib.txt b/etc/requirements_talib.txt new file mode 100644 index 00000000..478a392b --- /dev/null +++ b/etc/requirements_talib.txt @@ -0,0 +1 @@ +TA-Lib==0.4.9 diff --git a/setup.py b/setup.py index 9844a0ad..07ccb62f 100644 --- a/setup.py +++ b/setup.py @@ -195,15 +195,15 @@ def install_requires(strict_bounds=False, conda_format=False): def extras_requires(conda_format=False): - dev_reqs = read_requirements('etc/requirements_dev.txt', + extras = { + extra: read_requirements('etc/requirements_{0}.txt'.format(extra), strict_bounds=True, conda_format=conda_format) - talib_reqs = ['TA-Lib==0.4.9'] - return { - 'dev': dev_reqs, - 'talib': talib_reqs, - 'all': dev_reqs + talib_reqs, + for extra in ('dev', 'talib') } + extras['all'] = [req for reqs in extras.values() for req in reqs] + + return extras def module_requirements(requirements_path, module_names, strict_bounds, From 3315a0ffe7190f9c09d472b313e72d9d825ac29a Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 22 Mar 2016 19:27:05 -0400 Subject: [PATCH 47/51] BLD: Use TA-Lib version from reqs so we don't have to maintain the version in multiple places --- .travis.yml | 6 ++++-- appveyor.yml | 8 ++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9eb2b4fd..96db8756 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,7 +26,8 @@ before_install: install: - conda install conda-build=1.19.2 anaconda-client=1.3.1 --yes - - conda create -n testenv --yes -c quantopian pip python=$TRAVIS_PYTHON_VERSION numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION libgfortran=1.0 ta-lib=0.4.9 + - TALIB_VERSION=$(cat ./etc/requirements_talib.txt | sed "s/TA-Lib==\(.*\)/\1/") + - conda create -n testenv --yes -c quantopian pip python=$TRAVIS_PYTHON_VERSION numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION libgfortran=1.0 ta-lib=$TALIB_VERSION - source activate testenv - IFS='.' read -r -a NPY_VERSION_ARR <<< "$NUMPY_VERSION" - CONDA_NPY=${NPY_VERSION_ARR[0]}${NPY_VERSION_ARR[1]} @@ -36,7 +37,8 @@ install: - pip install -r etc/requirements.txt --cache-dir=$CACHE_DIR - pip install -r etc/requirements_dev.txt --cache-dir=$CACHE_DIR - pip install -r etc/requirements_blaze.txt --cache-dir=$CACHE_DIR # this uses git requirements right now - - pip install -e .[talib] --cache-dir=$CACHE_DIR + - pip install -r etc/requirements_talib.txt --cache-dir=$CACHE_DIR + - pip install -e .[all] --cache-dir=$CACHE_DIR before_script: - pip freeze | sort script: diff --git a/appveyor.yml b/appveyor.yml index c4a79ac7..725b27a8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -110,14 +110,18 @@ install: # test that we can conda install zipline in a new env - conda create -n installenv --yes -q --use-local python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% zipline -c quantopian -c https://conda.anaconda.org/quantopian/label/ci - - conda create -n testenv --yes -q --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=0.4.9 bcolz=0.12.1 numexpr=2.4.6 -c quantopian -c https://conda.anaconda.org/quantopian/label/ci + - ps: $env:BCOLZ_VERSION=(sls "bcolz==(.*)" .\etc\requirements.txt -ca).matches.groups[1].value + - ps: $env:NUMEXPR_VERSION=(sls "numexpr==(.*)" .\etc\requirements.txt -ca).matches.groups[1].value + - ps: $env:TALIB_VERSION=(sls "TA-Lib==(.*)" .\etc\requirements_talib.txt -ca).matches.groups[1].value + - conda create -n testenv --yes -q --use-local pip python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% scipy=%SCIPY_VERSION% ta-lib=%TALIB_VERSION% bcolz=%BCOLZ_VERSION% numexpr=%NUMEXPR_VERSION% -c quantopian -c https://conda.anaconda.org/quantopian/label/ci - activate testenv - SET CACHE_DIR=%LOCALAPPDATA%\pip\Cache\pip_np%CONDA_NPY%py%CONDA_PY% - pip install -r etc/requirements.txt --cache-dir=%CACHE_DIR% - pip install -r etc/requirements_dev.txt --cache-dir=%CACHE_DIR% # this uses git requirements right now - pip install -r etc/requirements_blaze.txt --cache-dir=%CACHE_DIR% - - pip install -e .[talib] --cache-dir=%CACHE_DIR% + - pip install -r etc/requirements_talib.txt --cache-dir=%CACHE_DIR% + - pip install -e .[all] --cache-dir=%CACHE_DIR% - pip freeze | sort test_script: From 6d66fb46ef21b42bf113c45d2ca3f451f160ceb5 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Wed, 23 Mar 2016 09:46:57 -0400 Subject: [PATCH 48/51] DOC: Added appveyor status of master to README --- README.rst | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 81729acc..23ee09c7 100644 --- a/README.rst +++ b/README.rst @@ -4,7 +4,8 @@ Zipline |Gitter| |version status| |downloads| -|build status| +|travis status| +|appveyor status| |Coverage Status| Zipline is a Pythonic algorithmic trading library. It is an event-driven @@ -179,8 +180,10 @@ https://github.com/quantopian/zipline/wiki/Contribution-Requests :target: https://pypi.python.org/pypi/zipline .. |downloads| image:: https://img.shields.io/pypi/dd/zipline.svg :target: https://pypi.python.org/pypi/zipline -.. |build status| image:: https://travis-ci.org/quantopian/zipline.png?branch=master +.. |travis status| image:: https://travis-ci.org/quantopian/zipline.png?branch=master :target: https://travis-ci.org/quantopian/zipline +.. |appveyor status| image:: https://ci.appveyor.com/api/projects/status/github/quantopian/zipline?branch=master&svg=true + :target: https://ci.appveyor.com/project/quantopian/zipline/branch/master .. |Coverage Status| image:: https://coveralls.io/repos/quantopian/zipline/badge.png :target: https://coveralls.io/r/quantopian/zipline From 70cbee0468d1c691f0a20958568fb95e20ec9614 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Wed, 23 Mar 2016 10:14:07 -0400 Subject: [PATCH 49/51] BLD: Only upload to anaconda if on master --- .travis.yml | 5 +++-- ci/make_conda_packages.py | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.travis.yml b/.travis.yml index 96db8756..70c3bc58 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,20 +46,21 @@ script: - flake8 zipline tests # deactive env to get access to anaconda command - source deactivate + - if [[ "$TRAVIS_SECURE_ENV_VARS" = "true" && "$TRAVIS_BRANCH" = "master" && "$TRAVIS_PULL_REQUEST" = "false" ]]; then DO_UPLOAD="true"; else DO_UPLOAD="false"; fi - | for recipe in $(ls -d conda/*/ | xargs -I {} basename {}); do if [[ "$recipe" = "zipline" ]]; then continue; fi conda build conda/$recipe --python=$CONDA_PY --numpy=$CONDA_NPY --skip-existing -c quantopian -c https://conda.anaconda.org/quantopian/label/ci RECIPE_OUTPUT=$(conda build conda/$recipe --python=$CONDA_PY --numpy=$CONDA_NPY --output) - if [[ -f "$RECIPE_OUTPUT" && "$TRAVIS_SECURE_ENV_VARS" = "true" ]]; then anaconda -t $ANACONDA_TOKEN upload "$RECIPE_OUTPUT" -u quantopian --label ci; fi + if [[ -f "$RECIPE_OUTPUT" && "$DO_UPLOAD" = "true" ]]; then anaconda -t $ANACONDA_TOKEN upload "$RECIPE_OUTPUT" -u quantopian --label ci; fi done # unshallow the clone so the conda build can clone it. - git fetch --unshallow - exec 3>&1; ZP_OUT=$(conda build conda/zipline --python=$CONDA_PY --numpy=$CONDA_NPY -c quantopian -c https://conda.anaconda.org/quantopian/label/ci | tee >(cat - >&3)) - ZP_OUTPUT=$(echo "$ZP_OUT" | grep "anaconda upload" | awk '{print $NF}') - - if [[ "$TRAVIS_SECURE_ENV_VARS" = "true" ]]; then anaconda -t $ANACONDA_TOKEN upload $ZP_OUTPUT -u quantopian --label ci; fi + - if [[ "$DO_UPLOAD" = "true" ]]; then anaconda -t $ANACONDA_TOKEN upload $ZP_OUTPUT -u quantopian --label ci; fi # reactivate env (necessary for coveralls) - source activate testenv diff --git a/ci/make_conda_packages.py b/ci/make_conda_packages.py index 162ef1e7..240e2606 100644 --- a/ci/make_conda_packages.py +++ b/ci/make_conda_packages.py @@ -25,11 +25,11 @@ def iter_stdout(cmd): PKG_PATH_PATTERN = re.compile(".* anaconda upload (?P.+)$") -def main(): +def main(env, do_upload): for recipe in get_immediate_subdirectories('conda'): cmd = ["conda", "build", os.path.join('conda', recipe), - "--python", os.environ['CONDA_PY'], - "--numpy", os.environ['CONDA_NPY'], + "--python", env['CONDA_PY'], + "--numpy", env['CONDA_NPY'], "--skip-existing", "-c", "quantopian", "-c", "https://conda.anaconda.org/quantopian/label/ci"] @@ -44,10 +44,8 @@ def main(): if match: output = match.group('pkg_path') - if (output and os.path.exists(output) and - os.environ.get('ANACONDA_TOKEN')): - - cmd = ["anaconda", "-t", os.environ['ANACONDA_TOKEN'], + if output and os.path.exists(output) and do_upload: + cmd = ["anaconda", "-t", env['ANACONDA_TOKEN'], "upload", output, "-u", "quantopian", "--label", "ci"] for line in iter_stdout(cmd): @@ -55,4 +53,8 @@ def main(): if __name__ == '__main__': - main() + env = os.environ.copy() + main(env, + do_upload=(env.get('ANACONDA_TOKEN') and + env.get('APPVEYOR_REPO_BRANCH') == 'master') and + 'APPVEYOR_PULL_REQUEST_NUMBER' not in env) From a9fcb244ee864550b57ef0e3cd9b48936e1ccd1e Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Wed, 23 Mar 2016 17:12:00 -0400 Subject: [PATCH 50/51] BLD: Removing 32-bit builds because of appveyor's slow speed --- appveyor.yml | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 725b27a8..43640664 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -19,48 +19,24 @@ environment: NUMPY_VERSION: "1.9.2" SCIPY_VERSION: "0.15.1" - - PYTHON_VERSION: "2.7" - PYTHON_ARCH: "32" - PANDAS_VERSION: "0.16.1" - NUMPY_VERSION: "1.9.2" - SCIPY_VERSION: "0.15.1" - - PYTHON_VERSION: "3.4" PYTHON_ARCH: "64" PANDAS_VERSION: "0.16.1" NUMPY_VERSION: "1.9.2" SCIPY_VERSION: "0.15.1" - - PYTHON_VERSION: "3.4" - PYTHON_ARCH: "32" - PANDAS_VERSION: "0.16.1" - NUMPY_VERSION: "1.9.2" - SCIPY_VERSION: "0.15.1" - - PYTHON_VERSION: "2.7" PYTHON_ARCH: "64" PANDAS_VERSION: "0.17.1" NUMPY_VERSION: "1.10.4" SCIPY_VERSION: "0.16.1" - - PYTHON_VERSION: "2.7" - PYTHON_ARCH: "32" - PANDAS_VERSION: "0.17.1" - NUMPY_VERSION: "1.10.4" - SCIPY_VERSION: "0.16.1" - - PYTHON_VERSION: "3.4" PYTHON_ARCH: "64" PANDAS_VERSION: "0.17.1" NUMPY_VERSION: "1.10.4" SCIPY_VERSION: "0.16.1" - - PYTHON_VERSION: "3.4" - PYTHON_ARCH: "32" - PANDAS_VERSION: "0.17.1" - NUMPY_VERSION: "1.10.4" - SCIPY_VERSION: "0.16.1" - # We always use a 64-bit machine, but can build x86 distributions # with the PYTHON_ARCH variable (which is used by CMD_IN_ENV). platform: From 98289e425785b70e3e3d359d2f0aa8964b587fe1 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Wed, 23 Mar 2016 17:32:21 -0400 Subject: [PATCH 51/51] BLD: Only build master on appveyor (and hopefully PRs too) --- appveyor.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/appveyor.yml b/appveyor.yml index 43640664..3bad42b3 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -103,3 +103,7 @@ install: test_script: - nosetests tests/ - flake8 zipline tests + +branches: + only: + - master