BLD: Moving powershell to python

.travis.yml

@@ -29,8 +29,9 @@ install:
   - conda create -n testenv --yes -c quantopian pip python=$TRAVIS_PYTHON_VERSION numpy=$NUMPY_VERSION scipy=$SCIPY_VERSION libgfortran=1.0 ta-lib=0.4.9
   - source activate testenv
   - IFS='.' read -r -a NPY_VERSION_ARR <<< "$NUMPY_VERSION"
-  - NPY_VERSION_MAJ_MIN=${NPY_VERSION_ARR[0]}${NPY_VERSION_ARR[1]}
-  - CACHE_DIR="$HOME/.cache/.pip/pip_np""$NPY_VERSION_MAJ_MIN"
+  - CONDA_NPY=${NPY_VERSION_ARR[0]}${NPY_VERSION_ARR[1]}
+  - CONDA_PY=$TRAVIS_PYTHON_VERSION
+  - CACHE_DIR="$HOME/.cache/.pip/pip_np""CONDA_NPY"
   - pip install --upgrade pip coverage coveralls --cache-dir=$CACHE_DIR
   - pip install -r etc/requirements.txt --cache-dir=$CACHE_DIR
   - pip install -r etc/requirements_dev.txt --cache-dir=$CACHE_DIR
@@ -47,13 +48,14 @@ script:
     for recipe in $(ls -d conda/*/ | xargs -I {} basename {}); do
       if [[ "$recipe" = "zipline" ]]; then continue; fi
 
-      conda build conda/$recipe --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --skip-existing -c quantopian -c https://conda.anaconda.org/quantopian/label/ci
-      RECIPE_OUTPUT=$(conda build conda/$recipe --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN --output)
+      conda build conda/$recipe --python=$CONDA_PY --numpy=$CONDA_NPY --skip-existing -c quantopian -c https://conda.anaconda.org/quantopian/label/ci
+      RECIPE_OUTPUT=$(conda build conda/$recipe --python=$CONDA_PY --numpy=$CONDA_NPY --output)
       if [[ -f "$RECIPE_OUTPUT" && "$TRAVIS_SECURE_ENV_VARS" = "true" ]]; then anaconda -t $ANACONDA_TOKEN upload "$RECIPE_OUTPUT" -u quantopian --label ci; fi
     done
+
   # unshallow the clone so the conda build can clone it.
   - git fetch --unshallow
-  - exec 3>&1; ZP_OUT=$(conda build conda/zipline --python=$TRAVIS_PYTHON_VERSION --numpy=$NPY_VERSION_MAJ_MIN -c quantopian -c https://conda.anaconda.org/quantopian/label/ci | tee >(cat - >&3))
+  - exec 3>&1; ZP_OUT=$(conda build conda/zipline --python=$CONDA_PY --numpy=$CONDA_NPY -c quantopian -c https://conda.anaconda.org/quantopian/label/ci | tee >(cat - >&3))
   - ZP_OUTPUT=$(echo "$ZP_OUT" | grep "anaconda upload" | awk '{print $NF}')
   - if [[ "$TRAVIS_SECURE_ENV_VARS" = "true" ]]; then anaconda -t $ANACONDA_TOKEN upload $ZP_OUTPUT -u quantopian --label ci; fi
   # reactivate env (necessary for coveralls)

appveyor.yml

@@ -105,25 +105,7 @@ install:
   - conda install conda-build=1.19.2 anaconda-client=1.3.1 --yes -q
   # https://blog.ionelmc.ro/2014/12/21/compiling-python-extensions-on-windows/ for 64bit C compilation
   - ps: copy .\ci\appveyor\vcvars64.bat "C:\Program Files (x86)\Microsoft Visual Studio 10.0\VC\bin\amd64"
-
-  - ps: >-
-      foreach ($recipe in dir .\conda -dir -name) { `
-        if ($recipe -eq "zipline") { continue } `
-        iex "$env:CMD_IN_ENV conda build conda/$recipe -q --python=$env:CONDA_PY --numpy=$env:CONDA_NPY --skip-existing -c quantopian -c https://conda.anaconda.org/quantopian/label/ci" 2>&1; `
-        if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } `
-        $RECIPE_OUTPUT=conda build conda/$recipe --python=$env:CONDA_PY --numpy=$env:CONDA_NPY --output; `
-        if ((Test-Path $RECIPE_OUTPUT) -and ($env:ANACONDA_TOKEN)) { `
-          iex "$env:CMD_IN_ENV anaconda -t $env:ANACONDA_TOKEN upload $RECIPE_OUTPUT -u quantopian --label ci --no-progress" 2>&1; `
-          if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } `
-        } `
-      }
-  - ps: iex "$env:CMD_IN_ENV conda build conda/zipline -q --python=$env:CONDA_PY --numpy=$env:CONDA_NPY -c quantopian -c https://conda.anaconda.org/quantopian/label/ci" 2>&1 | tee -variable ZP_OUT
-  - ps: $ZP_OUTPUT=(($ZP_OUT | sls "anaconda upload") -split ' ')[-1]
-  - ps: >-
-      if ((Test-Path $ZP_OUTPUT) -and ($env:ANACONDA_TOKEN)) { `
-        iex "$env:CMD_IN_ENV anaconda -t $env:ANACONDA_TOKEN upload $ZP_OUTPUT -u quantopian --label ci --no-progress" 2>&1; `
-        if ($LastExitCode -ne 0) { throw "Command failed with exit code $LastExitCode." } `
-      }
+  - "%CMD_IN_ENV% python .\\ci\\make_conda_packages.py"
 
   # test that we can conda install zipline in a new env
   - conda create -n installenv --yes -q --use-local python=%PYTHON_VERSION% numpy=%NUMPY_VERSION% zipline -c quantopian -c https://conda.anaconda.org/quantopian/label/ci

ci/make_conda_packages.py

@@ -0,0 +1,58 @@
+import os
+import re
+import subprocess
+
+
+def get_immediate_subdirectories(a_dir):
+    return [name for name in os.listdir(a_dir)
+            if os.path.isdir(os.path.join(a_dir, name))]
+
+
+def iter_stdout(cmd):
+    p = subprocess.Popen(cmd,
+                         stdout=subprocess.PIPE,
+                         stderr=subprocess.STDOUT)
+
+    try:
+        for line in iter(p.stdout.readline, b''):
+            yield line.decode().rstrip()
+    finally:
+        retcode = p.wait()
+        if retcode:
+            raise subprocess.CalledProcessError(retcode, cmd[0])
+
+
+PKG_PATH_PATTERN = re.compile(".* anaconda upload (?P<pkg_path>.+)$")
+
+
+def main():
+    for recipe in get_immediate_subdirectories('conda'):
+        cmd = ["conda", "build", os.path.join('conda', recipe),
+               "--python", os.environ['CONDA_PY'],
+               "--numpy", os.environ['CONDA_NPY'],
+               "--skip-existing",
+               "-c", "quantopian",
+               "-c", "https://conda.anaconda.org/quantopian/label/ci"]
+
+        output = None
+
+        for line in iter_stdout(cmd):
+            print(line)
+
+            if not output:
+                match = PKG_PATH_PATTERN.match(line)
+                if match:
+                    output = match.group('pkg_path')
+
+        if (output and os.path.exists(output) and
+                os.environ.get('ANACONDA_TOKEN')):
+
+            cmd = ["anaconda", "-t", os.environ['ANACONDA_TOKEN'],
+                   "upload", output, "-u", "quantopian", "--label", "ci"]
+
+            for line in iter_stdout(cmd):
+                print(line)
+
+
+if __name__ == '__main__':
+    main()