From 9896c3cdb6568b0150296ea997fc2b326dec89d3 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Wed, 30 Dec 2015 21:10:16 -0500 Subject: [PATCH 1/6] BLD: Build zipline conda package from source --- conda/zipline/meta.yaml | 33 ++++++++++++++++++++------------ setup.py | 42 ++++++++++++++++++++++++++++------------- 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/conda/zipline/meta.yaml b/conda/zipline/meta.yaml index e95aaa7f..030aba91 100644 --- a/conda/zipline/meta.yaml +++ b/conda/zipline/meta.yaml @@ -1,28 +1,37 @@ +{% set data = load_setuptools() %} + package: name: zipline - version: 0.8.3 + version: {{ environ.get('GIT_DESCRIBE_TAG', '')}} + +build: + number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }} + {% if environ.get('GIT_DESCRIBE_NUMBER', '0') == '0' %}string: py{{ environ.get('PY_VER').replace('.', '') }}_0 + {% else %}string: py{{ environ.get('PY_VER').replace('.', '') }}_{{ environ.get('GIT_BUILD_STR', 'GIT_STUB') }}{% endif %} + source: - fn: zipline-0.8.3.tar.gz - url: https://pypi.python.org/packages/source/z/zipline/zipline-0.8.3.tar.gz - md5: 042ffcee614d2279add9a1bfd27a33cf + git_url: ../../ requirements: build: - python - setuptools - - cython + - cython ==0.22.1 + - numpy ==1.9.2 run: - python - - pytz - - requests - - numpy - - pandas - - scipy - - matplotlib - - logbook + {% for req in data.get('install_requires', []) -%} + - {{req}} + {% endfor %} test: + {# When we include the tests module in the zipline package, we can use this: + requires: + {% for req in data.get('extras_require', {}).get('dev', []) -%} + - {{req}} + {% endfor %} + #} # Python imports imports: - zipline diff --git a/setup.py b/setup.py index 2699e403..2150a0c6 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. from __future__ import print_function - +import os import re import sys from operator import lt, gt, eq, le, ge @@ -133,7 +133,16 @@ def _with_bounds(req): return ''.join(with_bounds) -def read_requirements(path, strict_bounds): +def _conda_format(req): + return re.sub( + '([^=<>]+)([=<>]{1,2})', + lambda m: '%s %s' % (m.group(1).lower(), m.group(2)), + req, + 1, + ) + + +def read_requirements(path, strict_bounds, conda_format=False): """ Read a requirements.txt file, expressed as a path relative to Zipline root. @@ -144,20 +153,25 @@ def read_requirements(path, strict_bounds): with open(real_path) as f: reqs = _filter_requirements(f.readlines()) - if strict_bounds: - return list(reqs) - else: - return list(map(_with_bounds, reqs)) + if not strict_bounds: + reqs = map(_with_bounds, reqs) + + if conda_format: + reqs = map(_conda_format, reqs) + + return list(reqs) -def install_requires(strict_bounds=False): +def install_requires(strict_bounds=False, conda_format=False): return read_requirements('etc/requirements.txt', - strict_bounds=strict_bounds) + strict_bounds=strict_bounds, + conda_format=conda_format) -def extras_requires(): +def extras_requires(conda_format=False): dev_reqs = read_requirements('etc/requirements_dev.txt', - strict_bounds=True) + strict_bounds=True, + conda_format=conda_format) talib_reqs = ['TA-Lib==0.4.9'] return { 'dev': dev_reqs, @@ -171,7 +185,7 @@ def module_requirements(requirements_path, module_names): found = set() module_lines = [] parser = re.compile("([^=<>]+)([<=>]{1,2})(.*)") - for line in read_requirements(requirements_path, strict_bounds=False): + for line in read_requirements(requirements_path, strict_bounds=True): match = parser.match(line) if match is None: raise AssertionError("Could not parse requirement: '%s'" % line) @@ -214,6 +228,7 @@ def pre_setup(): pre_setup() +conda_build = os.path.basename(sys.argv[0]) == 'conda-build' setup( name='zipline', @@ -241,7 +256,8 @@ setup( 'Topic :: Scientific/Engineering :: Information Analysis', 'Topic :: System :: Distributed Computing', ], - install_requires=install_requires(), - extras_require=extras_requires(), + install_requires=install_requires(strict_bounds=conda_build, + conda_format=conda_build), + extras_require=extras_requires(conda_format=conda_build), url="http://zipline.io", ) From 4dd04b5bb6547e1ccec4e41ba556144eabe1efdb Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 31 Dec 2015 11:56:07 -0500 Subject: [PATCH 2/6] BLD: Updated dependency recipes --- conda/README.md | 3 +- conda/cyordereddict/bld.bat | 8 +++++ conda/cyordereddict/build.sh | 9 ++++++ conda/cyordereddict/meta.yaml | 61 +++++++++++++++++++++++++++++++++++ conda/logbook/meta.yaml | 18 +++++++---- 5 files changed, 91 insertions(+), 8 deletions(-) create mode 100644 conda/cyordereddict/bld.bat create mode 100644 conda/cyordereddict/build.sh create mode 100644 conda/cyordereddict/meta.yaml diff --git a/conda/README.md b/conda/README.md index 4b142e2b..133d12ad 100644 --- a/conda/README.md +++ b/conda/README.md @@ -12,6 +12,7 @@ conda-build you should be able to: ``` conda build ta-lib conda build logbook +conda build cyordereddict conda build zipline ``` @@ -23,4 +24,4 @@ Windows Building ta-lib on Windows requires Visual Studio (Express) and the [compiled ta-lib](ta-lib-0.4.0-msvc.zip) which you have to -unzip to C:\ta-lib. \ No newline at end of file +unzip to C:\ta-lib. diff --git a/conda/cyordereddict/bld.bat b/conda/cyordereddict/bld.bat new file mode 100644 index 00000000..87b1481d --- /dev/null +++ b/conda/cyordereddict/bld.bat @@ -0,0 +1,8 @@ +"%PYTHON%" setup.py install +if errorlevel 1 exit 1 + +:: Add more build steps here, if they are necessary. + +:: See +:: http://docs.continuum.io/conda/build.html +:: for a list of environment variables that are set during the build process. diff --git a/conda/cyordereddict/build.sh b/conda/cyordereddict/build.sh new file mode 100644 index 00000000..4d7fc032 --- /dev/null +++ b/conda/cyordereddict/build.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +$PYTHON setup.py install + +# Add more build steps here, if they are necessary. + +# See +# http://docs.continuum.io/conda/build.html +# for a list of environment variables that are set during the build process. diff --git a/conda/cyordereddict/meta.yaml b/conda/cyordereddict/meta.yaml new file mode 100644 index 00000000..24996015 --- /dev/null +++ b/conda/cyordereddict/meta.yaml @@ -0,0 +1,61 @@ +package: + name: cyordereddict + version: "0.2.2" + +source: + fn: cyordereddict-0.2.2.tar.gz + url: https://pypi.python.org/packages/source/c/cyordereddict/cyordereddict-0.2.2.tar.gz + md5: 6279eb0bf9819f0293ad5315b2d484d0 +# patches: + # List any patch files here + # - fix.patch + +# build: + # noarch_python: True + # preserve_egg_dir: True + # entry_points: + # Put any entry points (scripts to be generated automatically) here. The + # syntax is module:function. For example + # + # - cyordereddict = cyordereddict:main + # + # Would create an entry point called cyordereddict that calls cyordereddict.main() + + + # If this is a new build for the same version, increment the build + # number. If you do not include this key, it defaults to 0. + # number: 1 + +requirements: + build: + - python + + run: + - python + +test: + # Python imports + imports: + - cyordereddict + - cyordereddict.benchmark + + # commands: + # You can put test commands to be run here. Use this to test that the + # entry points work. + + + # You can also put a file called run_test.py in the recipe that will be run + # at test time. + + # requires: + # Put any additional test requirements here. For example + # - nose + +about: + home: https://github.com/shoyer/cyordereddict + license: BSD License + summary: "Cython implementation of Python's collections.OrderedDict" + +# See +# http://docs.continuum.io/conda/build.html for +# more information about meta.yaml diff --git a/conda/logbook/meta.yaml b/conda/logbook/meta.yaml index e21d8cf5..228de950 100644 --- a/conda/logbook/meta.yaml +++ b/conda/logbook/meta.yaml @@ -1,18 +1,19 @@ package: name: logbook - version: !!str 0.6.0 + version: "0.10.0" source: - fn: Logbook-0.6.0.tar.gz - url: https://pypi.python.org/packages/source/L/Logbook/Logbook-0.6.0.tar.gz - md5: 2c77da3adeafd191bb8071cc5ad447bf + fn: Logbook-0.10.0.tar.gz + url: https://pypi.python.org/packages/source/L/Logbook/Logbook-0.10.0.tar.gz + md5: 92439ce6f71f3120d65d84c2a3ab5047 # patches: # List any patch files here # - fix.patch # build: - #preserve_egg_dir: True - #entry_points: + # noarch_python: True + # preserve_egg_dir: True + # entry_points: # Put any entry points (scripts to be generated automatically) here. The # syntax is module:function. For example # @@ -29,16 +30,18 @@ requirements: build: - python - setuptools + - six >=1.4.0 run: - python + - six >=1.4.0 test: # Python imports imports: - logbook - #commands: + # commands: # You can put test commands to be run here. Use this to test that the # entry points work. @@ -53,6 +56,7 @@ test: about: home: http://logbook.pocoo.org/ license: BSD + summary: 'A logging replacement for Python' # See # http://docs.continuum.io/conda/build.html for From 6d83b06133073c3770f79154d7b26d9bd3b239b1 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Thu, 31 Dec 2015 12:26:51 -0500 Subject: [PATCH 3/6] MAINT: De-dupe regex usage --- setup.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 2150a0c6..fc2105c5 100644 --- a/setup.py +++ b/setup.py @@ -133,12 +133,14 @@ def _with_bounds(req): return ''.join(with_bounds) +REQ_PATTERN = re.compile("([^=<>]+)([<=>]{1,2})(.*)") + + def _conda_format(req): - return re.sub( - '([^=<>]+)([=<>]{1,2})', - lambda m: '%s %s' % (m.group(1).lower(), m.group(2)), - req, - 1, + return REQ_PATTERN.sub( + lambda m: '%s %s%s' % (m.group(1).lower(), m.group(2), m.group(3)), + req, + 1, ) @@ -184,14 +186,12 @@ def module_requirements(requirements_path, module_names): module_names = set(module_names) found = set() module_lines = [] - parser = re.compile("([^=<>]+)([<=>]{1,2})(.*)") for line in read_requirements(requirements_path, strict_bounds=True): - match = parser.match(line) + match = REQ_PATTERN.match(line) if match is None: raise AssertionError("Could not parse requirement: '%s'" % line) - groups = match.groups() - name = groups[0] + name = match.group(1) if name in module_names: found.add(name) module_lines.append(line) From 6b4287c390c8b4aa00ed64ea1ac0b3fa3e85cc4a Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 5 Jan 2016 20:39:20 -0500 Subject: [PATCH 4/6] BLD: Cap numpy at 1.9 and don't use exact version with other conda reqs --- conda/zipline/meta.yaml | 2 +- setup.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/conda/zipline/meta.yaml b/conda/zipline/meta.yaml index 030aba91..2cb91a34 100644 --- a/conda/zipline/meta.yaml +++ b/conda/zipline/meta.yaml @@ -18,7 +18,7 @@ requirements: - python - setuptools - cython ==0.22.1 - - numpy ==1.9.2 + - numpy run: - python {% for req in data.get('install_requires', []) -%} diff --git a/setup.py b/setup.py index fc2105c5..665f921b 100644 --- a/setup.py +++ b/setup.py @@ -117,6 +117,7 @@ def _filter_requirements(lines_iter): REQ_UPPER_BOUNDS = { + 'numpy': '<1.10', } @@ -256,8 +257,7 @@ setup( 'Topic :: Scientific/Engineering :: Information Analysis', 'Topic :: System :: Distributed Computing', ], - install_requires=install_requires(strict_bounds=conda_build, - conda_format=conda_build), + install_requires=install_requires(conda_format=conda_build), extras_require=extras_requires(conda_format=conda_build), url="http://zipline.io", ) From 80392e1cdb86b8a9ff66a4c92a1c590e44265acf Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 5 Jan 2016 21:15:08 -0500 Subject: [PATCH 5/6] BLD: Also don't need to specify cython version for conda build Will run setup.py --- conda/zipline/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/zipline/meta.yaml b/conda/zipline/meta.yaml index 2cb91a34..5bbe111f 100644 --- a/conda/zipline/meta.yaml +++ b/conda/zipline/meta.yaml @@ -17,7 +17,7 @@ requirements: build: - python - setuptools - - cython ==0.22.1 + - cython - numpy run: - python From aa0dae62920be26a2b7f62b2b378ce9156d7fc01 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Tue, 5 Jan 2016 21:30:43 -0500 Subject: [PATCH 6/6] DOC: Added whatsnew for conda build --- docs/source/whatsnew/0.8.4.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/source/whatsnew/0.8.4.txt b/docs/source/whatsnew/0.8.4.txt index 66df1f2e..cfdedeb9 100644 --- a/docs/source/whatsnew/0.8.4.txt +++ b/docs/source/whatsnew/0.8.4.txt @@ -126,6 +126,9 @@ Build * Use ``versioneer`` to manage the project ``__version__`` and setup.py version (:issue:`829`). * Fixed coveralls integration on travis build (:issue:`840`). +* Fixed conda build, which now uses git source as its source and reads + requirements using setup.py, instead of copying them and letting them get out + of sync (:issue:`937`). Documentation ~~~~~~~~~~~~~