From 39fbdd30e308dc979eb4785a314e77150f01f488 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 4 Apr 2016 18:17:01 -0400 Subject: [PATCH 1/3] MAINT: Use environment markers instead of separate requirements file for py2 --- etc/requirements.txt | 3 ++ etc/requirements_py2.txt | 2 - setup.py | 79 ++++++++++++---------------------------- 3 files changed, 27 insertions(+), 57 deletions(-) delete mode 100644 etc/requirements_py2.txt diff --git a/etc/requirements.txt b/etc/requirements.txt index 7ffa9527..3f19bdd4 100644 --- a/etc/requirements.txt +++ b/etc/requirements.txt @@ -33,6 +33,9 @@ cyordereddict==0.2.2 # faster array ops. bottleneck==1.0.0 +# lru_cache +functools32==3.2.3.post2; python_version < "3.0" + contextlib2==0.4.0 # networkx requires decorator diff --git a/etc/requirements_py2.txt b/etc/requirements_py2.txt deleted file mode 100644 index 1d2aa0f4..00000000 --- a/etc/requirements_py2.txt +++ /dev/null @@ -1,2 +0,0 @@ -# Caching and other utilities -functools32==3.2.3.post2 diff --git a/setup.py b/setup.py index 3d8ed36c..02706c43 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. from __future__ import print_function -from functools import partial import os import re import sys @@ -112,35 +111,8 @@ STR_TO_CMP = { def _filter_requirements(lines_iter): for line in lines_iter: line = line.strip() - if not line or line.startswith('#'): - continue - - # pip install -r understands line with ;python_version<'3.0', but - # whatever happens inside extras_requires doesn't. Parse the line - # manually and conditionally add it if needed. - if ';' not in line: + if line and not line.startswith('#'): yield line - continue - - requirement, version_spec = line.split(';') - try: - groups = re.match( - "(python_version)([<>=]{1,2})(')([0-9\.]+)(')(.*)", - version_spec, - ).groups() - comp = STR_TO_CMP[groups[1]] - version_spec = StrictVersion(groups[3]) - except Exception as e: - # My kingdom for a 'raise from'! - raise AssertionError( - "Couldn't parse requirement line; '%s'\n" - "Error was:\n" - "%r" % (line, e) - ) - - sys_version = '.'.join(list(map(str, sys.version_info[:3]))) - if comp(sys_version, version_spec): - yield requirement # We don't currently have any known upper bounds. @@ -160,30 +132,34 @@ def _with_bounds(req): return ''.join(with_bounds) -REQ_PATTERN = re.compile("([^=<>]+)([<=>]{1,2})(.*)") +REQ_PATTERN = re.compile("(?P[^=<>]+)(?P[<=>]{1,2})(?P[^;]+)" + "(?:(;\W*python_version\W*(?P[<=>]{1,2})\W*" + "(?P[0-9\.]+)))?") -def _conda_format(req, selector=None): - match = REQ_PATTERN.match(req) - if match and match.group(1).lower() == 'numpy': - line = 'numpy x.x' - else: - line = REQ_PATTERN.sub( - lambda m: '%s %s%s' % (m.group(1).lower(), m.group(2), m.group(3)), - req, - 1, - ) +def _conda_format(req): + def _sub(m): + name = m.group('name').lower() + if name == 'numpy': + return 'numpy x.x' - if selector is not None: - line += ' # [%s]' % selector + formatted = '%s %s%s' % ((name,) + m.group('comp', 'spec')) + pycomp, pyspec = m.group('pycomp', 'pyspec') + if pyspec: + # Compare the two-digit string versions as ints. + selector = ' # [int(py) %s int(%s)]' % ( + pycomp, ''.join(pyspec.split('.')[:2]).ljust(2, '0') + ) + return formatted + selector - return line + return formatted + + return REQ_PATTERN.sub(_sub, req, 1) def read_requirements(path, strict_bounds, - conda_format=False, - conda_selector=None): + conda_format=False): """ Read a requirements.txt file, expressed as a path relative to Zipline root. @@ -198,22 +174,15 @@ def read_requirements(path, reqs = map(_with_bounds, reqs) if conda_format: - reqs = map(partial(_conda_format, selector=conda_selector), reqs) + reqs = map(_conda_format, reqs) return list(reqs) def install_requires(strict_bounds=False, conda_format=False): - reqs = read_requirements('etc/requirements.txt', + return read_requirements('etc/requirements.txt', strict_bounds=strict_bounds, conda_format=conda_format) - if sys.version_info.major == 2 or conda_format: - reqs += read_requirements('etc/requirements_py2.txt', - strict_bounds=strict_bounds, - conda_format=conda_format, - conda_selector='py2k') - - return reqs def extras_requires(conda_format=False): @@ -239,7 +208,7 @@ def module_requirements(requirements_path, module_names, strict_bounds, if match is None: raise AssertionError("Could not parse requirement: '%s'" % line) - name = match.group(1) + name = match.group('name') if name in module_names: found.add(name) if conda_format: From 79f89c4779a30b53a30d8fd610a775850ac8cdf2 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 4 Apr 2016 18:33:14 -0400 Subject: [PATCH 2/3] MAINT: Moved filtering logic into _filter_requirements --- setup.py | 45 ++++++++++++++++++++++----------------------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/setup.py b/setup.py index 02706c43..d40e5ee9 100644 --- a/setup.py +++ b/setup.py @@ -108,11 +108,18 @@ STR_TO_CMP = { } -def _filter_requirements(lines_iter): +def _filter_requirements(lines_iter, filter_names=None): for line in lines_iter: line = line.strip() if line and not line.startswith('#'): - yield line + match = REQ_PATTERN.match(line) + if match is None: + raise AssertionError( + "Could not parse requirement: '%s'" % line) + + name = match.group('name') + if filter_names is None or name in filter_names: + yield line # We don't currently have any known upper bounds. @@ -159,7 +166,8 @@ def _conda_format(req): def read_requirements(path, strict_bounds, - conda_format=False): + conda_format=False, + filter_names=None): """ Read a requirements.txt file, expressed as a path relative to Zipline root. @@ -168,7 +176,7 @@ def read_requirements(path, """ real_path = join(dirname(abspath(__file__)), path) with open(real_path) as f: - reqs = _filter_requirements(f.readlines()) + reqs = _filter_requirements(f.readlines(), filter_names=filter_names) if not strict_bounds: reqs = map(_with_bounds, reqs) @@ -197,34 +205,25 @@ def extras_requires(conda_format=False): return extras -def module_requirements(requirements_path, module_names, strict_bounds, - conda_format=False): +def setup_requirements(requirements_path, module_names, strict_bounds, + conda_format=False): module_names = set(module_names) - found = set() - module_lines = [] - for line in read_requirements(requirements_path, - strict_bounds=strict_bounds): - match = REQ_PATTERN.match(line) - if match is None: - raise AssertionError("Could not parse requirement: '%s'" % line) + module_lines = read_requirements(requirements_path, + strict_bounds=strict_bounds, + conda_format=conda_format, + filter_names=module_names) - name = match.group('name') - if name in module_names: - found.add(name) - if conda_format: - line = _conda_format(line) - module_lines.append(line) - - if found != module_names: + if len(set(module_lines)) != len(module_names): raise AssertionError( - "No requirements found for %s." % (module_names - found) + "Missing requirements. Looking for %s, but found %s." + % (module_names, module_lines) ) return module_lines conda_build = os.path.basename(sys.argv[0]) in ('conda-build', # unix 'conda-build-script.py') # win -setup_requires = module_requirements( +setup_requires = setup_requirements( 'etc/requirements.txt', ('Cython', 'numpy'), strict_bounds=conda_build, From c9a7b936ed2b5110a55547ddad6942eb44615328 Mon Sep 17 00:00:00 2001 From: Richard Frank Date: Mon, 4 Apr 2016 23:07:41 -0400 Subject: [PATCH 3/3] MAINT: Fix pip install -e . --- setup.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index d40e5ee9..edddf402 100644 --- a/setup.py +++ b/setup.py @@ -107,19 +107,37 @@ STR_TO_CMP = { '>=': ge, } +SYS_VERSION = '.'.join(list(map(str, sys.version_info[:3]))) -def _filter_requirements(lines_iter, filter_names=None): + +def _filter_requirements(lines_iter, filter_names=None, + filter_sys_version=False): for line in lines_iter: line = line.strip() - if line and not line.startswith('#'): - match = REQ_PATTERN.match(line) - if match is None: - raise AssertionError( - "Could not parse requirement: '%s'" % line) + if not line or line.startswith('#'): + continue - name = match.group('name') - if filter_names is None or name in filter_names: - yield line + match = REQ_PATTERN.match(line) + if match is None: + raise AssertionError( + "Could not parse requirement: '%s'" % line) + + name = match.group('name') + if filter_names is not None and name not in filter_names: + continue + + if filter_sys_version and match.group('pyspec'): + pycomp, pyspec = match.group('pycomp', 'pyspec') + comp = STR_TO_CMP[pycomp] + pyver_spec = StrictVersion(pyspec) + if comp(SYS_VERSION, pyver_spec): + # pip install -r understands lines with ;python_version<'3.0', + # but pip install -e does not. Filter here, removing the + # env marker. + yield line.split(';')[0] + continue + + yield line # We don't currently have any known upper bounds. @@ -176,7 +194,8 @@ def read_requirements(path, """ real_path = join(dirname(abspath(__file__)), path) with open(real_path) as f: - reqs = _filter_requirements(f.readlines(), filter_names=filter_names) + reqs = _filter_requirements(f.readlines(), filter_names=filter_names, + filter_sys_version=not conda_format) if not strict_bounds: reqs = map(_with_bounds, reqs)