diff --git a/README.md b/README.md index d048fdf..1845247 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,13 @@ # pip-package-list A small and definitely faulty tool that tries to form a list of packages that you depend on. This is useful in mono-repo's where all dependencies are split into dozens of `requirements.txt` and `setup.py` files. + +One particular use-case that fueled the development of this tool was to create a flat list of dependencies to pre-install in a Docker base image. + +Although there is a number of tools that parse and resolve requirement files, I did not find any that parse `setup.py` files and extract `install_requires`. + +## Usage + + pip-package-list [requirements.txt or setup.py file...] + +You can specify one or more `requirements.txt` or `setup.py` files to be parsed and resolved. diff --git a/pippackagelist/requirements.py b/pippackagelist/requirements.py index bb58b47..6d4a079 100644 --- a/pippackagelist/requirements.py +++ b/pippackagelist/requirements.py @@ -1,4 +1,5 @@ import os +import re from dataclasses import dataclass from typing import Generator, List, Optional @@ -63,7 +64,7 @@ def parse_requirements( yield parse_editable_requirements_entry(line_source, stripped_line) # TODO: add support for other VCS's - elif stripped_line.startswith("git+"): + elif re.match(r"^(.+)\+", stripped_line): yield parse_vcs_requirements_entry(line_source, stripped_line) else: yield parse_package_requirements_entry(line_source, stripped_line) @@ -120,4 +121,10 @@ def parse_package_requirements_entry( def _clean_line(line: str) -> str: - return line.strip().replace("\n", "").replace("\r", "") + return ( + line.strip() + .replace("\n", "") + .replace("\r", "") + .replace(" ", " ") + .replace(" ", " ") + ) diff --git a/pippackagelist/setup_py_parser.py b/pippackagelist/setup_py_parser.py index e587e40..c489827 100644 --- a/pippackagelist/setup_py_parser.py +++ b/pippackagelist/setup_py_parser.py @@ -1,4 +1,3 @@ -import os from typing import Generator @@ -19,11 +18,12 @@ def parse_setup_py(file_path: str) -> Generator[RequirementsEntry, None, None]: setuptools.setup = _setup_proxy - path = os.path.join(file_path, "setup.py") - with open(path, "r") as fp: + with open(file_path, "r") as fp: exec(fp.read()) - source = RequirementsEntrySource(path=path, line=None, line_number=None) + source = RequirementsEntrySource( + path=file_path, line=None, line_number=None + ) requirements = setup_kwargs.get("install_requires") or [] diff --git a/setup.py b/setup.py index 1c736db..d2f92e8 100644 --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ setup( python_requires=">=3.7", install_requires=["setuptools==45.2.0"], extras_require={ + "test": ["pytest==5.2.2", "pytest-cov==2.8.1", ], "analysis": [ "black==19.10b0", "flake8==3.7.7", @@ -64,7 +65,7 @@ setup( "autopep8==1.4.4", "isort==4.3.20", "docformatter==1.3.1", - ] + ], }, cmdclass={ "lint": create_command( diff --git a/tests/fixtures/setup_py.py b/tests/fixtures/setup_py.py new file mode 100644 index 0000000..44276ad --- /dev/null +++ b/tests/fixtures/setup_py.py @@ -0,0 +1,6 @@ +from setuptools import setup + +setup( + name="mywhopackage", + install_requires=["django==1.0", "cookie>=1.2", "-r ../test.txt", "-e ..", ], +) diff --git a/tests/fixtures/setup_py_with_extras.py b/tests/fixtures/setup_py_with_extras.py new file mode 100644 index 0000000..bbc1461 --- /dev/null +++ b/tests/fixtures/setup_py_with_extras.py @@ -0,0 +1,7 @@ +from setuptools import setup + +setup( + name="mywhopackage", + install_requires=["django==1.0", ], + extras_require={"test": ["pytest==2.0", ], "docs": ["Sphinx==1.0", ], }, +) diff --git a/tests/test_parse_requirements.py b/tests/test_parse_requirements.py new file mode 100644 index 0000000..ffa109f --- /dev/null +++ b/tests/test_parse_requirements.py @@ -0,0 +1,145 @@ +import os + +import pytest + +from pippackagelist.requirements import ( + RequirementsEditableEntry, + RequirementsEntrySource, + RequirementsPackageEntry, + RequirementsRecursiveEntry, + RequirementsVCSPackageEntry, + parse_requirements, +) + +source = RequirementsEntrySource( + path="requirements.txt", line=None, line_number=None, +) + + +@pytest.mark.parametrize("path", ["../bla.txt", "./bla.txt", "/test.txt"]) +def test_parse_requirements_recursive_entry(path): + line = "-r %s" % path + + requirements = list(parse_requirements(source, [line])) + assert len(requirements) == 1 + + assert isinstance(requirements[0], RequirementsRecursiveEntry) + assert requirements[0].source.path == source.path + assert requirements[0].source.line == line + assert requirements[0].source.line_number == 1 + assert requirements[0].path == os.path.realpath( + os.path.join(os.getcwd(), path) + ) + + +@pytest.mark.parametrize("path", ["../bla", "./bla", "/mypackage", "."]) +def test_parse_requirements_editable_entry(path): + line = "-e %s" % path + + requirements = list(parse_requirements(source, [line])) + assert len(requirements) == 1 + + assert isinstance(requirements[0], RequirementsEditableEntry) + assert requirements[0].source.path == source.path + assert requirements[0].source.line == line + assert requirements[0].source.line_number == 1 + assert requirements[0].path == os.path.realpath( + os.path.join(os.getcwd(), path) + ) + + +@pytest.mark.parametrize("vcs", ["git", "hg"]) +@pytest.mark.parametrize( + "uri", ["https://github.com/org/repo", "git@github.com:org/repo.git"] +) +@pytest.mark.parametrize("tag", ["test", "1234", None]) +def test_parse_requirements_vcs_package_entry(vcs, uri, tag): + line = f"{vcs}+{uri}" + if tag: + line += f"#{tag}" + + requirements = list(parse_requirements(source, [line])) + assert len(requirements) == 1 + + assert isinstance(requirements[0], RequirementsVCSPackageEntry) + assert requirements[0].source.path == source.path + assert requirements[0].source.line == line + assert requirements[0].source.line_number == 1 + assert requirements[0].vcs == vcs + assert requirements[0].uri == uri + assert requirements[0].tag == tag + + +@pytest.mark.parametrize("operator", ["==", ">=", ">", "<=", "<"]) +def test_parse_requirements_package_entry(operator): + line = "django%s1.0" % operator + + requirements = list(parse_requirements(source, [line])) + assert len(requirements) == 1 + + assert isinstance(requirements[0], RequirementsPackageEntry) + assert requirements[0].source.path == source.path + assert requirements[0].source.line == line + assert requirements[0].source.line_number == 1 + assert requirements[0].name == "django" + assert requirements[0].version == "1.0" + assert requirements[0].operator == operator + + +def test_parse_requirements_skips_comments_and_blank_lines(): + lines = [ + "# this is a comment", + "", + "django==1.0", + " ", + " # another comment", + ] + + requirements = list(parse_requirements(source, lines)) + assert len(requirements) == 1 + assert isinstance(requirements[0], RequirementsPackageEntry) + + +def test_parse_requirements_ignores_leading_and_trailing_whitespace(): + lines = [ + " django==1.0 ", + " -r ./otherfile.txt ", + " -e ../", + " git+https://github.com/test/test#tag", + ] + + requirements = list(parse_requirements(source, lines)) + assert len(requirements) == 4 + + assert isinstance(requirements[2], RequirementsEditableEntry) + assert isinstance(requirements[3], RequirementsVCSPackageEntry) + + assert isinstance(requirements[0], RequirementsPackageEntry) + assert requirements[0].source.path == source.path + assert requirements[0].source.line == "django==1.0" + assert requirements[0].source.line_number == 1 + assert requirements[0].name == "django" + assert requirements[0].version == "1.0" + assert requirements[0].operator == "==" + + assert isinstance(requirements[1], RequirementsRecursiveEntry) + assert requirements[1].source.path == source.path + assert requirements[1].source.line == "-r ./otherfile.txt" + assert requirements[1].source.line_number == 2 + assert requirements[1].path == os.path.join(os.getcwd(), "otherfile.txt") + + assert isinstance(requirements[2], RequirementsEditableEntry) + assert requirements[2].source.path == source.path + assert requirements[2].source.line == "-e ../" + assert requirements[2].source.line_number == 3 + assert requirements[2].path == os.path.realpath( + os.path.join(os.getcwd(), "..") + ) + + assert isinstance(requirements[3], RequirementsVCSPackageEntry) + assert requirements[3].source.path == source.path + assert requirements[3].source.line == "git+https://github.com/test/test#tag" + assert requirements[3].source.line_number == 4 + assert requirements[3].vcs == "git" + assert requirements[3].uri == "https://github.com/test/test" + assert requirements[3].tag == "tag" diff --git a/tests/test_parse_setup_py.py b/tests/test_parse_setup_py.py new file mode 100644 index 0000000..5371596 --- /dev/null +++ b/tests/test_parse_setup_py.py @@ -0,0 +1,51 @@ +import os + +from pippackagelist.requirements import ( + RequirementsEditableEntry, + RequirementsEntrySource, + RequirementsPackageEntry, + RequirementsRecursiveEntry, + RequirementsVCSPackageEntry, +) +from pippackagelist.setup_py_parser import parse_setup_py + +setup_py_path = os.path.join( + os.path.dirname(__file__), "./fixtures/setup_py.py" +) +setup_py_with_extras_path = os.path.join( + os.path.dirname(__file__), "./fixtures/setup_py_with_extras.py" +) + + +def test_parse_setup_py(): + requirements = list(parse_setup_py(setup_py_path)) + assert len(requirements) == 4 + + assert isinstance(requirements[0], RequirementsPackageEntry) + assert isinstance(requirements[1], RequirementsPackageEntry) + assert isinstance(requirements[2], RequirementsRecursiveEntry) + assert isinstance(requirements[3], RequirementsEditableEntry) + + for index, requirement in enumerate(requirements): + assert requirement.source.path == setup_py_path + assert requirement.source.line_number == index + 1 + + +def test_parse_setup_py_with_extras(): + requirements = list(parse_setup_py(setup_py_with_extras_path)) + assert len(requirements) == 3 + + assert isinstance(requirements[0], RequirementsPackageEntry) + assert requirements[0].name == "django" + assert requirements[0].version == "1.0" + assert requirements[0].operator == "==" + + assert isinstance(requirements[1], RequirementsPackageEntry) + assert requirements[1].name == "pytest" + assert requirements[1].version == "2.0" + assert requirements[1].operator == "==" + + assert isinstance(requirements[2], RequirementsPackageEntry) + assert requirements[2].name == "Sphinx" + assert requirements[2].version == "1.0" + assert requirements[2].operator == "=="