Add some basic tests

README.md

@@ -1,3 +1,13 @@
 # pip-package-list
 
 A small and definitely faulty tool that tries to form a list of packages that you depend on. This is useful in mono-repo's where all dependencies are split into dozens of `requirements.txt` and `setup.py` files.
+
+One particular use-case that fueled the development of this tool was to create a flat list of dependencies to pre-install in a Docker base image.
+
+Although there is a number of tools that parse and resolve requirement files, I did not find any that parse `setup.py` files and extract `install_requires`.
+
+## Usage
+
+    pip-package-list [requirements.txt or setup.py file...]
+
+You can specify one or more `requirements.txt` or `setup.py` files to be parsed and resolved.

pippackagelist/requirements.py

@@ -1,4 +1,5 @@
 import os
+import re
 
 from dataclasses import dataclass
 from typing import Generator, List, Optional
@@ -63,7 +64,7 @@ def parse_requirements(
             yield parse_editable_requirements_entry(line_source, stripped_line)
 
         # TODO: add support for other VCS's
-        elif stripped_line.startswith("git+"):
+        elif re.match(r"^(.+)\+", stripped_line):
             yield parse_vcs_requirements_entry(line_source, stripped_line)
         else:
             yield parse_package_requirements_entry(line_source, stripped_line)
@@ -120,4 +121,10 @@ def parse_package_requirements_entry(
 
 
 def _clean_line(line: str) -> str:
-    return line.strip().replace("\n", "").replace("\r", "")
+    return (
+        line.strip()
+        .replace("\n", "")
+        .replace("\r", "")
+        .replace("  ", " ")
+        .replace("  ", " ")
+    )

pippackagelist/setup_py_parser.py

@@ -1,4 +1,3 @@
-import os
 
 from typing import Generator
 
@@ -19,11 +18,12 @@ def parse_setup_py(file_path: str) -> Generator[RequirementsEntry, None, None]:
 
     setuptools.setup = _setup_proxy
 
-    path = os.path.join(file_path, "setup.py")
-    with open(path, "r") as fp:
+    with open(file_path, "r") as fp:
         exec(fp.read())
 
-    source = RequirementsEntrySource(path=path, line=None, line_number=None)
+    source = RequirementsEntrySource(
+        path=file_path, line=None, line_number=None
+    )
 
     requirements = setup_kwargs.get("install_requires") or []
 

setup.py

@@ -57,6 +57,7 @@ setup(
     python_requires=">=3.7",
     install_requires=["setuptools==45.2.0"],
     extras_require={
+        "test": ["pytest==5.2.2", "pytest-cov==2.8.1", ],
         "analysis": [
             "black==19.10b0",
             "flake8==3.7.7",
@@ -64,7 +65,7 @@ setup(
             "autopep8==1.4.4",
             "isort==4.3.20",
             "docformatter==1.3.1",
-        ]
+        ],
     },
     cmdclass={
         "lint": create_command(

tests/fixtures/setup_py.py

@@ -0,0 +1,6 @@
+from setuptools import setup
+
+setup(
+    name="mywhopackage",
+    install_requires=["django==1.0", "cookie>=1.2", "-r ../test.txt", "-e ..", ],
+)

tests/fixtures/setup_py_with_extras.py

@@ -0,0 +1,7 @@
+from setuptools import setup
+
+setup(
+    name="mywhopackage",
+    install_requires=["django==1.0", ],
+    extras_require={"test": ["pytest==2.0", ], "docs": ["Sphinx==1.0", ], },
+)

tests/test_parse_requirements.py

@@ -0,0 +1,145 @@
+import os
+
+import pytest
+
+from pippackagelist.requirements import (
+    RequirementsEditableEntry,
+    RequirementsEntrySource,
+    RequirementsPackageEntry,
+    RequirementsRecursiveEntry,
+    RequirementsVCSPackageEntry,
+    parse_requirements,
+)
+
+source = RequirementsEntrySource(
+    path="requirements.txt", line=None, line_number=None,
+)
+
+
+@pytest.mark.parametrize("path", ["../bla.txt", "./bla.txt", "/test.txt"])
+def test_parse_requirements_recursive_entry(path):
+    line = "-r %s" % path
+
+    requirements = list(parse_requirements(source, [line]))
+    assert len(requirements) == 1
+
+    assert isinstance(requirements[0], RequirementsRecursiveEntry)
+    assert requirements[0].source.path == source.path
+    assert requirements[0].source.line == line
+    assert requirements[0].source.line_number == 1
+    assert requirements[0].path == os.path.realpath(
+        os.path.join(os.getcwd(), path)
+    )
+
+
+@pytest.mark.parametrize("path", ["../bla", "./bla", "/mypackage", "."])
+def test_parse_requirements_editable_entry(path):
+    line = "-e %s" % path
+
+    requirements = list(parse_requirements(source, [line]))
+    assert len(requirements) == 1
+
+    assert isinstance(requirements[0], RequirementsEditableEntry)
+    assert requirements[0].source.path == source.path
+    assert requirements[0].source.line == line
+    assert requirements[0].source.line_number == 1
+    assert requirements[0].path == os.path.realpath(
+        os.path.join(os.getcwd(), path)
+    )
+
+
+@pytest.mark.parametrize("vcs", ["git", "hg"])
+@pytest.mark.parametrize(
+    "uri", ["https://github.com/org/repo", "git@github.com:org/repo.git"]
+)
+@pytest.mark.parametrize("tag", ["test", "1234", None])
+def test_parse_requirements_vcs_package_entry(vcs, uri, tag):
+    line = f"{vcs}+{uri}"
+    if tag:
+        line += f"#{tag}"
+
+    requirements = list(parse_requirements(source, [line]))
+    assert len(requirements) == 1
+
+    assert isinstance(requirements[0], RequirementsVCSPackageEntry)
+    assert requirements[0].source.path == source.path
+    assert requirements[0].source.line == line
+    assert requirements[0].source.line_number == 1
+    assert requirements[0].vcs == vcs
+    assert requirements[0].uri == uri
+    assert requirements[0].tag == tag
+
+
+@pytest.mark.parametrize("operator", ["==", ">=", ">", "<=", "<"])
+def test_parse_requirements_package_entry(operator):
+    line = "django%s1.0" % operator
+
+    requirements = list(parse_requirements(source, [line]))
+    assert len(requirements) == 1
+
+    assert isinstance(requirements[0], RequirementsPackageEntry)
+    assert requirements[0].source.path == source.path
+    assert requirements[0].source.line == line
+    assert requirements[0].source.line_number == 1
+    assert requirements[0].name == "django"
+    assert requirements[0].version == "1.0"
+    assert requirements[0].operator == operator
+
+
+def test_parse_requirements_skips_comments_and_blank_lines():
+    lines = [
+        "# this is a comment",
+        "",
+        "django==1.0",
+        "    ",
+        "   # another comment",
+    ]
+
+    requirements = list(parse_requirements(source, lines))
+    assert len(requirements) == 1
+    assert isinstance(requirements[0], RequirementsPackageEntry)
+
+
+def test_parse_requirements_ignores_leading_and_trailing_whitespace():
+    lines = [
+        "   django==1.0   ",
+        "  -r    ./otherfile.txt  ",
+        " -e ../",
+        "    git+https://github.com/test/test#tag",
+    ]
+
+    requirements = list(parse_requirements(source, lines))
+    assert len(requirements) == 4
+
+    assert isinstance(requirements[2], RequirementsEditableEntry)
+    assert isinstance(requirements[3], RequirementsVCSPackageEntry)
+
+    assert isinstance(requirements[0], RequirementsPackageEntry)
+    assert requirements[0].source.path == source.path
+    assert requirements[0].source.line == "django==1.0"
+    assert requirements[0].source.line_number == 1
+    assert requirements[0].name == "django"
+    assert requirements[0].version == "1.0"
+    assert requirements[0].operator == "=="
+
+    assert isinstance(requirements[1], RequirementsRecursiveEntry)
+    assert requirements[1].source.path == source.path
+    assert requirements[1].source.line == "-r ./otherfile.txt"
+    assert requirements[1].source.line_number == 2
+    assert requirements[1].path == os.path.join(os.getcwd(), "otherfile.txt")
+
+    assert isinstance(requirements[2], RequirementsEditableEntry)
+    assert requirements[2].source.path == source.path
+    assert requirements[2].source.line == "-e ../"
+    assert requirements[2].source.line_number == 3
+    assert requirements[2].path == os.path.realpath(
+        os.path.join(os.getcwd(), "..")
+    )
+
+    assert isinstance(requirements[3], RequirementsVCSPackageEntry)
+    assert requirements[3].source.path == source.path
+    assert requirements[3].source.line == "git+https://github.com/test/test#tag"
+    assert requirements[3].source.line_number == 4
+    assert requirements[3].vcs == "git"
+    assert requirements[3].uri == "https://github.com/test/test"
+    assert requirements[3].tag == "tag"

tests/test_parse_setup_py.py

@@ -0,0 +1,51 @@
+import os
+
+from pippackagelist.requirements import (
+    RequirementsEditableEntry,
+    RequirementsEntrySource,
+    RequirementsPackageEntry,
+    RequirementsRecursiveEntry,
+    RequirementsVCSPackageEntry,
+)
+from pippackagelist.setup_py_parser import parse_setup_py
+
+setup_py_path = os.path.join(
+    os.path.dirname(__file__), "./fixtures/setup_py.py"
+)
+setup_py_with_extras_path = os.path.join(
+    os.path.dirname(__file__), "./fixtures/setup_py_with_extras.py"
+)
+
+
+def test_parse_setup_py():
+    requirements = list(parse_setup_py(setup_py_path))
+    assert len(requirements) == 4
+
+    assert isinstance(requirements[0], RequirementsPackageEntry)
+    assert isinstance(requirements[1], RequirementsPackageEntry)
+    assert isinstance(requirements[2], RequirementsRecursiveEntry)
+    assert isinstance(requirements[3], RequirementsEditableEntry)
+
+    for index, requirement in enumerate(requirements):
+        assert requirement.source.path == setup_py_path
+        assert requirement.source.line_number == index + 1
+
+
+def test_parse_setup_py_with_extras():
+    requirements = list(parse_setup_py(setup_py_with_extras_path))
+    assert len(requirements) == 3
+
+    assert isinstance(requirements[0], RequirementsPackageEntry)
+    assert requirements[0].name == "django"
+    assert requirements[0].version == "1.0"
+    assert requirements[0].operator == "=="
+
+    assert isinstance(requirements[1], RequirementsPackageEntry)
+    assert requirements[1].name == "pytest"
+    assert requirements[1].version == "2.0"
+    assert requirements[1].operator == "=="
+
+    assert isinstance(requirements[2], RequirementsPackageEntry)
+    assert requirements[2].name == "Sphinx"
+    assert requirements[2].version == "1.0"
+    assert requirements[2].operator == "=="