First push

.gitignore

@@ -0,0 +1,6 @@
+.env/
+venv/
+env/
+
+*.pyc
+__pycache__

README.md

@@ -0,0 +1,3 @@
+# pip-package-list
+
+A small and definitely faulty tool that tries to form a list of packages that you depend on. This is useful in mono-repo's where all dependencies are split into dozens of `requirements.txt` and `setup.py` files.

pippackagelist/__main__.py

@@ -0,0 +1,25 @@
+import sys
+
+from .setup_py_parser import parse_setup_py
+from .requirements_txt_parser import parse_requirements_txt
+from .requirements import RequirementsRecursiveEntry, RequirementsEditableEntry
+
+
+def main() -> int:
+    input_file = sys.argv[1]
+
+    aggregated_entries = []
+    for requirement in parse_requirements_txt(input_file):
+        if isinstance(requirement, RequirementsRecursiveEntry):
+            aggregated_entries.extend(list(parse_requirements_txt(requirement.path)))
+        elif isinstance(requirement, RequirementsEditableEntry):
+            aggregated_entries.extend(list(parse_setup_py(requirement.path)))
+        else:
+            aggregated_entries.append(requirement)
+
+    print(aggregated_entries)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

pippackagelist/requirements.py

@@ -0,0 +1,131 @@
+import os
+
+from dataclasses import dataclass
+from typing import Generator, Optional, List
+
+
+@dataclass
+class RequirementsEntrySource:
+    path: str
+    line: Optional[str]
+    line_number: Optional[str]
+
+
+@dataclass
+class RequirementsEntry:
+    source: Optional[RequirementsEntrySource]
+
+
+@dataclass
+class RequirementsRecursiveEntry(RequirementsEntry):
+    path: str
+
+
+@dataclass
+class RequirementsEditableEntry(RequirementsEntry):
+    path: str
+
+
+@dataclass
+class RequirementsVCSPackageEntry(RequirementsEntry):
+    vcs: str
+    uri: str
+    tag: Optional[str]
+
+
+@dataclass
+class RequirementsPackageEntry(RequirementsEntry):
+    name: str
+    operator: str
+    version: str
+
+
+def parse_requirements(source: Optional[RequirementsEntrySource], lines: List[str]) -> Generator[RequirementsEntry, None, None]:
+
+    for index, line in enumerate(lines):
+        stripped_line = line.strip()
+
+        if not len(stripped_line) or stripped_line.startswith('#'):
+            continue
+
+        line_source = None
+        if source:
+            line_source = RequirementsEntrySource(
+                path=source.path,
+                line=line,
+                line_number=index + 1,
+            )
+
+        if stripped_line.startswith('-r'):
+            yield parse_recursive_requirements_entry(
+                line_source,
+                stripped_line,
+            )
+
+        elif stripped_line.startswith('-e'):
+            yield parse_editable_requirements_entry(
+                line_source,
+                stripped_line,
+            )
+
+        # TODO: add support for other VCS's
+        elif stripped_line.startswith('git+'):
+            yield parse_vcs_requirements_entry(
+                line_source,
+                stripped_line,
+            )
+        else:
+            yield parse_package_requirements_entry(
+                line_source,
+                stripped_line,
+            )
+
+
+def parse_recursive_requirements_entry(source: RequirementsEntrySource, line: str) -> RequirementsRecursiveEntry:
+    path = line.replace('-r', '').strip()
+    return RequirementsRecursiveEntry(
+        source=source,
+        path=os.path.join(os.path.dirname(source.path), path),
+    )
+
+
+def parse_editable_requirements_entry(source: RequirementsEntrySource, line: str) -> RequirementsEditableEntry:
+    path = line.replace('-e', '').strip()
+    return RequirementsEditableEntry(
+        source=source,
+        path=os.path.join(os.path.dirname(source.path), path),
+    )
+
+
+def parse_vcs_requirements_entry(source: RequirementsEntrySource, line: str) -> RequirementsVCSPackageEntry:
+    vcs_uri_split = line.split('+')
+    vcs = vcs_uri_split[0]
+
+    uri_tag_split = vcs_uri_split[1].split('#')
+    uri = uri_tag_split[0]
+
+    tag = None
+    if len(uri_tag_split) > 1:
+        tag = uri_tag_split[1]
+
+    return RequirementsVCSPackageEntry(
+        source=source,
+        vcs=vcs,
+        uri=uri,
+        tag=tag,
+    )
+
+
+def parse_package_requirements_entry(source: RequirementsEntrySource, line: str) -> RequirementsPackageEntry:
+    operators = ["==", ">=", ">", "<=", "<"]
+    for operator in operators:
+        parts = line.split(operator)
+        if len(parts) == 1:
+            continue
+
+        return RequirementsPackageEntry(
+            source=source,
+            name=parts[0],
+            operator=operator,
+            version=parts[1],
+        )

pippackagelist/requirements_txt_parser.py

@@ -0,0 +1,11 @@
+from typing import Generator
+
+from .requirements import parse_requirements, RequirementsEntry, RequirementsEntrySource
+
+
+def parse_requirements_txt(file_path: str) -> Generator[RequirementsEntry, None, None]:
+    source = RequirementsEntrySource(path=file_path, line=None, line_number=None)
+
+    with open(file_path, 'r') as fp:
+        for requirement in parse_requirements(source, fp.readlines()):
+            yield requirement

pippackagelist/setup_py_parser.py

@@ -0,0 +1,28 @@
+import os
+import setuptools
+
+from typing import Generator
+
+from .requirements import RequirementsEntry, RequirementsEntrySource, parse_requirements
+
+
+def parse_setup_py(file_path: str) -> Generator[RequirementsEntry, None, None]:
+    setup_kwargs = {}
+
+    def _setup_proxy(*args, **kwargs):
+        setup_kwargs.update(dict(kwargs))
+    setuptools.setup = _setup_proxy
+
+    path = os.path.join(file_path, "setup.py")
+    with open(path, "r") as fp:
+        exec(fp.read())
+
+    source = RequirementsEntrySource(
+        path=path,
+        line=None,
+        line_number=None,
+    )
+
+    requirements = setup_kwargs.get("install_requires") or []
+    for requirement in parse_requirements(source, requirements):
+        yield requirement