From 540e2962f7279f97e5a22db3a9a7ede521064fae Mon Sep 17 00:00:00 2001 From: Swen Kooij Date: Sat, 29 Feb 2020 12:39:28 +0200 Subject: [PATCH] Add auto-formatting and linting --- .gitignore | 1 + pippackagelist/__main__.py | 11 +- pippackagelist/requirements.py | 84 ++++++------ pippackagelist/requirements_txt_parser.py | 16 ++- pippackagelist/setup_py_parser.py | 21 ++- pyproject.toml | 2 + setup.cfg | 10 ++ setup.py | 153 ++++++++++++++++++++++ 8 files changed, 237 insertions(+), 61 deletions(-) create mode 100644 pyproject.toml create mode 100644 setup.cfg create mode 100644 setup.py diff --git a/.gitignore b/.gitignore index c60542b..19cfe38 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ env/ *.pyc __pycache__ +*.egg-info diff --git a/pippackagelist/__main__.py b/pippackagelist/__main__.py index 30ab11d..d7e7ecc 100644 --- a/pippackagelist/__main__.py +++ b/pippackagelist/__main__.py @@ -1,8 +1,8 @@ import sys -from .setup_py_parser import parse_setup_py +from .requirements import RequirementsEditableEntry, RequirementsRecursiveEntry from .requirements_txt_parser import parse_requirements_txt -from .requirements import RequirementsRecursiveEntry, RequirementsEditableEntry +from .setup_py_parser import parse_setup_py def main() -> int: @@ -11,13 +11,16 @@ def main() -> int: aggregated_entries = [] for requirement in parse_requirements_txt(input_file): if isinstance(requirement, RequirementsRecursiveEntry): - aggregated_entries.extend(list(parse_requirements_txt(requirement.path))) + aggregated_entries.extend( + list(parse_requirements_txt(requirement.path)) + ) elif isinstance(requirement, RequirementsEditableEntry): aggregated_entries.extend(list(parse_setup_py(requirement.path))) else: aggregated_entries.append(requirement) - print(aggregated_entries) + for req in aggregated_entries: + print(req.source.line, req.source.path) return 0 diff --git a/pippackagelist/requirements.py b/pippackagelist/requirements.py index 0fcdf8d..bb58b47 100644 --- a/pippackagelist/requirements.py +++ b/pippackagelist/requirements.py @@ -1,7 +1,7 @@ import os from dataclasses import dataclass -from typing import Generator, Optional, List +from typing import Generator, List, Optional @dataclass @@ -40,83 +40,74 @@ class RequirementsPackageEntry(RequirementsEntry): version: str -def parse_requirements(source: Optional[RequirementsEntrySource], lines: List[str]) -> Generator[RequirementsEntry, None, None]: +def parse_requirements( + source: Optional[RequirementsEntrySource], lines: List[str] +) -> Generator[RequirementsEntry, None, None]: for index, line in enumerate(lines): - stripped_line = line.strip() + stripped_line = _clean_line(line) - if not len(stripped_line) or stripped_line.startswith('#'): + if not len(stripped_line) or stripped_line.startswith("#"): continue line_source = None if source: line_source = RequirementsEntrySource( - path=source.path, - line=line, - line_number=index + 1, + path=source.path, line=stripped_line, line_number=index + 1 ) - if stripped_line.startswith('-r'): - yield parse_recursive_requirements_entry( - line_source, - stripped_line, - ) + if stripped_line.startswith("-r"): + yield parse_recursive_requirements_entry(line_source, stripped_line) - elif stripped_line.startswith('-e'): - yield parse_editable_requirements_entry( - line_source, - stripped_line, - ) + elif stripped_line.startswith("-e"): + yield parse_editable_requirements_entry(line_source, stripped_line) # TODO: add support for other VCS's - elif stripped_line.startswith('git+'): - yield parse_vcs_requirements_entry( - line_source, - stripped_line, - ) + elif stripped_line.startswith("git+"): + yield parse_vcs_requirements_entry(line_source, stripped_line) else: - yield parse_package_requirements_entry( - line_source, - stripped_line, - ) + yield parse_package_requirements_entry(line_source, stripped_line) -def parse_recursive_requirements_entry(source: RequirementsEntrySource, line: str) -> RequirementsRecursiveEntry: - path = line.replace('-r', '').strip() +def parse_recursive_requirements_entry( + source: RequirementsEntrySource, line: str +) -> RequirementsRecursiveEntry: + path = _clean_line(line.replace("-r", "")) return RequirementsRecursiveEntry( source=source, - path=os.path.join(os.path.dirname(source.path), path), + path=os.path.realpath(os.path.join(os.path.dirname(source.path), path)), ) -def parse_editable_requirements_entry(source: RequirementsEntrySource, line: str) -> RequirementsEditableEntry: - path = line.replace('-e', '').strip() +def parse_editable_requirements_entry( + source: RequirementsEntrySource, line: str +) -> RequirementsEditableEntry: + path = _clean_line(line.replace("-e", "")) return RequirementsEditableEntry( source=source, - path=os.path.join(os.path.dirname(source.path), path), + path=os.path.realpath(os.path.join(os.path.dirname(source.path), path)), ) -def parse_vcs_requirements_entry(source: RequirementsEntrySource, line: str) -> RequirementsVCSPackageEntry: - vcs_uri_split = line.split('+') +def parse_vcs_requirements_entry( + source: RequirementsEntrySource, line: str +) -> RequirementsVCSPackageEntry: + vcs_uri_split = line.split("+") vcs = vcs_uri_split[0] - uri_tag_split = vcs_uri_split[1].split('#') + uri_tag_split = vcs_uri_split[1].split("#") uri = uri_tag_split[0] tag = None if len(uri_tag_split) > 1: tag = uri_tag_split[1] - return RequirementsVCSPackageEntry( - source=source, - vcs=vcs, - uri=uri, - tag=tag, - ) + return RequirementsVCSPackageEntry(source=source, vcs=vcs, uri=uri, tag=tag) -def parse_package_requirements_entry(source: RequirementsEntrySource, line: str) -> RequirementsPackageEntry: +def parse_package_requirements_entry( + source: RequirementsEntrySource, line: str +) -> RequirementsPackageEntry: operators = ["==", ">=", ">", "<=", "<"] for operator in operators: parts = line.split(operator) @@ -124,8 +115,9 @@ def parse_package_requirements_entry(source: RequirementsEntrySource, line: str) continue return RequirementsPackageEntry( - source=source, - name=parts[0], - operator=operator, - version=parts[1], + source=source, name=parts[0], operator=operator, version=parts[1] ) + + +def _clean_line(line: str) -> str: + return line.strip().replace("\n", "").replace("\r", "") diff --git a/pippackagelist/requirements_txt_parser.py b/pippackagelist/requirements_txt_parser.py index f6b6a11..9b6facd 100644 --- a/pippackagelist/requirements_txt_parser.py +++ b/pippackagelist/requirements_txt_parser.py @@ -1,11 +1,19 @@ from typing import Generator -from .requirements import parse_requirements, RequirementsEntry, RequirementsEntrySource +from .requirements import ( + RequirementsEntry, + RequirementsEntrySource, + parse_requirements, +) -def parse_requirements_txt(file_path: str) -> Generator[RequirementsEntry, None, None]: - source = RequirementsEntrySource(path=file_path, line=None, line_number=None) +def parse_requirements_txt( + file_path: str, +) -> Generator[RequirementsEntry, None, None]: + source = RequirementsEntrySource( + path=file_path, line=None, line_number=None + ) - with open(file_path, 'r') as fp: + with open(file_path, "r") as fp: for requirement in parse_requirements(source, fp.readlines()): yield requirement diff --git a/pippackagelist/setup_py_parser.py b/pippackagelist/setup_py_parser.py index 705b2a7..e587e40 100644 --- a/pippackagelist/setup_py_parser.py +++ b/pippackagelist/setup_py_parser.py @@ -1,9 +1,14 @@ import os -import setuptools from typing import Generator -from .requirements import RequirementsEntry, RequirementsEntrySource, parse_requirements +import setuptools + +from .requirements import ( + RequirementsEntry, + RequirementsEntrySource, + parse_requirements, +) def parse_setup_py(file_path: str) -> Generator[RequirementsEntry, None, None]: @@ -11,18 +16,20 @@ def parse_setup_py(file_path: str) -> Generator[RequirementsEntry, None, None]: def _setup_proxy(*args, **kwargs): setup_kwargs.update(dict(kwargs)) + setuptools.setup = _setup_proxy path = os.path.join(file_path, "setup.py") with open(path, "r") as fp: exec(fp.read()) - source = RequirementsEntrySource( - path=path, - line=None, - line_number=None, - ) + source = RequirementsEntrySource(path=path, line=None, line_number=None) requirements = setup_kwargs.get("install_requires") or [] + + extras_require = setup_kwargs.get("extras_require") or {} + for _, extra_requirements in extras_require.items(): + requirements.extend(extra_requirements) + for requirement in parse_requirements(source, requirements): yield requirement diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..83c116e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.black] +line-length = 80 diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..1a47aab --- /dev/null +++ b/setup.cfg @@ -0,0 +1,10 @@ +[flake8] +ignore = E252,E501,W503 +exclude = env,.tox,.git + +[isort] +line_length=80 +multi_line_output=3 +lines_between_types=1 +include_trailing_comma=True +not_skip=__init__.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..1c736db --- /dev/null +++ b/setup.py @@ -0,0 +1,153 @@ +import distutils.cmd +import os +import subprocess + +from setuptools import find_packages, setup + + +class BaseCommand(distutils.cmd.Command): + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + +def create_command(text, commands): + """Creates a custom setup.py command.""" + + class CustomCommand(BaseCommand): + description = text + + def run(self): + for cmd in commands: + subprocess.check_call(cmd) + + return CustomCommand + + +with open( + os.path.join(os.path.dirname(__file__), "README.md"), encoding="utf-8" +) as readme: + README = readme.read() + + +setup( + name="pip-package-list", + version="1.0.0", + packages=find_packages(), + include_package_data=True, + license="MIT License", + description="Generate a flat list of packages Pip would install.", + long_description=README, + long_description_content_type="text/markdown", + url="https://github.com/Photonios/pip-package-list", + author="Swen Kooij", + author_email="swenkooij@gmail.com", + keywords=["pip", "package", "resolver", "list", "requirements"], + classifiers=[ + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3.5", + ], + python_requires=">=3.7", + install_requires=["setuptools==45.2.0"], + extras_require={ + "analysis": [ + "black==19.10b0", + "flake8==3.7.7", + "autoflake==1.3", + "autopep8==1.4.4", + "isort==4.3.20", + "docformatter==1.3.1", + ] + }, + cmdclass={ + "lint": create_command( + "Lints the code", + [["flake8", "setup.py", "pippackagelist", "tests"]], + ), + "lint_fix": create_command( + "Lints the code", + [ + [ + "autoflake", + "--remove-all-unused-imports", + "-i", + "-r", + "setup.py", + "pippackagelist", + "tests", + ], + ["autopep8", "-i", "-r", "setup.py", "pippackagelist", "tests"], + ], + ), + "format": create_command( + "Formats the code", + [["black", "setup.py", "pippackagelist", "tests"]], + ), + "format_verify": create_command( + "Checks if the code is auto-formatted", + [["black", "--check", "setup.py", "pippackagelist", "tests"]], + ), + "format_docstrings": create_command( + "Auto-formats doc strings", [["docformatter", "-r", "-i", "."]] + ), + "format_docstrings_verify": create_command( + "Verifies that doc strings are properly formatted", + [["docformatter", "-r", "-c", "."]], + ), + "sort_imports": create_command( + "Automatically sorts imports", + [ + ["isort", "setup.py"], + ["isort", "-rc", "pippackagelist"], + ["isort", "-rc", "tests"], + ], + ), + "sort_imports_verify": create_command( + "Verifies all imports are properly sorted.", + [ + ["isort", "-c", "setup.py"], + ["isort", "-c", "-rc", "pippackagelist"], + ["isort", "-c", "-rc", "tests"], + ], + ), + "fix": create_command( + "Automatically format code and fix linting errors", + [ + ["python", "setup.py", "format"], + ["python", "setup.py", "format_docstrings"], + ["python", "setup.py", "sort_imports"], + ["python", "setup.py", "lint_fix"], + ], + ), + "verify": create_command( + "Verifies whether the code is auto-formatted and has no linting errors", + [ + ["python", "setup.py", "format_verify"], + ["python", "setup.py", "format_docstrings_verify"], + ["python", "setup.py", "sort_imports_verify"], + ["python", "setup.py", "lint"], + ], + ), + "test": create_command( + "Runs all the tests", + [ + [ + "pytest", + "--cov=pippackagelist", + "--cov-report=term", + "--cov-report=xml:reports/xml", + "--cov-report=html:reports/html", + "--junitxml=reports/junit/tests.xml", + "--reuse-db", + ] + ], + ), + }, +)