From bfa9f754c258a10f7dfa3b76e436f35323652e4a Mon Sep 17 00:00:00 2001 From: "Colin B. Macdonald" Date: Sat, 18 Mar 2023 08:41:54 -0700 Subject: [PATCH] Add yaml output (#41) Adds yaml output. * Write yaml if output file ends in yaml or yml extension * Writes yaml if a "--yaml" cmdline arg was provided * Adds test for yaml output * Adds ci tests with a matrix of extra dependencies. * Adds yaml dependency: `pyyaml` as an extra dependency in pyproject.toml * Adds yaml dependeny: `types-pyyaml` as a lint dev dependency. * Makes the indentation of .json output consistent Co-authored-by: yfprojects <62463991+real-yfprojects@users.noreply.github.com> Co-authored-by: real-yfprojects --- .github/workflows/ci.yml | 35 +++++++++++++++++- poetry.lock | 17 ++++++++- pyproject.toml | 35 ++++++++++-------- req2flatpak.py | 77 +++++++++++++++++++++++++++++++-------- tests/test_req2flatpak.py | 46 ++++++++++++++++++++--- 5 files changed, 172 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7df49c..d78daf0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,8 +23,41 @@ jobs: - name: Run linters run: make lint + generate_test_matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - name: Checkout + uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: setup python + uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0 + with: + python-version: "3.11" + + - name: Extract extras from `pyproject.toml` + id: set-matrix + shell: python + run: | + import tomllib + import os + import json + with open('pyproject.toml', 'rb') as f: + manifest = tomllib.load(f) + yaml = { 'include' : [{ 'extras' : extra} for extra in [''] + list(manifest['tool']['poetry']['extras'])]} + out = json.dumps(yaml) + print(out) + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + f.write('matrix=' + out) + test: + name: test ${{ matrix.extras && 'with' || '' }} ${{ matrix.extras }} runs-on: ubuntu-latest + needs: generate_test_matrix + strategy: + matrix: ${{ fromJson(needs.generate_test_matrix.outputs.matrix) }} + fail-fast: false steps: - name: Checkout uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 @@ -33,7 +66,7 @@ jobs: id: setup uses: ./.github/actions/setup with: - install-options: --without lint + install-options: --without lint ${{ matrix.extras && format('--extras "{0}"', matrix.extras) || '' }} - name: Run Tests run: make test diff --git a/poetry.lock b/poetry.lock index 7f8abe1..fa66278 100644 --- a/poetry.lock +++ b/poetry.lock @@ -719,7 +719,7 @@ files = [ name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1094,6 +1094,18 @@ files = [ {file = "types_docutils-0.19.1.4-py3-none-any.whl", hash = "sha256:870d71f3663141f67a3c59d26d2c54a8c478c842208bb0b345fbf6036f49f561"}, ] +[[package]] +name = "types-pyyaml" +version = "6.0.12.8" +description = "Typing stubs for PyYAML" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "types-PyYAML-6.0.12.8.tar.gz", hash = "sha256:19304869a89d49af00be681e7b267414df213f4eb89634c4495fa62e8f942b9f"}, + {file = "types_PyYAML-6.0.12.8-py3-none-any.whl", hash = "sha256:5314a4b2580999b2ea06b2e5f9a7763d860d6e09cdf21c0e9561daa9cbd60178"}, +] + [[package]] name = "types-setuptools" version = "65.7.0.4" @@ -1230,8 +1242,9 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [extras] packaging = ["packaging"] +yaml = ["pyyaml"] [metadata] lock-version = "2.0" python-versions = "^3.7.2" -content-hash = "750c7f502c3eaec5a8d8349251f90fd9fcf577fb4185ca3716e0a54f21e96337" +content-hash = "c5b1d81dd25cb29f850a0b2a212ea42efd3afe44050829e25d70854669f82d45" diff --git a/pyproject.toml b/pyproject.toml index 55a2291..b770e7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,24 +1,26 @@ [build-system] -requires = ["poetry-core"] +requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.poetry] -name = "req2flatpak" -version = "0.1.0" +name = "req2flatpak" +version = "0.1.0" description = "Generates a flatpak-builder build module for installing python packages defined in requirements.txt files." -authors = ["johannesjh "] -license = "MIT" -readme = "README.rst" +authors = ["johannesjh "] +license = "MIT" +readme = "README.rst" [tool.poetry.scripts] req2flatpak = 'req2flatpak:main' [tool.poetry.dependencies] -python = "^3.7.2" +python = "^3.7.2" packaging = { version = "^21.3", optional = true } +pyyaml = { version = "^6.0", optional = true } [tool.poetry.extras] packaging = ["packaging"] +yaml = ["pyyaml"] [tool.poetry.group.lint.dependencies] pylama = { extras = [ @@ -28,7 +30,10 @@ pylama = { extras = [ "toml", ], version = "^8.4.1" } bandit = { extras = ["toml"], version = "^1.7.4" } -types-setuptools = "^65.5.0.2" # type stubs for mypy linting + +# type stubs for mypy linting +types-setuptools = "^65.5.0.2" +types-pyyaml = "^6.0.12.8" # [tool.poetry.group.tests.dependencies] pydocstyle = "<6.2" @@ -37,18 +42,18 @@ pydocstyle = "<6.2" optional = true [tool.poetry.group.docs.dependencies] -sphinx = "^5.3.0" -sphinx-argparse = "^0.3.2" +sphinx = "^5.3.0" +sphinx-argparse = "^0.3.2" sphinx-rtd-theme-github-versions = "^1.1" [tool.isort] -profile = "black" +profile = "black" skip_gitignore = true [tool.pylama] max_line_length = 88 -concurrent = true -linters = "pycodestyle,pydocstyle,pyflakes,pylint,eradicate,mypy" +concurrent = true +linters = "pycodestyle,pydocstyle,pyflakes,pylint,eradicate,mypy" [tool.pylama.linter.pycodestyle] ignore = "W503,E203,E501" @@ -63,8 +68,8 @@ ignore = ["D202", "D203", "D205", "D401", "D212"] [tool.pylint] format.max-line-length = 88 -main.disable = ["C0301"] # ignore line-too-long -basic.good-names = ["e", "f", "py"] +main.disable = ["C0301"] # ignore line-too-long +basic.good-names = ["e", "f", "py"] [tool.bandit] exclude_dirs = ['tests'] diff --git a/req2flatpak.py b/req2flatpak.py index 3cd42f8..e0c8b71 100755 --- a/req2flatpak.py +++ b/req2flatpak.py @@ -64,6 +64,10 @@ logger = logging.getLogger(__name__) +try: + import yaml +except ImportError: + yaml = None # type: ignore # ============================================================================= # Helper functions / semi vendored code @@ -73,7 +77,6 @@ # added with py 3.8 from functools import cached_property # type: ignore[attr-defined] except ImportError: - # Inspired by the implementation in the standard library # pylint: disable=invalid-name,too-few-public-methods class cached_property: # type: ignore[no-redef] @@ -551,9 +554,7 @@ def downloads( preferred downloads are returned first. """ cache = set() - for (platform_tag, download) in product( - platform.python_tags, release.downloads - ): + for platform_tag, download in product(platform.python_tags, release.downloads): if download in cache: continue if wheels_only and not download.is_wheel: @@ -634,12 +635,30 @@ def sources(downloads: Iterable[Download]) -> List[dict]: @classmethod def build_module_as_str(cls, *args, **kwargs) -> str: """ - Generates a build module for inclusion in a flatpak-builder build manifest. + Generate JSON build module for inclusion in a flatpak-builder build manifest. + + The args and kwargs are the same as in + :py:meth:`~req2flatpak.FlatpakGenerator.build_module` + """ + return json.dumps(cls.build_module(*args, **kwargs), indent=4) + + @classmethod + def build_module_as_yaml_str(cls, *args, **kwargs) -> str: + """ + Generate YAML build module for inclusion in a flatpak-builder build manifest. The args and kwargs are the same as in :py:meth:`~req2flatpak.FlatpakGenerator.build_module` """ - return json.dumps(cls.build_module(*args, **kwargs), indent=2) + # optional dependency, not imported at top + if not yaml: + raise ImportError( + "Package `pyyaml` has to be installed for the yaml format." + ) + + return yaml.dump( + cls.build_module(*args, **kwargs), default_flow_style=False, sort_keys=False + ) # ============================================================================= @@ -676,12 +695,22 @@ def cli_parser() -> argparse.ArgumentParser: default=False, help="Uses a persistent cache when querying pypi.", ) + parser.add_argument( + "--yaml", + action="store_true", + help="Write YAML instead of the default JSON. Needs the 'pyyaml' package.", + ) + parser.add_argument( "--outfile", "-o", nargs="?", type=argparse.FileType("w"), default=sys.stdout, + help=""" + By default, writes JSON but specify a '.yaml' extension and YAML + will be written instead, provided you have the 'pyyaml' package. + """, ) parser.add_argument( "--platform-info", @@ -698,7 +727,7 @@ def cli_parser() -> argparse.ArgumentParser: return parser -def main(): +def main(): # pylint: disable=too-many-branches """Main function that provides req2flatpak's commandline interface.""" # process commandline arguments @@ -706,13 +735,25 @@ def main(): options = parser.parse_args() # stream output to a file or to stdout - output_stream = options.outfile if hasattr(options.outfile, "write") else sys.stdout + if hasattr(options.outfile, "write"): + output_stream = options.outfile + if pathlib.Path(output_stream.name).suffix.casefold() in (".yaml", ".yml"): + options.yaml = True + else: + output_stream = sys.stdout + + if options.yaml and not yaml: + parser.error( + "Outputing YAML requires 'pyyaml' package: try 'pip install pyyaml'" + ) # print platform info if requested, and exit if options.platform_info: - json.dump( - asdict(PlatformFactory.from_current_interpreter()), output_stream, indent=4 - ) + info = asdict(PlatformFactory.from_current_interpreter()) + if options.yaml: + yaml.dump(info, output_stream, default_flow_style=False, sort_keys=False) + else: + json.dump(info, output_stream, indent=4) parser.exit() # print installed packages if requested, and exit @@ -770,10 +811,16 @@ def main(): } # generate flatpak-builder build module - build_module = FlatpakGenerator.build_module(requirements, downloads) - - # write output - json.dump(build_module, output_stream, indent=4) + if options.yaml: + # write yaml + output_stream.write( + FlatpakGenerator.build_module_as_yaml_str(requirements, downloads) + ) + else: + # write json + output_stream.write( + FlatpakGenerator.build_module_as_str(requirements, downloads) + ) if __name__ == "__main__": diff --git a/tests/test_req2flatpak.py b/tests/test_req2flatpak.py index 59e4292..ead2a7f 100644 --- a/tests/test_req2flatpak.py +++ b/tests/test_req2flatpak.py @@ -4,11 +4,18 @@ import tempfile import unittest from abc import ABC +from contextlib import contextmanager from itertools import product from pathlib import Path -from typing import List +from typing import Generator, List +from unittest import skipUnless from unittest.mock import patch +try: + import yaml +except ImportError: + yaml = None # type: ignore + from req2flatpak import ( DownloadChooser, FlatpakGenerator, @@ -81,6 +88,17 @@ def validate_build_module(self, build_module: dict) -> None: """To be implemented by subclasses.""" raise NotImplementedError + @contextmanager + def requirements_file( + self, + ) -> Generator[tempfile._TemporaryFileWrapper, None, None]: + """Create a temporary requirements file.""" + with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8") as req_file: + req_file.write("\n".join(self.requirements)) + req_file.flush() + req_file.seek(0) + yield req_file + def _run_r2f(self, args: List[str]) -> subprocess.CompletedProcess: """Runs req2flatpak's cli in a subprocess.""" cwd = Path(__file__).parent / ".." @@ -95,18 +113,36 @@ def test_cli_with_reqs_as_args(self): build_module = json.loads(result.stdout) self.validate_build_module(build_module) + @skipUnless(yaml, "The yaml extra dependency is needed for this feature.") + def test_cli_with_reqs_as_args_yaml(self): + """Runs req2flatpak in yaml mode by passing requirements as cmdline arg.""" + args = ["--requirements"] + self.requirements + args += ["--target-platforms"] + self.target_platforms + args += ["--yaml"] + result = self._run_r2f(args) + build_module = yaml.safe_load(result.stdout) + self.validate_build_module(build_module) + def test_cli_with_reqs_as_file(self): """Runs req2flatpak by passing requirements as requirements.txt file.""" - with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8") as req_file: - req_file.write("\n".join(self.requirements)) - req_file.flush() - req_file.seek(0) + with self.requirements_file() as req_file: args = ["--requirements-file", req_file.name] args += ["--target-platforms"] + self.target_platforms result = self._run_r2f(args) build_module = json.loads(result.stdout) self.validate_build_module(build_module) + @skipUnless(yaml, "The yaml extra dependency is needed for this feature.") + def test_cli_with_reqs_as_file_yaml(self): + """Runs req2flatpak by passing requirements as requirements.txt file.""" + with self.requirements_file() as req_file: + args = ["--requirements-file", req_file.name] + args += ["--target-platforms"] + self.target_platforms + args += ["--yaml"] + result = self._run_r2f(args) + build_module = yaml.safe_load(result.stdout) + self.validate_build_module(build_module) + def test_api(self): """Runs req2flatpak by calling its python api.""" platforms = [