Skip to content

Commit

Permalink
Add yaml output (#41)
Browse files Browse the repository at this point in the history 
Adds yaml output. 

* Write yaml if output file ends in yaml or yml extension
* Writes yaml if a "--yaml" cmdline arg was provided
* Adds test for yaml output
* Adds ci tests with a matrix of extra dependencies.
* Adds yaml dependency: `pyyaml` as an extra dependency in pyproject.toml
* Adds yaml dependeny:  `types-pyyaml` as a lint dev dependency.
* Makes the indentation of .json output consistent

Co-authored-by: yfprojects <62463991+real-yfprojects@users.noreply.github.com>
Co-authored-by: real-yfprojects <real-yfprojects@users.noreply.github.com>
  • Loading branch information
@cbm755 @real-yfprojects
3 people committed Mar 18, 2023
1 parent e0183fd commit bfa9f75
Show file tree
Hide file tree
Showing 5 changed files with 172 additions and 38 deletions.
35 changes: 34 additions & 1 deletion .github/workflows/ci.yml
View file
Expand Up @@ -23,8 +23,41 @@ jobs:
- name: Run linters
run: make lint

generate_test_matrix:
runs-on: ubuntu-latest
outputs:
matrix: ${{ steps.set-matrix.outputs.matrix }}
steps:
- name: Checkout
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0

- name: setup python
uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 # v4.5.0
with:
python-version: "3.11"

- name: Extract extras from `pyproject.toml`
id: set-matrix
shell: python
run: |
import tomllib
import os
import json
with open('pyproject.toml', 'rb') as f:
manifest = tomllib.load(f)
yaml = { 'include' : [{ 'extras' : extra} for extra in [''] + list(manifest['tool']['poetry']['extras'])]}
out = json.dumps(yaml)
print(out)
with open(os.environ['GITHUB_OUTPUT'], 'a') as f:
f.write('matrix=' + out)
test:
name: test ${{ matrix.extras && 'with' || '' }} ${{ matrix.extras }}
runs-on: ubuntu-latest
needs: generate_test_matrix
strategy:
matrix: ${{ fromJson(needs.generate_test_matrix.outputs.matrix) }}
fail-fast: false
steps:
- name: Checkout
uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0
Expand All @@ -33,7 +66,7 @@ jobs:
id: setup
uses: ./.github/actions/setup
with:
install-options: --without lint
install-options: --without lint ${{ matrix.extras && format('--extras "{0}"', matrix.extras) || '' }}

- name: Run Tests
run: make test
Expand Down
17 changes: 15 additions & 2 deletions poetry.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

35 changes: 20 additions & 15 deletions pyproject.toml
View file
@@ -1,24 +1,26 @@
[build-system]
requires = ["poetry-core"]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

[tool.poetry]
name = "req2flatpak"
version = "0.1.0"
name = "req2flatpak"
version = "0.1.0"
description = "Generates a flatpak-builder build module for installing python packages defined in requirements.txt files."
authors = ["johannesjh <johannesjh@users.noreply.github.com>"]
license = "MIT"
readme = "README.rst"
authors = ["johannesjh <johannesjh@users.noreply.github.com>"]
license = "MIT"
readme = "README.rst"

[tool.poetry.scripts]
req2flatpak = 'req2flatpak:main'

[tool.poetry.dependencies]
python = "^3.7.2"
python = "^3.7.2"
packaging = { version = "^21.3", optional = true }
pyyaml = { version = "^6.0", optional = true }

[tool.poetry.extras]
packaging = ["packaging"]
yaml = ["pyyaml"]

[tool.poetry.group.lint.dependencies]
pylama = { extras = [
Expand All @@ -28,7 +30,10 @@ pylama = { extras = [
"toml",
], version = "^8.4.1" }
bandit = { extras = ["toml"], version = "^1.7.4" }
types-setuptools = "^65.5.0.2" # type stubs for mypy linting

# type stubs for mypy linting
types-setuptools = "^65.5.0.2"
types-pyyaml = "^6.0.12.8"

# [tool.poetry.group.tests.dependencies]
pydocstyle = "<6.2"
Expand All @@ -37,18 +42,18 @@ pydocstyle = "<6.2"
optional = true

[tool.poetry.group.docs.dependencies]
sphinx = "^5.3.0"
sphinx-argparse = "^0.3.2"
sphinx = "^5.3.0"
sphinx-argparse = "^0.3.2"
sphinx-rtd-theme-github-versions = "^1.1"

[tool.isort]
profile = "black"
profile = "black"
skip_gitignore = true

[tool.pylama]
max_line_length = 88
concurrent = true
linters = "pycodestyle,pydocstyle,pyflakes,pylint,eradicate,mypy"
concurrent = true
linters = "pycodestyle,pydocstyle,pyflakes,pylint,eradicate,mypy"

[tool.pylama.linter.pycodestyle]
ignore = "W503,E203,E501"
Expand All @@ -63,8 +68,8 @@ ignore = ["D202", "D203", "D205", "D401", "D212"]

[tool.pylint]
format.max-line-length = 88
main.disable = ["C0301"] # ignore line-too-long
basic.good-names = ["e", "f", "py"]
main.disable = ["C0301"] # ignore line-too-long
basic.good-names = ["e", "f", "py"]

[tool.bandit]
exclude_dirs = ['tests']
77 changes: 62 additions & 15 deletions req2flatpak.py
View file
Expand Up @@ -64,6 +64,10 @@

logger = logging.getLogger(__name__)

try:
import yaml
except ImportError:
yaml = None # type: ignore

# =============================================================================
# Helper functions / semi vendored code
Expand All @@ -73,7 +77,6 @@
# added with py 3.8
from functools import cached_property # type: ignore[attr-defined]
except ImportError:

# Inspired by the implementation in the standard library
# pylint: disable=invalid-name,too-few-public-methods
class cached_property: # type: ignore[no-redef]
Expand Down Expand Up @@ -551,9 +554,7 @@ def downloads(
preferred downloads are returned first.
"""
cache = set()
for (platform_tag, download) in product(
platform.python_tags, release.downloads
):
for platform_tag, download in product(platform.python_tags, release.downloads):
if download in cache:
continue
if wheels_only and not download.is_wheel:
Expand Down Expand Up @@ -634,12 +635,30 @@ def sources(downloads: Iterable[Download]) -> List[dict]:
@classmethod
def build_module_as_str(cls, *args, **kwargs) -> str:
"""
Generates a build module for inclusion in a flatpak-builder build manifest.
Generate JSON build module for inclusion in a flatpak-builder build manifest.
The args and kwargs are the same as in
:py:meth:`~req2flatpak.FlatpakGenerator.build_module`
"""
return json.dumps(cls.build_module(*args, **kwargs), indent=4)

@classmethod
def build_module_as_yaml_str(cls, *args, **kwargs) -> str:
"""
Generate YAML build module for inclusion in a flatpak-builder build manifest.
The args and kwargs are the same as in
:py:meth:`~req2flatpak.FlatpakGenerator.build_module`
"""
return json.dumps(cls.build_module(*args, **kwargs), indent=2)
# optional dependency, not imported at top
if not yaml:
raise ImportError(
"Package `pyyaml` has to be installed for the yaml format."
)

return yaml.dump(
cls.build_module(*args, **kwargs), default_flow_style=False, sort_keys=False
)


# =============================================================================
Expand Down Expand Up @@ -676,12 +695,22 @@ def cli_parser() -> argparse.ArgumentParser:
default=False,
help="Uses a persistent cache when querying pypi.",
)
parser.add_argument(
"--yaml",
action="store_true",
help="Write YAML instead of the default JSON. Needs the 'pyyaml' package.",
)

parser.add_argument(
"--outfile",
"-o",
nargs="?",
type=argparse.FileType("w"),
default=sys.stdout,
help="""
By default, writes JSON but specify a '.yaml' extension and YAML
will be written instead, provided you have the 'pyyaml' package.
""",
)
parser.add_argument(
"--platform-info",
Expand All @@ -698,21 +727,33 @@ def cli_parser() -> argparse.ArgumentParser:
return parser


def main():
def main(): # pylint: disable=too-many-branches
"""Main function that provides req2flatpak's commandline interface."""

# process commandline arguments
parser = cli_parser()
options = parser.parse_args()

# stream output to a file or to stdout
output_stream = options.outfile if hasattr(options.outfile, "write") else sys.stdout
if hasattr(options.outfile, "write"):
output_stream = options.outfile
if pathlib.Path(output_stream.name).suffix.casefold() in (".yaml", ".yml"):
options.yaml = True
else:
output_stream = sys.stdout

if options.yaml and not yaml:
parser.error(
"Outputing YAML requires 'pyyaml' package: try 'pip install pyyaml'"
)

# print platform info if requested, and exit
if options.platform_info:
json.dump(
asdict(PlatformFactory.from_current_interpreter()), output_stream, indent=4
)
info = asdict(PlatformFactory.from_current_interpreter())
if options.yaml:
yaml.dump(info, output_stream, default_flow_style=False, sort_keys=False)
else:
json.dump(info, output_stream, indent=4)
parser.exit()

# print installed packages if requested, and exit
Expand Down Expand Up @@ -770,10 +811,16 @@ def main():
}

# generate flatpak-builder build module
build_module = FlatpakGenerator.build_module(requirements, downloads)

# write output
json.dump(build_module, output_stream, indent=4)
if options.yaml:
# write yaml
output_stream.write(
FlatpakGenerator.build_module_as_yaml_str(requirements, downloads)
)
else:
# write json
output_stream.write(
FlatpakGenerator.build_module_as_str(requirements, downloads)
)


if __name__ == "__main__":
Expand Down
46 changes: 41 additions & 5 deletions tests/test_req2flatpak.py
View file
Expand Up @@ -4,11 +4,18 @@
import tempfile
import unittest
from abc import ABC
from contextlib import contextmanager
from itertools import product
from pathlib import Path
from typing import List
from typing import Generator, List
from unittest import skipUnless
from unittest.mock import patch

try:
import yaml
except ImportError:
yaml = None # type: ignore

from req2flatpak import (
DownloadChooser,
FlatpakGenerator,
Expand Down Expand Up @@ -81,6 +88,17 @@ def validate_build_module(self, build_module: dict) -> None:
"""To be implemented by subclasses."""
raise NotImplementedError

@contextmanager
def requirements_file(
self,
) -> Generator[tempfile._TemporaryFileWrapper, None, None]:
"""Create a temporary requirements file."""
with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8") as req_file:
req_file.write("\n".join(self.requirements))
req_file.flush()
req_file.seek(0)
yield req_file

def _run_r2f(self, args: List[str]) -> subprocess.CompletedProcess:
"""Runs req2flatpak's cli in a subprocess."""
cwd = Path(__file__).parent / ".."
Expand All @@ -95,18 +113,36 @@ def test_cli_with_reqs_as_args(self):
build_module = json.loads(result.stdout)
self.validate_build_module(build_module)

@skipUnless(yaml, "The yaml extra dependency is needed for this feature.")
def test_cli_with_reqs_as_args_yaml(self):
"""Runs req2flatpak in yaml mode by passing requirements as cmdline arg."""
args = ["--requirements"] + self.requirements
args += ["--target-platforms"] + self.target_platforms
args += ["--yaml"]
result = self._run_r2f(args)
build_module = yaml.safe_load(result.stdout)
self.validate_build_module(build_module)

def test_cli_with_reqs_as_file(self):
"""Runs req2flatpak by passing requirements as requirements.txt file."""
with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8") as req_file:
req_file.write("\n".join(self.requirements))
req_file.flush()
req_file.seek(0)
with self.requirements_file() as req_file:
args = ["--requirements-file", req_file.name]
args += ["--target-platforms"] + self.target_platforms
result = self._run_r2f(args)
build_module = json.loads(result.stdout)
self.validate_build_module(build_module)

@skipUnless(yaml, "The yaml extra dependency is needed for this feature.")
def test_cli_with_reqs_as_file_yaml(self):
"""Runs req2flatpak by passing requirements as requirements.txt file."""
with self.requirements_file() as req_file:
args = ["--requirements-file", req_file.name]
args += ["--target-platforms"] + self.target_platforms
args += ["--yaml"]
result = self._run_r2f(args)
build_module = yaml.safe_load(result.stdout)
self.validate_build_module(build_module)

def test_api(self):
"""Runs req2flatpak by calling its python api."""
platforms = [
Expand Down

0 comments on commit bfa9f75

Please sign in to comment.