diff --git a/src/black_primer/cli.py b/src/black_primer/cli.py index 3f18cf6cc81..65200c1f642 100644 --- a/src/black_primer/cli.py +++ b/src/black_primer/cli.py @@ -52,9 +52,10 @@ async def async_main( return -1 try: - return await lib.process_queue( + ret_val = await lib.process_queue( config, work_path, workers, keep, long_checkouts, rebase ) + return int(ret_val) finally: if not keep and work_path.exists(): LOG.debug(f"Removing {work_path}") @@ -120,7 +121,12 @@ async def async_main( def main(ctx: click.core.Context, **kwargs: Any) -> None: """primer - prime projects for blackening ... 🏴""" LOG.debug(f"Starting {sys.argv[0]}") - ctx.exit(asyncio.run(async_main(**kwargs))) + # TODO: Change to asyncio.run when black >= 3.7 only + loop = asyncio.get_event_loop() + try: + ctx.exit(loop.run_until_complete(async_main(**kwargs))) + finally: + loop.close() if __name__ == "__main__": diff --git a/src/black_primer/lib.py b/src/black_primer/lib.py index aefe3a20d78..bdd6d138e6c 100644 --- a/src/black_primer/lib.py +++ b/src/black_primer/lib.py @@ -7,7 +7,7 @@ from shutil import which from subprocess import CalledProcessError from sys import version_info -from typing import Dict, Optional, Sequence, Tuple, Union +from typing import Any, Dict, NamedTuple, Optional, Sequence, Tuple from urllib.parse import urlparse import click @@ -16,9 +16,14 @@ LOG = logging.getLogger(__name__) +class Results(NamedTuple): + stats: Dict[str, int] = {} + failed_projects: Dict[str, CalledProcessError] = {} + + async def _gen_check_output( cmd: Sequence[str], - timeout: Union[int, float] = 30, + timeout: float = 30, env: Optional[Dict[str, str]] = None, cwd: Optional[Path] = None, ) -> Tuple[bytes, bytes]: @@ -45,29 +50,33 @@ async def _gen_check_output( return (stdout, stderr) -async def analyze_results(project_count: int, results: Dict) -> int: - failed_pct = round(((results["failed"] / project_count) * 100), 2) - success_pct = round(((results["success"] / project_count) * 100), 2) +async def analyze_results(project_count: int, results: Results) -> int: + failed_pct = round(((results.stats["failed"] / project_count) * 100), 2) + success_pct = round(((results.stats["success"] / project_count) * 100), 2) click.secho(f"-- primer results 📊 --\n", bold=True) click.secho( - f"{results['success']} / {project_count} succeeded ({success_pct}%) ✅", + f"{results.stats['success']} / {project_count} succeeded ({success_pct}%) ✅", bold=True, fg="green", ) click.secho( - f"{results['failed']} / {project_count} FAILED ({failed_pct}%) 💩", - bold=bool(results["failed"]), + f"{results.stats['failed']} / {project_count} FAILED ({failed_pct}%) 💩", + bold=bool(results.stats["failed"]), fg="red", ) - click.echo(f" - {results['disabled']} projects Disabled by config") - click.echo(f" - {results['wrong_py_ver']} projects skipped due to Python Version ") - click.echo(f" - {results['skipped_long_checkout']} skipped due to long checkout") + click.echo(f" - {results.stats['disabled']} projects Disabled by config") + click.echo( + f" - {results.stats['wrong_py_ver']} projects skipped due to Python Version" + ) + click.echo( + f" - {results.stats['skipped_long_checkout']} skipped due to long checkout" + ) - if results["projects"]: + if results.failed_projects: click.secho(f"\nFailed Projects:\n", bold=True) - for project_name, project_cpe in results["projects"].items(): + for project_name, project_cpe in results.failed_projects.items(): print(f"## {project_name}:") print(f" - Returned {project_cpe.returncode}") if project_cpe.stderr: @@ -76,45 +85,51 @@ async def analyze_results(project_count: int, results: Dict) -> int: print(f" - stdout:\n{project_cpe.stdout.decode('utf8')}") print("") - return results["failed"] + return results.stats["failed"] -async def black_run(repo_path: Path, project_config: Dict, results: Dict) -> None: +async def black_run(repo_path: Path, project_config: Dict[str, Any], results: Results) -> None: """Run black and record failures""" - cmd = [which("black")] + cmd = [str(which("black"))] if project_config["cli_arguments"]: - cmd.extend(project_config["cli_arguments"]) + cmd.extend(*project_config["cli_arguments"]) cmd.extend(["--check", "--diff", "."]) try: _stdout, _stderr = await _gen_check_output(cmd, cwd=repo_path) except asyncio.TimeoutError: - results["failed"] += 1 + results.stats["failed"] += 1 LOG.error(f"Running black for {repo_path} timedout ({cmd})") except CalledProcessError as cpe: # TODO: This might need to be tuned and made smarter for higher signal if not project_config["expect_formatting_changes"] and cpe.returncode == 1: - results["failed"] += 1 - results["projects"][repo_path.name] = cpe + results.stats["failed"] += 1 + results.failed_projects[repo_path.name] = cpe return - results["success"] += 1 + results.stats["success"] += 1 async def git_checkout_or_rebase( - work_path: Path, project_config: Dict, rebase: bool = False + work_path: Path, project_config: Dict[str, Any], rebase: bool = False ) -> Optional[Path]: """git Clone project or rebase""" - git_bin = which("git") + git_bin = str(which("git")) + if not git_bin: + LOG.error(f"No git binary found") + return None + repo_url_parts = urlparse(project_config["git_clone_url"]) path_parts = repo_url_parts.path[1:].split("/", maxsplit=1) - repo_path = work_path / path_parts[1].replace(".git", "") + repo_path: Path = work_path / path_parts[1].replace(".git", "") cmd = [git_bin, "clone", project_config["git_clone_url"]] cwd = work_path - if repo_path.exists(): + if repo_path.exists() and rebase: cmd = [git_bin, "pull", "--rebase"] cwd = repo_path + elif repo_path.exists(): + return repo_path try: _stdout, _stderr = await _gen_check_output(cmd, cwd=cwd) @@ -125,7 +140,7 @@ async def git_checkout_or_rebase( return repo_path -async def load_projects_queue(config_path: Path) -> Tuple[Dict, asyncio.Queue]: +async def load_projects_queue(config_path: Path) -> Tuple[Dict[str, Any], asyncio.Queue[str]]: """Load project config and fill queue with all the project names""" with config_path.open("r") as cfp: config = json.load(cfp) @@ -133,7 +148,7 @@ async def load_projects_queue(config_path: Path) -> Tuple[Dict, asyncio.Queue]: # TODO: Offer more options here # e.g. Run on X random packages or specific sub list etc. project_names = sorted(config["projects"].keys()) - queue: asyncio.Queue = asyncio.Queue(maxsize=len(project_names)) + queue: asyncio.Queue[str] = asyncio.Queue(maxsize=len(project_names)) for project in project_names: await queue.put(project) @@ -142,10 +157,10 @@ async def load_projects_queue(config_path: Path) -> Tuple[Dict, asyncio.Queue]: async def project_runner( idx: int, - config: Dict, - queue: asyncio.Queue, + config: Dict[str, Any], + queue: asyncio.Queue[str], work_path: Path, - results: Dict, + results: Results, long_checkouts: bool = False, rebase: bool = False, ) -> None: @@ -162,7 +177,7 @@ async def project_runner( # Check if disabled by config if "disabled" in project_config and project_config["disabled"]: - results["disabled"] += 1 + results.stats["disabled"] += 1 LOG.info(f"Skipping {project_name} as it's disabled via config") continue @@ -171,22 +186,24 @@ async def project_runner( "all" not in project_config["py_versions"] and py_version not in project_config["py_versions"] ): - results["wrong_py_ver"] += 1 + results.stats["wrong_py_ver"] += 1 LOG.debug(f"Skipping {project_name} as it's not enabled for {py_version}") continue # Check if we're doing big projects / long checkouts if not long_checkouts and project_config["long_checkout"]: - results["skipped_long_checkout"] += 1 + results.stats["skipped_long_checkout"] += 1 LOG.debug(f"Skipping {project_name} as it's configured as a long checkout") continue repo_path = await git_checkout_or_rebase(work_path, project_config, rebase) + if not repo_path: + continue await black_run(repo_path, project_config, results) async def process_queue( - config: str, + config_file: str, work_path: Path, workers: int, keep: bool = False, @@ -199,15 +216,14 @@ async def process_queue( Integer return equals the number of failed projects """ - results: Dict[str, Dict] = {} - results["disabled"] = 0 - results["failed"] = 0 - results["skipped_long_checkout"] = 0 - results["success"] = 0 - results["wrong_py_ver"] = 0 - results["projects"]: Dict[str, CalledProcessError] = {} - - config, queue = await load_projects_queue(Path(config)) + results = Results() + results.stats["disabled"] = 0 + results.stats["failed"] = 0 + results.stats["skipped_long_checkout"] = 0 + results.stats["success"] = 0 + results.stats["wrong_py_ver"] = 0 + + config, queue = await load_projects_queue(Path(config_file)) project_count = queue.qsize() LOG.info(f"{project_count} projects to run black over") if not project_count: