From 8b239eb4d646ad98302936d0afeec9c1c3770750 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sat, 9 May 2020 13:38:06 -0700 Subject: [PATCH 1/6] =?UTF-8?q?Add=20primer=20CI=20tool=20=F0=9F=92=A9=20-?= =?UTF-8?q?=20Run=20in=20PATH=20`black`=20binary=20on=20configured=20proje?= =?UTF-8?q?cts=20-=20Can=20set=20wether=20we=20expect=20changes=20or=20not?= =?UTF-8?q?=20per=20project=20-=20Can=20set=20what=20python=20versions=20a?= =?UTF-8?q?re=20supported=20for=20a=20project=20-=20if=20`long=5Fcheckout`?= =?UTF-8?q?=20True=20project=20will=20not=20be=20ran=20on=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Will add to CI after I finish unit tests to avoid silly bugs I'm sure I have ðŸĪŠ Tests: - Manual Run - Will add unit tests if people think it will be useful - Output: ```shell (b) cooper-mbp1:black cooper$ time /tmp/b/bin/black-primer -k -w /tmp/cooper_primer_1 [2020-05-10 08:48:25,696] INFO: 4 projects to run black over (lib.py:212) [2020-05-10 08:48:25,697] INFO: Skipping aioexabgp as it's disabled via config (lib.py:166) [2020-05-10 08:48:25,699] INFO: Skipping bandersnatch as it's disabled via config (lib.py:166) [2020-05-10 08:48:28,676] INFO: Analyzing results (lib.py:225) -- primer results 📊 -- 2 / 4 succeeded (50.0%) ✅ 0 / 4 FAILED (0.0%) ðŸ’Đ - 2 projects Disabled by config - 0 projects skipped due to Python Version - 0 skipped due to long checkout real 0m3.304s user 0m9.529s sys 0m1.019s ``` - ls of /tmp/cooper_primer_1 ``` (b) cooper-mbp1:black cooper$ ls -lh /tmp/cooper_primer_1 total 0 drwxr-xr-x 21 cooper wheel 672B May 10 08:48 attrs drwxr-xr-x 14 cooper wheel 448B May 10 08:48 flake8-bugbear ``` --- setup.py | 3 +- src/black_primer/cli.py | 127 ++++++++++++++++++++ src/black_primer/lib.py | 226 +++++++++++++++++++++++++++++++++++ src/black_primer/primer.json | 37 ++++++ 4 files changed, 392 insertions(+), 1 deletion(-) create mode 100644 src/black_primer/cli.py create mode 100644 src/black_primer/lib.py create mode 100644 src/black_primer/primer.json diff --git a/setup.py b/setup.py index 44f83928e5c..bff439c6d8c 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,7 @@ def get_long_description() -> str: license="MIT", py_modules=["_black_version"], ext_modules=ext_modules, - packages=["blackd", "black", "blib2to3", "blib2to3.pgen2"], + packages=["blackd", "black", "blib2to3", "blib2to3.pgen2", "black_primer"], package_dir={"": "src"}, package_data={"blib2to3": ["*.txt"], "black": ["py.typed"]}, python_requires=">=3.6", @@ -102,6 +102,7 @@ def get_long_description() -> str: "console_scripts": [ "black=black:patched_main", "blackd=blackd:patched_main [d]", + "black-primer=black_primer.cli:main", ] }, ) diff --git a/src/black_primer/cli.py b/src/black_primer/cli.py new file mode 100644 index 00000000000..3f18cf6cc81 --- /dev/null +++ b/src/black_primer/cli.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 + +import asyncio +import logging +import sys +from os import cpu_count, getpid +from pathlib import Path +from shutil import rmtree, which +from tempfile import gettempdir +from typing import Any, Union + +import click + +from black_primer import lib + + +DEFAULT_CONFIG = Path(__file__).parent / "primer.json" +DEFAULT_WORKDIR = Path(gettempdir()) / f"primer.{getpid()}" +LOG = logging.getLogger(__name__) + + +def _handle_debug( + ctx: click.core.Context, + param: Union[click.core.Option, click.core.Parameter], + debug: Union[bool, int, str], +) -> Union[bool, int, str]: + """Turn on debugging if asked otherwise INFO default""" + log_level = logging.DEBUG if debug else logging.INFO + logging.basicConfig( + format="[%(asctime)s] %(levelname)s: %(message)s (%(filename)s:%(lineno)d)", + level=log_level, + ) + return debug + + +async def async_main( + config: str, + debug: bool, + keep: bool, + long_checkouts: bool, + rebase: bool, + workdir: str, + workers: int, +) -> int: + work_path = Path(workdir) + if not work_path.exists(): + LOG.debug(f"Creating {work_path}") + work_path.mkdir() + + if not which("black"): + LOG.error(f"Can not find 'black' executable in PATH. No point in running") + return -1 + + try: + return await lib.process_queue( + config, work_path, workers, keep, long_checkouts, rebase + ) + finally: + if not keep and work_path.exists(): + LOG.debug(f"Removing {work_path}") + rmtree(work_path) + + return -1 + + +@click.command(context_settings={"help_option_names": ["-h", "--help"]}) +@click.option( + "-c", + "--config", + default=str(DEFAULT_CONFIG), + type=click.Path(exists=True), + show_default=True, + help="JSON config file path", +) +@click.option( + "--debug", + is_flag=True, + callback=_handle_debug, + show_default=True, + help="Turn on debug logging", +) +@click.option( + "-k", + "--keep", + is_flag=True, + show_default=True, + help="Keep workdir + repos post run", +) +@click.option( + "-L", + "--long-checkouts", + is_flag=True, + show_default=True, + help="Pull big projects to test", +) +@click.option( + "-R", + "--rebase", + is_flag=True, + show_default=True, + help="Rebase project if already checked out", +) +@click.option( + "-w", + "--workdir", + default=str(DEFAULT_WORKDIR), + type=click.Path(exists=False), + show_default=True, + help="Directory Path for repo checkouts", +) +@click.option( + "-W", + "--workers", + default=int((cpu_count() or 4) / 2) or 1, + type=int, + show_default=True, + help="Number of parallel worker coroutines", +) +@click.pass_context +def main(ctx: click.core.Context, **kwargs: Any) -> None: + """primer - prime projects for blackening ... ðŸī""" + LOG.debug(f"Starting {sys.argv[0]}") + ctx.exit(asyncio.run(async_main(**kwargs))) + + +if __name__ == "__main__": + main() diff --git a/src/black_primer/lib.py b/src/black_primer/lib.py new file mode 100644 index 00000000000..aefe3a20d78 --- /dev/null +++ b/src/black_primer/lib.py @@ -0,0 +1,226 @@ +#!/usr/bin/env python3 + +import asyncio +import json +import logging +from pathlib import Path +from shutil import which +from subprocess import CalledProcessError +from sys import version_info +from typing import Dict, Optional, Sequence, Tuple, Union +from urllib.parse import urlparse + +import click + + +LOG = logging.getLogger(__name__) + + +async def _gen_check_output( + cmd: Sequence[str], + timeout: Union[int, float] = 30, + env: Optional[Dict[str, str]] = None, + cwd: Optional[Path] = None, +) -> Tuple[bytes, bytes]: + process = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + env=env, + cwd=cwd, + ) + try: + (stdout, stderr) = await asyncio.wait_for(process.communicate(), timeout) + except asyncio.TimeoutError: + process.kill() + await process.wait() + raise + + if process.returncode != 0: + cmd_str = " ".join(cmd) + raise CalledProcessError( + process.returncode, cmd_str, output=stdout, stderr=stderr + ) + + return (stdout, stderr) + + +async def analyze_results(project_count: int, results: Dict) -> int: + failed_pct = round(((results["failed"] / project_count) * 100), 2) + success_pct = round(((results["success"] / project_count) * 100), 2) + + click.secho(f"-- primer results 📊 --\n", bold=True) + click.secho( + f"{results['success']} / {project_count} succeeded ({success_pct}%) ✅", + bold=True, + fg="green", + ) + click.secho( + f"{results['failed']} / {project_count} FAILED ({failed_pct}%) ðŸ’Đ", + bold=bool(results["failed"]), + fg="red", + ) + click.echo(f" - {results['disabled']} projects Disabled by config") + click.echo(f" - {results['wrong_py_ver']} projects skipped due to Python Version ") + click.echo(f" - {results['skipped_long_checkout']} skipped due to long checkout") + + if results["projects"]: + click.secho(f"\nFailed Projects:\n", bold=True) + + for project_name, project_cpe in results["projects"].items(): + print(f"## {project_name}:") + print(f" - Returned {project_cpe.returncode}") + if project_cpe.stderr: + print(f" - stderr:\n{project_cpe.stderr.decode('utf8')}") + if project_cpe.stdout: + print(f" - stdout:\n{project_cpe.stdout.decode('utf8')}") + print("") + + return results["failed"] + + +async def black_run(repo_path: Path, project_config: Dict, results: Dict) -> None: + """Run black and record failures""" + cmd = [which("black")] + if project_config["cli_arguments"]: + cmd.extend(project_config["cli_arguments"]) + cmd.extend(["--check", "--diff", "."]) + + try: + _stdout, _stderr = await _gen_check_output(cmd, cwd=repo_path) + except asyncio.TimeoutError: + results["failed"] += 1 + LOG.error(f"Running black for {repo_path} timedout ({cmd})") + except CalledProcessError as cpe: + # TODO: This might need to be tuned and made smarter for higher signal + if not project_config["expect_formatting_changes"] and cpe.returncode == 1: + results["failed"] += 1 + results["projects"][repo_path.name] = cpe + return + + results["success"] += 1 + + +async def git_checkout_or_rebase( + work_path: Path, project_config: Dict, rebase: bool = False +) -> Optional[Path]: + """git Clone project or rebase""" + git_bin = which("git") + repo_url_parts = urlparse(project_config["git_clone_url"]) + path_parts = repo_url_parts.path[1:].split("/", maxsplit=1) + + repo_path = work_path / path_parts[1].replace(".git", "") + cmd = [git_bin, "clone", project_config["git_clone_url"]] + cwd = work_path + if repo_path.exists(): + cmd = [git_bin, "pull", "--rebase"] + cwd = repo_path + + try: + _stdout, _stderr = await _gen_check_output(cmd, cwd=cwd) + except (asyncio.TimeoutError, CalledProcessError) as e: + LOG.error(f"Unable to git clone / pull {project_config['git_clone_url']}: {e}") + return None + + return repo_path + + +async def load_projects_queue(config_path: Path) -> Tuple[Dict, asyncio.Queue]: + """Load project config and fill queue with all the project names""" + with config_path.open("r") as cfp: + config = json.load(cfp) + + # TODO: Offer more options here + # e.g. Run on X random packages or specific sub list etc. + project_names = sorted(config["projects"].keys()) + queue: asyncio.Queue = asyncio.Queue(maxsize=len(project_names)) + for project in project_names: + await queue.put(project) + + return config, queue + + +async def project_runner( + idx: int, + config: Dict, + queue: asyncio.Queue, + work_path: Path, + results: Dict, + long_checkouts: bool = False, + rebase: bool = False, +) -> None: + """Checkout project and run black on it + record result""" + py_version = f"{version_info[0]}.{version_info[1]}" + while True: + try: + project_name = queue.get_nowait() + except asyncio.QueueEmpty: + LOG.debug(f"project_runner {idx} exiting") + return + + project_config = config["projects"][project_name] + + # Check if disabled by config + if "disabled" in project_config and project_config["disabled"]: + results["disabled"] += 1 + LOG.info(f"Skipping {project_name} as it's disabled via config") + continue + + # Check if we should run on this version of Python + if ( + "all" not in project_config["py_versions"] + and py_version not in project_config["py_versions"] + ): + results["wrong_py_ver"] += 1 + LOG.debug(f"Skipping {project_name} as it's not enabled for {py_version}") + continue + + # Check if we're doing big projects / long checkouts + if not long_checkouts and project_config["long_checkout"]: + results["skipped_long_checkout"] += 1 + LOG.debug(f"Skipping {project_name} as it's configured as a long checkout") + continue + + repo_path = await git_checkout_or_rebase(work_path, project_config, rebase) + await black_run(repo_path, project_config, results) + + +async def process_queue( + config: str, + work_path: Path, + workers: int, + keep: bool = False, + long_checkouts: bool = False, + rebase: bool = False, +) -> int: + """ + Process the queue with X workers and evaluate results + - Success is guaged via the config "expect_formatting_changes" + + Integer return equals the number of failed projects + """ + results: Dict[str, Dict] = {} + results["disabled"] = 0 + results["failed"] = 0 + results["skipped_long_checkout"] = 0 + results["success"] = 0 + results["wrong_py_ver"] = 0 + results["projects"]: Dict[str, CalledProcessError] = {} + + config, queue = await load_projects_queue(Path(config)) + project_count = queue.qsize() + LOG.info(f"{project_count} projects to run black over") + if not project_count: + return -1 + + LOG.debug(f"Using {workers} parallel workers to run black") + # Wait until we finish running all the projects before analyzing + await asyncio.gather( + *[ + project_runner(i, config, queue, work_path, results, long_checkouts, rebase) + for i in range(workers) + ] + ) + + LOG.info("Analyzing results") + return await analyze_results(project_count, results) diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json new file mode 100644 index 00000000000..72db8c98184 --- /dev/null +++ b/src/black_primer/primer.json @@ -0,0 +1,37 @@ +{ + "configuration_format_version": 20200509, + "projects": { + "aioexabgp": { + "cli_arguments": [], + "disabled": true, + "disabled_reason": "TOT black hits a bug - Need to debug it - @cooperlees", + "expect_formatting_changes": false, + "git_clone_url": "https://github.com/cooperlees/aioexabgp.git", + "long_checkout": false, + "py_versions": ["all"] + }, + "attrs": { + "cli_arguments": [], + "expect_formatting_changes": true, + "git_clone_url": "https://github.com/python-attrs/attrs.git", + "long_checkout": false, + "py_versions": ["all"] + }, + "bandersnatch": { + "cli_arguments": [], + "disabled": true, + "disabled_reason": "TOT black hits a bug - Need to debug it - @cooperlees", + "expect_formatting_changes": false, + "git_clone_url": "https://github.com/pypa/bandersnatch.git", + "long_checkout": false, + "py_versions": ["all"] + }, + "flake8-bugbear": { + "cli_arguments": [], + "expect_formatting_changes": true, + "git_clone_url": "https://github.com/PyCQA/flake8-bugbear.git", + "long_checkout": false, + "py_versions": ["all"] + } + } +} From a29c80f38cb7a4fcf40115c3408f8879dd962072 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sun, 10 May 2020 09:12:22 -0700 Subject: [PATCH 2/6] Address mypy 3.6 type errors - Don't use asyncio.run() ... go back to the past :P - Refactor results into a named tuple of two dicts to avoid typing nightmare - Fix some variable names - Fix bug with rebase logic in git_checkout_or_rebase --- src/black_primer/cli.py | 10 +++- src/black_primer/lib.py | 106 ++++++++++++++++++++++++---------------- 2 files changed, 71 insertions(+), 45 deletions(-) diff --git a/src/black_primer/cli.py b/src/black_primer/cli.py index 3f18cf6cc81..65200c1f642 100644 --- a/src/black_primer/cli.py +++ b/src/black_primer/cli.py @@ -52,9 +52,10 @@ async def async_main( return -1 try: - return await lib.process_queue( + ret_val = await lib.process_queue( config, work_path, workers, keep, long_checkouts, rebase ) + return int(ret_val) finally: if not keep and work_path.exists(): LOG.debug(f"Removing {work_path}") @@ -120,7 +121,12 @@ async def async_main( def main(ctx: click.core.Context, **kwargs: Any) -> None: """primer - prime projects for blackening ... ðŸī""" LOG.debug(f"Starting {sys.argv[0]}") - ctx.exit(asyncio.run(async_main(**kwargs))) + # TODO: Change to asyncio.run when black >= 3.7 only + loop = asyncio.get_event_loop() + try: + ctx.exit(loop.run_until_complete(async_main(**kwargs))) + finally: + loop.close() if __name__ == "__main__": diff --git a/src/black_primer/lib.py b/src/black_primer/lib.py index aefe3a20d78..c858291fd8e 100644 --- a/src/black_primer/lib.py +++ b/src/black_primer/lib.py @@ -7,7 +7,7 @@ from shutil import which from subprocess import CalledProcessError from sys import version_info -from typing import Dict, Optional, Sequence, Tuple, Union +from typing import Any, Dict, NamedTuple, Optional, Sequence, Tuple from urllib.parse import urlparse import click @@ -16,9 +16,14 @@ LOG = logging.getLogger(__name__) +class Results(NamedTuple): + stats: Dict[str, int] = {} + failed_projects: Dict[str, CalledProcessError] = {} + + async def _gen_check_output( cmd: Sequence[str], - timeout: Union[int, float] = 30, + timeout: float = 30, env: Optional[Dict[str, str]] = None, cwd: Optional[Path] = None, ) -> Tuple[bytes, bytes]: @@ -45,29 +50,33 @@ async def _gen_check_output( return (stdout, stderr) -async def analyze_results(project_count: int, results: Dict) -> int: - failed_pct = round(((results["failed"] / project_count) * 100), 2) - success_pct = round(((results["success"] / project_count) * 100), 2) +async def analyze_results(project_count: int, results: Results) -> int: + failed_pct = round(((results.stats["failed"] / project_count) * 100), 2) + success_pct = round(((results.stats["success"] / project_count) * 100), 2) click.secho(f"-- primer results 📊 --\n", bold=True) click.secho( - f"{results['success']} / {project_count} succeeded ({success_pct}%) ✅", + f"{results.stats['success']} / {project_count} succeeded ({success_pct}%) ✅", bold=True, fg="green", ) click.secho( - f"{results['failed']} / {project_count} FAILED ({failed_pct}%) ðŸ’Đ", - bold=bool(results["failed"]), + f"{results.stats['failed']} / {project_count} FAILED ({failed_pct}%) ðŸ’Đ", + bold=bool(results.stats["failed"]), fg="red", ) - click.echo(f" - {results['disabled']} projects Disabled by config") - click.echo(f" - {results['wrong_py_ver']} projects skipped due to Python Version ") - click.echo(f" - {results['skipped_long_checkout']} skipped due to long checkout") + click.echo(f" - {results.stats['disabled']} projects Disabled by config") + click.echo( + f" - {results.stats['wrong_py_ver']} projects skipped due to Python Version" + ) + click.echo( + f" - {results.stats['skipped_long_checkout']} skipped due to long checkout" + ) - if results["projects"]: + if results.failed_projects: click.secho(f"\nFailed Projects:\n", bold=True) - for project_name, project_cpe in results["projects"].items(): + for project_name, project_cpe in results.failed_projects.items(): print(f"## {project_name}:") print(f" - Returned {project_cpe.returncode}") if project_cpe.stderr: @@ -76,45 +85,53 @@ async def analyze_results(project_count: int, results: Dict) -> int: print(f" - stdout:\n{project_cpe.stdout.decode('utf8')}") print("") - return results["failed"] + return results.stats["failed"] -async def black_run(repo_path: Path, project_config: Dict, results: Dict) -> None: +async def black_run( + repo_path: Path, project_config: Dict[str, Any], results: Results +) -> None: """Run black and record failures""" - cmd = [which("black")] + cmd = [str(which("black"))] if project_config["cli_arguments"]: - cmd.extend(project_config["cli_arguments"]) + cmd.extend(*project_config["cli_arguments"]) cmd.extend(["--check", "--diff", "."]) try: _stdout, _stderr = await _gen_check_output(cmd, cwd=repo_path) except asyncio.TimeoutError: - results["failed"] += 1 + results.stats["failed"] += 1 LOG.error(f"Running black for {repo_path} timedout ({cmd})") except CalledProcessError as cpe: # TODO: This might need to be tuned and made smarter for higher signal if not project_config["expect_formatting_changes"] and cpe.returncode == 1: - results["failed"] += 1 - results["projects"][repo_path.name] = cpe + results.stats["failed"] += 1 + results.failed_projects[repo_path.name] = cpe return - results["success"] += 1 + results.stats["success"] += 1 async def git_checkout_or_rebase( - work_path: Path, project_config: Dict, rebase: bool = False + work_path: Path, project_config: Dict[str, Any], rebase: bool = False ) -> Optional[Path]: """git Clone project or rebase""" - git_bin = which("git") + git_bin = str(which("git")) + if not git_bin: + LOG.error(f"No git binary found") + return None + repo_url_parts = urlparse(project_config["git_clone_url"]) path_parts = repo_url_parts.path[1:].split("/", maxsplit=1) - repo_path = work_path / path_parts[1].replace(".git", "") + repo_path: Path = work_path / path_parts[1].replace(".git", "") cmd = [git_bin, "clone", project_config["git_clone_url"]] cwd = work_path - if repo_path.exists(): + if repo_path.exists() and rebase: cmd = [git_bin, "pull", "--rebase"] cwd = repo_path + elif repo_path.exists(): + return repo_path try: _stdout, _stderr = await _gen_check_output(cmd, cwd=cwd) @@ -125,7 +142,9 @@ async def git_checkout_or_rebase( return repo_path -async def load_projects_queue(config_path: Path) -> Tuple[Dict, asyncio.Queue]: +async def load_projects_queue( + config_path: Path, +) -> Tuple[Dict[str, Any], asyncio.Queue[str]]: """Load project config and fill queue with all the project names""" with config_path.open("r") as cfp: config = json.load(cfp) @@ -133,7 +152,7 @@ async def load_projects_queue(config_path: Path) -> Tuple[Dict, asyncio.Queue]: # TODO: Offer more options here # e.g. Run on X random packages or specific sub list etc. project_names = sorted(config["projects"].keys()) - queue: asyncio.Queue = asyncio.Queue(maxsize=len(project_names)) + queue: asyncio.Queue[str] = asyncio.Queue(maxsize=len(project_names)) for project in project_names: await queue.put(project) @@ -142,10 +161,10 @@ async def load_projects_queue(config_path: Path) -> Tuple[Dict, asyncio.Queue]: async def project_runner( idx: int, - config: Dict, - queue: asyncio.Queue, + config: Dict[str, Any], + queue: asyncio.Queue[str], work_path: Path, - results: Dict, + results: Results, long_checkouts: bool = False, rebase: bool = False, ) -> None: @@ -162,7 +181,7 @@ async def project_runner( # Check if disabled by config if "disabled" in project_config and project_config["disabled"]: - results["disabled"] += 1 + results.stats["disabled"] += 1 LOG.info(f"Skipping {project_name} as it's disabled via config") continue @@ -171,22 +190,24 @@ async def project_runner( "all" not in project_config["py_versions"] and py_version not in project_config["py_versions"] ): - results["wrong_py_ver"] += 1 + results.stats["wrong_py_ver"] += 1 LOG.debug(f"Skipping {project_name} as it's not enabled for {py_version}") continue # Check if we're doing big projects / long checkouts if not long_checkouts and project_config["long_checkout"]: - results["skipped_long_checkout"] += 1 + results.stats["skipped_long_checkout"] += 1 LOG.debug(f"Skipping {project_name} as it's configured as a long checkout") continue repo_path = await git_checkout_or_rebase(work_path, project_config, rebase) + if not repo_path: + continue await black_run(repo_path, project_config, results) async def process_queue( - config: str, + config_file: str, work_path: Path, workers: int, keep: bool = False, @@ -199,15 +220,14 @@ async def process_queue( Integer return equals the number of failed projects """ - results: Dict[str, Dict] = {} - results["disabled"] = 0 - results["failed"] = 0 - results["skipped_long_checkout"] = 0 - results["success"] = 0 - results["wrong_py_ver"] = 0 - results["projects"]: Dict[str, CalledProcessError] = {} - - config, queue = await load_projects_queue(Path(config)) + results = Results() + results.stats["disabled"] = 0 + results.stats["failed"] = 0 + results.stats["skipped_long_checkout"] = 0 + results.stats["success"] = 0 + results.stats["wrong_py_ver"] = 0 + + config, queue = await load_projects_queue(Path(config_file)) project_count = queue.qsize() LOG.info(f"{project_count} projects to run black over") if not project_count: From 6a2c786bb768860626333ee814aa118df98dac84 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sun, 10 May 2020 09:55:36 -0700 Subject: [PATCH 3/6] Prettier the JSON config file for primer --- src/black_primer/primer.json | 68 ++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 72db8c98184..d08714337b7 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -1,37 +1,37 @@ { - "configuration_format_version": 20200509, - "projects": { - "aioexabgp": { - "cli_arguments": [], - "disabled": true, - "disabled_reason": "TOT black hits a bug - Need to debug it - @cooperlees", - "expect_formatting_changes": false, - "git_clone_url": "https://github.com/cooperlees/aioexabgp.git", - "long_checkout": false, - "py_versions": ["all"] - }, - "attrs": { - "cli_arguments": [], - "expect_formatting_changes": true, - "git_clone_url": "https://github.com/python-attrs/attrs.git", - "long_checkout": false, - "py_versions": ["all"] - }, - "bandersnatch": { - "cli_arguments": [], - "disabled": true, - "disabled_reason": "TOT black hits a bug - Need to debug it - @cooperlees", - "expect_formatting_changes": false, - "git_clone_url": "https://github.com/pypa/bandersnatch.git", - "long_checkout": false, - "py_versions": ["all"] - }, - "flake8-bugbear": { - "cli_arguments": [], - "expect_formatting_changes": true, - "git_clone_url": "https://github.com/PyCQA/flake8-bugbear.git", - "long_checkout": false, - "py_versions": ["all"] - } + "configuration_format_version": 20200509, + "projects": { + "aioexabgp": { + "cli_arguments": [], + "disabled": true, + "disabled_reason": "TOT black hits a bug - Need to debug it - @cooperlees", + "expect_formatting_changes": false, + "git_clone_url": "https://github.com/cooperlees/aioexabgp.git", + "long_checkout": false, + "py_versions": ["all"] + }, + "attrs": { + "cli_arguments": [], + "expect_formatting_changes": true, + "git_clone_url": "https://github.com/python-attrs/attrs.git", + "long_checkout": false, + "py_versions": ["all"] + }, + "bandersnatch": { + "cli_arguments": [], + "disabled": true, + "disabled_reason": "TOT black hits a bug - Need to debug it - @cooperlees", + "expect_formatting_changes": false, + "git_clone_url": "https://github.com/pypa/bandersnatch.git", + "long_checkout": false, + "py_versions": ["all"] + }, + "flake8-bugbear": { + "cli_arguments": [], + "expect_formatting_changes": true, + "git_clone_url": "https://github.com/PyCQA/flake8-bugbear.git", + "long_checkout": false, + "py_versions": ["all"] } + } } From cb4bce37730b145cd665ac0d3c14427a6c4159e0 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sat, 16 May 2020 12:26:31 -0700 Subject: [PATCH 4/6] Delete projects when finished, move dir to be timestamped + shallow copy --- src/black_primer/cli.py | 6 ++++-- src/black_primer/lib.py | 24 +++++++++++++++++++----- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/src/black_primer/cli.py b/src/black_primer/cli.py index 65200c1f642..010ea6c7af2 100644 --- a/src/black_primer/cli.py +++ b/src/black_primer/cli.py @@ -3,7 +3,8 @@ import asyncio import logging import sys -from os import cpu_count, getpid +from datetime import datetime +from os import cpu_count from pathlib import Path from shutil import rmtree, which from tempfile import gettempdir @@ -15,7 +16,8 @@ DEFAULT_CONFIG = Path(__file__).parent / "primer.json" -DEFAULT_WORKDIR = Path(gettempdir()) / f"primer.{getpid()}" +_timestamp = datetime.now().strftime("%Y%m%d%H%M%S") +DEFAULT_WORKDIR = Path(gettempdir()) / f"primer.{_timestamp}" LOG = logging.getLogger(__name__) diff --git a/src/black_primer/lib.py b/src/black_primer/lib.py index c858291fd8e..c808d43d46e 100644 --- a/src/black_primer/lib.py +++ b/src/black_primer/lib.py @@ -1,10 +1,12 @@ #!/usr/bin/env python3 +from __future__ import annotations + import asyncio import json import logging from pathlib import Path -from shutil import which +from shutil import rmtree, which from subprocess import CalledProcessError from sys import version_info from typing import Any, Dict, NamedTuple, Optional, Sequence, Tuple @@ -101,7 +103,7 @@ async def black_run( _stdout, _stderr = await _gen_check_output(cmd, cwd=repo_path) except asyncio.TimeoutError: results.stats["failed"] += 1 - LOG.error(f"Running black for {repo_path} timedout ({cmd})") + LOG.error(f"Running black for {repo_path} timed out ({cmd})") except CalledProcessError as cpe: # TODO: This might need to be tuned and made smarter for higher signal if not project_config["expect_formatting_changes"] and cpe.returncode == 1: @@ -113,7 +115,11 @@ async def black_run( async def git_checkout_or_rebase( - work_path: Path, project_config: Dict[str, Any], rebase: bool = False + work_path: Path, + project_config: Dict[str, Any], + rebase: bool = False, + *, + depth: int = 1, ) -> Optional[Path]: """git Clone project or rebase""" git_bin = str(which("git")) @@ -125,7 +131,7 @@ async def git_checkout_or_rebase( path_parts = repo_url_parts.path[1:].split("/", maxsplit=1) repo_path: Path = work_path / path_parts[1].replace(".git", "") - cmd = [git_bin, "clone", project_config["git_clone_url"]] + cmd = [git_bin, "clone", "--depth", str(depth), project_config["git_clone_url"]] cwd = work_path if repo_path.exists() and rebase: cmd = [git_bin, "pull", "--rebase"] @@ -167,8 +173,10 @@ async def project_runner( results: Results, long_checkouts: bool = False, rebase: bool = False, + keep: bool = False, ) -> None: """Checkout project and run black on it + record result""" + loop = asyncio.get_event_loop() py_version = f"{version_info[0]}.{version_info[1]}" while True: try: @@ -205,6 +213,10 @@ async def project_runner( continue await black_run(repo_path, project_config, results) + if not keep: + LOG.debug(f"Removing {repo_path}") + await loop.run_in_executor(None, rmtree, repo_path) + async def process_queue( config_file: str, @@ -237,7 +249,9 @@ async def process_queue( # Wait until we finish running all the projects before analyzing await asyncio.gather( *[ - project_runner(i, config, queue, work_path, results, long_checkouts, rebase) + project_runner( + i, config, queue, work_path, results, long_checkouts, rebase, keep + ) for i in range(workers) ] ) From 14503182e8a447c258a8d4a11d5c13467a840aad Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sat, 16 May 2020 13:31:03 -0700 Subject: [PATCH 5/6] Re-enable disabled projects post @JelleZijlstra's docstring fix --- src/black_primer/primer.json | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index d08714337b7..678942cc721 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -3,9 +3,7 @@ "projects": { "aioexabgp": { "cli_arguments": [], - "disabled": true, - "disabled_reason": "TOT black hits a bug - Need to debug it - @cooperlees", - "expect_formatting_changes": false, + "expect_formatting_changes": true, "git_clone_url": "https://github.com/cooperlees/aioexabgp.git", "long_checkout": false, "py_versions": ["all"] @@ -19,9 +17,7 @@ }, "bandersnatch": { "cli_arguments": [], - "disabled": true, - "disabled_reason": "TOT black hits a bug - Need to debug it - @cooperlees", - "expect_formatting_changes": false, + "expect_formatting_changes": true, "git_clone_url": "https://github.com/pypa/bandersnatch.git", "long_checkout": false, "py_versions": ["all"] From a5b300404bc85aa8a174f37c86b3c89a10386f20 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Sat, 16 May 2020 17:30:08 -0700 Subject: [PATCH 6/6] Workaround for future annotations until someone tells me the correct fix --- src/black_primer/lib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/black_primer/lib.py b/src/black_primer/lib.py index c808d43d46e..87028d72509 100644 --- a/src/black_primer/lib.py +++ b/src/black_primer/lib.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 -from __future__ import annotations +# Module '__future__' has no attribute 'annotations' +from __future__ import annotations # type: ignore import asyncio import json