From 8151df2cd7ebab8161baf057c5f3a52322225072 Mon Sep 17 00:00:00 2001 From: Nipunn Koorapati Date: Thu, 21 Oct 2021 17:18:07 -0700 Subject: [PATCH] Add --projects cli flag to black-primer Makes it possible to run a subset of projects on black primer --- CHANGES.md | 1 + src/black_primer/cli.py | 7 +++++++ src/black_primer/lib.py | 17 ++++++++++++++--- tests/test_format.py | 2 ++ tests/test_primer.py | 41 ++++++++++++++++++++++++++++++++++++++++- 5 files changed, 64 insertions(+), 4 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 2a3a60f82c6..a8307ee61ec 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,7 @@ - Add new `--workers` parameter (#2514) - Fixed feature detection for positional-only arguments in lambdas (#2532) - Bumped typed-ast version minimum to 1.4.3 for 3.10 compatiblity (#2519) +- Add primer support for --projects (#2555) ### _Blackd_ diff --git a/src/black_primer/cli.py b/src/black_primer/cli.py index 8360fc3c703..f924e8bf35c 100644 --- a/src/black_primer/cli.py +++ b/src/black_primer/cli.py @@ -48,11 +48,13 @@ async def async_main( keep: bool, long_checkouts: bool, no_diff: bool, + projects: str, rebase: bool, workdir: str, workers: int, ) -> int: work_path = Path(workdir) + projects_to_run = set(p for p in projects.split(",")) if projects else None if not work_path.exists(): LOG.debug(f"Creating {work_path}") work_path.mkdir() @@ -70,6 +72,7 @@ async def async_main( long_checkouts, rebase, no_diff, + projects_to_run, ) return int(ret_val) finally: @@ -116,6 +119,10 @@ async def async_main( show_default=True, help="Disable showing source file changes in black output", ) +@click.option( + "--projects", + help="Comma separated list of projects to run (Default: run all)", +) @click.option( "-R", "--rebase", diff --git a/src/black_primer/lib.py b/src/black_primer/lib.py index c7842797485..10fc92dfb9f 100644 --- a/src/black_primer/lib.py +++ b/src/black_primer/lib.py @@ -20,6 +20,7 @@ NamedTuple, Optional, Sequence, + Set, Tuple, Union, ) @@ -283,17 +284,26 @@ def handle_PermissionError( async def load_projects_queue( config_path: Path, + projects_to_run: Optional[Set[str]], ) -> Tuple[Dict[str, Any], asyncio.Queue]: """Load project config and fill queue with all the project names""" with config_path.open("r") as cfp: config = json.load(cfp) # TODO: Offer more options here - # e.g. Run on X random packages or specific sub list etc. + # e.g. Run on X random packages etc. project_names = sorted(config["projects"].keys()) + projects_to_run = projects_to_run or set(project_names) queue: asyncio.Queue = asyncio.Queue(maxsize=len(project_names)) for project in project_names: - await queue.put(project) + if project in projects_to_run: + await queue.put(project) + projects_to_run.remove(project) + + if projects_to_run: + LOG.error( + f"Project not found: {projects_to_run}. Available projects: {project_names}" + ) return config, queue @@ -369,6 +379,7 @@ async def process_queue( long_checkouts: bool = False, rebase: bool = False, no_diff: bool = False, + projects_to_run: Optional[Set[str]] = None, ) -> int: """ Process the queue with X workers and evaluate results @@ -383,7 +394,7 @@ async def process_queue( results.stats["success"] = 0 results.stats["wrong_py_ver"] = 0 - config, queue = await load_projects_queue(Path(config_file)) + config, queue = await load_projects_queue(Path(config_file), projects_to_run) project_count = queue.qsize() s = "" if project_count == 1 else "s" LOG.info(f"{project_count} project{s} to run Black over") diff --git a/tests/test_format.py b/tests/test_format.py index a659382092a..649c1572bee 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -93,6 +93,8 @@ "src/black/strings.py", "src/black/trans.py", "src/blackd/__init__.py", + "src/black_primer/cli.py", + "src/black_primer/lib.py", "src/blib2to3/pygram.py", "src/blib2to3/pytree.py", "src/blib2to3/pgen2/conv.py", diff --git a/tests/test_primer.py b/tests/test_primer.py index dc30a7a2244..9b168fb6ef0 100644 --- a/tests/test_primer.py +++ b/tests/test_primer.py @@ -11,7 +11,7 @@ from platform import system from subprocess import CalledProcessError from tempfile import TemporaryDirectory, gettempdir -from typing import Any, Callable, Iterator, Tuple +from typing import Any, Callable, Iterator, List, Tuple, TypeVar from unittest.mock import Mock, patch from click.testing import CliRunner @@ -89,6 +89,24 @@ async def return_zero(*args: Any, **kwargs: Any) -> int: return 0 +if sys.version_info >= (3, 9): + T = TypeVar("T") + Q = asyncio.Queue[T] +else: + T = Any + Q = asyncio.Queue + + +def collect(queue: Q) -> List[T]: + ret = [] + while True: + try: + item = queue.get_nowait() + ret.append(item) + except asyncio.QueueEmpty: + return ret + + class PrimerLibTests(unittest.TestCase): def test_analyze_results(self) -> None: fake_results = lib.Results( @@ -202,6 +220,26 @@ def test_process_queue(self, mock_stdout: Mock) -> None: ) self.assertEqual(0, return_val) + @event_loop() + def test_load_projects_queue(self) -> None: + """Test the process queue on primer itself + - If you have non black conforming formatting in primer itself this can fail""" + loop = asyncio.get_event_loop() + config_path = Path(lib.__file__).parent / "primer.json" + + config, projects_queue = loop.run_until_complete( + lib.load_projects_queue(config_path, None) + ) + projects = collect(projects_queue) + self.assertEqual(len(config["projects"].keys()), 22) + self.assertEqual(set(projects), set(config["projects"].keys())) + + config, projects_queue = loop.run_until_complete( + lib.load_projects_queue(config_path, set(["django", "pyramid", "nonsense"])) + ) + projects = collect(projects_queue) + self.assertEqual(projects, ["django", "pyramid"]) + class PrimerCLITests(unittest.TestCase): @event_loop() @@ -217,6 +255,7 @@ def test_async_main(self) -> None: "workdir": str(work_dir), "workers": 69, "no_diff": False, + "projects": "", } with patch("black_primer.cli.lib.process_queue", return_zero): return_val = loop.run_until_complete(cli.async_main(**args)) # type: ignore