From 00491e1dcb19a795867249471ca5bb95efcd8cd3 Mon Sep 17 00:00:00 2001 From: Kaleb Barrett Date: Sat, 1 May 2021 13:47:59 -0500 Subject: [PATCH 01/19] Add ability to pass posargs to pytest run in tox.ini (#2173) --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 71b02e920c7..affc3c9876a 100644 --- a/tox.ini +++ b/tox.ini @@ -9,7 +9,7 @@ deps = commands = pip install -e .[d,python2] coverage erase - coverage run -m pytest tests + coverage run -m pytest tests {posargs} coverage report [testenv:fuzz] From 24bd6b983ac459eabd9352eb816be53e7f447812 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Sat, 1 May 2021 22:17:20 +0300 Subject: [PATCH 02/19] Tox has been formatted with Black 21.4b0 (#2175) --- src/black_primer/primer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 75469ad815b..5017a5625a3 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -112,7 +112,7 @@ }, "tox": { "cli_arguments": [], - "expect_formatting_changes": true, + "expect_formatting_changes": false, "git_clone_url": "https://github.com/tox-dev/tox.git", "long_checkout": false, "py_versions": ["all"] From 35e8d1560d68e8113ff926a9f832582f8f4a694f Mon Sep 17 00:00:00 2001 From: Bryan Forbes Date: Sun, 2 May 2021 07:48:54 -0500 Subject: [PATCH 03/19] Set `is_pyi` if `stdin_filename` ends with `.pyi` (#2169) Fixes #2167 Co-authored-by: Jelle Zijlstra --- CHANGES.md | 6 ++++++ src/black/__init__.py | 2 ++ tests/test_black.py | 32 +++++++++++++++++++++++++++++--- 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6675c484c75..321f8c083b2 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,5 +1,11 @@ ## Change Log +### Unreleased + +#### _Black_ + +- Set `--pyi` mode if `--stdin-filename` ends in `.pyi` (#2169) + ### 21.4b2 #### _Black_ diff --git a/src/black/__init__.py b/src/black/__init__.py index 1c69cc41cdc..49d088b531d 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -755,6 +755,8 @@ def reformat_one( is_stdin = False if is_stdin: + if src.suffix == ".pyi": + mode = replace(mode, is_pyi=True) if format_stdin_to_stdout(fast=fast, write_back=write_back, mode=mode): changed = Changed.YES else: diff --git a/tests/test_black.py b/tests/test_black.py index c643f27cacf..43368d4bbe9 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1578,8 +1578,34 @@ def test_reformat_one_with_stdin_filename(self) -> None: mode=DEFAULT_MODE, report=report, ) - fsts.assert_called_once() - # __BLACK_STDIN_FILENAME__ should have been striped + fsts.assert_called_once_with( + fast=True, write_back=black.WriteBack.YES, mode=DEFAULT_MODE + ) + # __BLACK_STDIN_FILENAME__ should have been stripped + report.done.assert_called_with(expected, black.Changed.YES) + + def test_reformat_one_with_stdin_filename_pyi(self) -> None: + with patch( + "black.format_stdin_to_stdout", + return_value=lambda *args, **kwargs: black.Changed.YES, + ) as fsts: + report = MagicMock() + p = "foo.pyi" + path = Path(f"__BLACK_STDIN_FILENAME__{p}") + expected = Path(p) + black.reformat_one( + path, + fast=True, + write_back=black.WriteBack.YES, + mode=DEFAULT_MODE, + report=report, + ) + fsts.assert_called_once_with( + fast=True, + write_back=black.WriteBack.YES, + mode=replace(DEFAULT_MODE, is_pyi=True), + ) + # __BLACK_STDIN_FILENAME__ should have been stripped report.done.assert_called_with(expected, black.Changed.YES) def test_reformat_one_with_stdin_and_existing_path(self) -> None: @@ -1603,7 +1629,7 @@ def test_reformat_one_with_stdin_and_existing_path(self) -> None: report=report, ) fsts.assert_called_once() - # __BLACK_STDIN_FILENAME__ should have been striped + # __BLACK_STDIN_FILENAME__ should have been stripped report.done.assert_called_with(expected, black.Changed.YES) def test_gitignore_exclude(self) -> None: From a669b64091c149132f4e234abd230f7b33d692e8 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Mon, 3 May 2021 14:58:17 -0700 Subject: [PATCH 04/19] primer: Renable pandas (#2185) - It no longer crashes black so we should test on it's code - Update django reason to name the file causing error - Seems it has a syntax error on purpose --- src/black_primer/primer.json | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 5017a5625a3..137d075c68e 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -30,7 +30,7 @@ "py_versions": ["all"] }, "django": { - "disabled_reason": "black --check --diff returned 123 on two files", + "disabled_reason": "black --check --diff returned 123 on tests_syntax_error.py", "disabled": true, "cli_arguments": [], "expect_formatting_changes": true, @@ -53,8 +53,6 @@ "py_versions": ["all"] }, "pandas": { - "disabled_reason": "black --check --diff returned 123 on one file", - "disabled": true, "cli_arguments": [], "expect_formatting_changes": true, "git_clone_url": "https://github.com/pandas-dev/pandas.git", From a18c7bc09942951e93cbd142fb7384aa2af18951 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Tue, 4 May 2021 01:44:40 -0700 Subject: [PATCH 05/19] primer: Add `--no-diff` option (#2187) - Allow runs with no code diff output - This is handy for reducing output to see which file is erroring Test: - Edit config for 'channels' to expect no changes and run with `--no-diff` and see no diff output --- CHANGES.md | 1 + src/black_primer/cli.py | 15 ++++++++++++++- src/black_primer/lib.py | 25 +++++++++++++++++++++---- tests/test_primer.py | 1 + 4 files changed, 37 insertions(+), 5 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 321f8c083b2..7741d92772d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -5,6 +5,7 @@ #### _Black_ - Set `--pyi` mode if `--stdin-filename` ends in `.pyi` (#2169) +- Add `--no-diff` to black-primer to suppress formatting changes (#2187) ### 21.4b2 diff --git a/src/black_primer/cli.py b/src/black_primer/cli.py index b2d4159b1da..00b54d6511f 100644 --- a/src/black_primer/cli.py +++ b/src/black_primer/cli.py @@ -39,6 +39,7 @@ async def async_main( debug: bool, keep: bool, long_checkouts: bool, + no_diff: bool, rebase: bool, workdir: str, workers: int, @@ -54,7 +55,13 @@ async def async_main( try: ret_val = await lib.process_queue( - config, work_path, workers, keep, long_checkouts, rebase + config, + work_path, + workers, + keep, + long_checkouts, + rebase, + no_diff, ) return int(ret_val) finally: @@ -95,6 +102,12 @@ async def async_main( show_default=True, help="Pull big projects to test", ) +@click.option( + "--no-diff", + is_flag=True, + show_default=True, + help="Disable showing source file changes in black output", +) @click.option( "-R", "--rebase", diff --git a/src/black_primer/lib.py b/src/black_primer/lib.py index ecc704f2cca..aba694b0e60 100644 --- a/src/black_primer/lib.py +++ b/src/black_primer/lib.py @@ -112,13 +112,20 @@ def analyze_results(project_count: int, results: Results) -> int: async def black_run( - repo_path: Path, project_config: Dict[str, Any], results: Results + repo_path: Path, + project_config: Dict[str, Any], + results: Results, + no_diff: bool = False, ) -> None: """Run Black and record failures""" cmd = [str(which(BLACK_BINARY))] if "cli_arguments" in project_config and project_config["cli_arguments"]: cmd.extend(*project_config["cli_arguments"]) - cmd.extend(["--check", "--diff", "."]) + cmd.append("--check") + if no_diff: + cmd.append(".") + else: + cmd.extend(["--diff", "."]) with TemporaryDirectory() as tmp_path: # Prevent reading top-level user configs by manipulating envionment variables @@ -246,6 +253,7 @@ async def project_runner( long_checkouts: bool = False, rebase: bool = False, keep: bool = False, + no_diff: bool = False, ) -> None: """Check out project and run Black on it + record result""" loop = asyncio.get_event_loop() @@ -284,7 +292,7 @@ async def project_runner( repo_path = await git_checkout_or_rebase(work_path, project_config, rebase) if not repo_path: continue - await black_run(repo_path, project_config, results) + await black_run(repo_path, project_config, results, no_diff) if not keep: LOG.debug(f"Removing {repo_path}") @@ -303,6 +311,7 @@ async def process_queue( keep: bool = False, long_checkouts: bool = False, rebase: bool = False, + no_diff: bool = False, ) -> int: """ Process the queue with X workers and evaluate results @@ -330,7 +339,15 @@ async def process_queue( await asyncio.gather( *[ project_runner( - i, config, queue, work_path, results, long_checkouts, rebase, keep + i, + config, + queue, + work_path, + results, + long_checkouts, + rebase, + keep, + no_diff, ) for i in range(workers) ] diff --git a/tests/test_primer.py b/tests/test_primer.py index a8ad8a7c5af..8bfecd61a57 100644 --- a/tests/test_primer.py +++ b/tests/test_primer.py @@ -198,6 +198,7 @@ def test_async_main(self) -> None: "rebase": False, "workdir": str(work_dir), "workers": 69, + "no_diff": False, } with patch("black_primer.cli.lib.process_queue", return_zero): return_val = loop.run_until_complete(cli.async_main(**args)) From e42f9921e291790202147c1e3dc1a3df7036c652 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Tue, 4 May 2021 04:46:46 -0400 Subject: [PATCH 06/19] Detect `'@' dotted_name '(' ')' NEWLINE` as a simple decorator (#2182) Previously the RELAXED_DECORATOR detection would be falsely True on that example. The problem was that an argument-less parentheses pair didn't pass the `is_simple_decorator_trailer` check even it should. OTOH a parentheses pair containing an argument or more passed as expected. --- CHANGES.md | 3 +++ src/black/__init__.py | 7 +++++++ tests/data/decorators.py | 6 ++++++ 3 files changed, 16 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index 7741d92772d..00d6782dc6b 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,9 @@ - Set `--pyi` mode if `--stdin-filename` ends in `.pyi` (#2169) - Add `--no-diff` to black-primer to suppress formatting changes (#2187) +- Stop detecting target version as Python 3.9+ with pre-PEP-614 decorators that are + being called but with no arguments (#2182) + ### 21.4b2 #### _Black_ diff --git a/src/black/__init__.py b/src/black/__init__.py index 49d088b531d..cf257876c03 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -5761,6 +5761,13 @@ def is_simple_decorator_trailer(node: LN, last: bool = False) -> bool: and node.children[0].type == token.DOT and node.children[1].type == token.NAME ) + # last trailer can be an argument-less parentheses pair + or ( + last + and len(node.children) == 2 + and node.children[0].type == token.LPAR + and node.children[1].type == token.RPAR + ) # last trailer can be arguments or ( last diff --git a/tests/data/decorators.py b/tests/data/decorators.py index acfad51fcb8..a0f38ca7b9d 100644 --- a/tests/data/decorators.py +++ b/tests/data/decorators.py @@ -13,6 +13,12 @@ def f(): ## +@decorator() +def f(): + ... + +## + @decorator(arg) def f(): ... From 204f76e0c02a12aeccb3ab708fe41f7e95435a29 Mon Sep 17 00:00:00 2001 From: KotlinIsland <65446343+KotlinIsland@users.noreply.github.com> Date: Tue, 4 May 2021 18:47:22 +1000 Subject: [PATCH 07/19] add test configurations that don't contain python2 optional install (#2190) add test for negative scenario: formatting python2 code tag python2 only tests Co-authored-by: KotlinIsland --- pyproject.toml | 3 +++ tests/test_black.py | 30 ++++++++++++++++++++++++++++++ tests/test_format.py | 16 +++++++++++++--- tox.ini | 6 ++++-- 4 files changed, 50 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 7f632f2839d..ca75f8f92ef 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,3 +25,6 @@ extend-exclude = ''' [build-system] requires = ["setuptools>=41.0", "setuptools-scm", "wheel"] build-backend = "setuptools.build_meta" + +[tool.pytest.ini_options] +markers = ['python2', "without_python2"] \ No newline at end of file diff --git a/tests/test_black.py b/tests/test_black.py index 43368d4bbe9..7d855cab3d3 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -24,6 +24,7 @@ Iterator, TypeVar, ) +import pytest import unittest from unittest.mock import patch, MagicMock @@ -459,6 +460,34 @@ def test_skip_magic_trailing_comma(self) -> None: ) self.assertEqual(expected, actual, msg) + @pytest.mark.without_python2 + def test_python2_should_fail_without_optional_install(self) -> None: + # python 3.7 and below will install typed-ast and will be able to parse Python 2 + if sys.version_info < (3, 8): + return + source = "x = 1234l" + tmp_file = Path(black.dump_to_file(source)) + try: + runner = BlackRunner() + result = runner.invoke(black.main, [str(tmp_file)]) + self.assertEqual(result.exit_code, 123) + finally: + os.unlink(tmp_file) + actual = ( + runner.stderr_bytes.decode() + .replace("\n", "") + .replace("\\n", "") + .replace("\\r", "") + .replace("\r", "") + ) + msg = ( + "The requested source code has invalid Python 3 syntax." + "If you are trying to format Python 2 files please reinstall Black" + " with the 'python2' extra: `python3 -m pip install black[python2]`." + ) + self.assertIn(msg, actual) + + @pytest.mark.python2 @patch("black.dump_to_file", dump_to_stderr) def test_python2_print_function(self) -> None: source, expected = read_data("python2_print_function") @@ -1971,6 +2000,7 @@ def test_bpo_2142_workaround(self) -> None: actual = diff_header.sub(DETERMINISTIC_HEADER, actual) self.assertEqual(actual, expected) + @pytest.mark.python2 def test_docstring_reformat_for_py27(self) -> None: """ Check that stripping trailing whitespace from Python 2 docstrings diff --git a/tests/test_format.py b/tests/test_format.py index e1335aedf43..78f2b558a4f 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -1,6 +1,7 @@ from unittest.mock import patch import black +import pytest from parameterized import parameterized from tests.util import ( @@ -46,9 +47,6 @@ "function2", "function_trailing_comma", "import_spacing", - "numeric_literals_py2", - "python2", - "python2_unicode_literals", "remove_parens", "slices", "string_prefixes", @@ -56,6 +54,12 @@ "tupleassign", ] +SIMPLE_CASES_PY2 = [ + "numeric_literals_py2", + "python2", + "python2_unicode_literals", +] + EXPERIMENTAL_STRING_PROCESSING_CASES = [ "cantfit", "comments7", @@ -86,6 +90,12 @@ class TestSimpleFormat(BlackBaseTestCase): + @parameterized.expand(SIMPLE_CASES_PY2) + @pytest.mark.python2 + @patch("black.dump_to_file", dump_to_stderr) + def test_simple_format_py2(self, filename: str) -> None: + self.check_file(filename, DEFAULT_MODE) + @parameterized.expand(SIMPLE_CASES) @patch("black.dump_to_file", dump_to_stderr) def test_simple_format(self, filename: str) -> None: diff --git a/tox.ini b/tox.ini index affc3c9876a..cbb0f75d145 100644 --- a/tox.ini +++ b/tox.ini @@ -7,9 +7,11 @@ skip_install = True deps = -r{toxinidir}/test_requirements.txt commands = - pip install -e .[d,python2] + pip install -e .[d] coverage erase - coverage run -m pytest tests {posargs} + coverage run -m pytest tests -m "not python2" {posargs} + pip install -e .[d,python2] + coverage run -m pytest tests -m "not without_python2" {posargs} coverage report [testenv:fuzz] From 5918a016ff82e5fa12097d07b1624a89ec4e60ac Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Tue, 4 May 2021 04:47:59 -0400 Subject: [PATCH 08/19] Drop Travis CI and migrate Coveralls (#2186) Travis CI for Open Source is shutting down in a few weeks so the queue for jobs is insane due to lower resources. I'm 99.99% sure we don't need it as our Test, Lint, Docs, Upload / Package, Primer, and Fuzz workflows are all on GitHub Actions. So even though we *can* migrate to the .com version with its 1000 free Linux minutes(?), I don't think we need to. more information here: - https://blog.travis-ci.com/oss-announcement - https://blog.travis-ci.com/2020-11-02-travis-ci-new-billing - https://docs.travis-ci.com/user/migrate/open-source-repository-migration This commit does the following: - delete the Travis CI configuration - add to the GHA test workflows so coverage continues to be recorded - tweaked coverage configuration so this wouldn't break - remove any references to Travis CI in the docs (i.e. readme + sphinx docs) Regarding the Travis CI to GitHub Actions Coveralls transition, the official action doesn't support the coverage files produced by coverage.py unfornately. Also no, I don't really know what I am doing so don't @ me if this breaks :p (well you can, but don't expect me to be THAT useful). The Coveralls setup has two downfalls AFAIK: - Only Linux runs are used because AndreMiras/coveralls-python-action only supports Linux. Although this isn't a big issue since the Travis Coveralls configuration only used Linux data too. - Pull requests from an internal branch (i.e. one on psf/black) will be marked as a push coverage build by Coveralls since our anti-duplicate- workflows system runs under the push even for such cases. --- .coveragerc | 3 +++ .github/workflows/test.yml | 31 +++++++++++++++++++++++++++++++ .travis.yml | 31 ------------------------------- README.md | 1 - docs/conf.py | 1 - 5 files changed, 34 insertions(+), 33 deletions(-) delete mode 100644 .travis.yml diff --git a/.coveragerc b/.coveragerc index 32a8da521ba..f05041ca1dc 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,3 +3,6 @@ omit = src/blib2to3/* tests/data/* */site-packages/* + +[run] +relative_files = True diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bec769064c2..03adc7f0bea 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,3 +34,34 @@ jobs: - name: Unit tests run: | tox -e py + + - name: Publish coverage to Coveralls + # If pushed / is a pull request against main repo AND + # we're running on Linux (this action only supports Linux) + if: + ((github.event_name == 'push' && github.repository == 'psf/black') || + github.event.pull_request.base.repo.full_name == 'psf/black') && matrix.os == + 'ubuntu-latest' + + uses: AndreMiras/coveralls-python-action@v20201129 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + parallel: true + flag-name: py${{ matrix.python-version }}-${{ matrix.os }} + debug: true + + coveralls-finish: + needs: build + # If pushed / is a pull request against main repo + if: + (github.event_name == 'push' && github.repository == 'psf/black') || + github.event.pull_request.base.repo.full_name == 'psf/black' + + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Coveralls finished + uses: AndreMiras/coveralls-python-action@v20201129 + with: + parallel-finished: true + debug: true diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index e782272e813..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -language: python -cache: - pip: true - directories: - - $HOME/.cache/pre-commit -env: - - TEST_CMD="tox -e py" -install: - - pip install coverage coveralls pre-commit tox - - pip install -e '.[d]' -script: - - $TEST_CMD -after_success: - - coveralls -notifications: - on_success: change - on_failure: always -matrix: - include: - - name: "lint" - python: 3.7 - env: - - TEST_CMD="pre-commit run --all-files --show-diff-on-failure" - - name: "3.6" - python: 3.6 - - name: "3.7" - python: 3.7 - - name: "3.8" - python: 3.8 - - name: "3.9" - python: 3.9 diff --git a/README.md b/README.md index 41cea761df5..696bfa7e064 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,6 @@

The Uncompromising Code Formatter

-Build Status Actions Status Actions Status Documentation Status diff --git a/docs/conf.py b/docs/conf.py index ec6aad9a27b..9e03d05e937 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -314,7 +314,6 @@ def process_sections( "show_powered_by": True, "fixed_sidebar": True, "logo": "logo2.png", - "travis_button": True, } From 0c60ccc06646030bd48d4e160c6aae755307c2d9 Mon Sep 17 00:00:00 2001 From: reka <382113+reka@users.noreply.github.com> Date: Tue, 4 May 2021 10:48:59 +0200 Subject: [PATCH 09/19] compatible isort config: mention profile first (#2180) Change the order of possible ways to configure isort: 1. using the profile black 2. custom configuration Formats section: change the examples to use the profile black Co-authored-by: Richard Si <63936253+ichard26@users.noreply.github.com> --- docs/compatible_configs.md | 50 +++++++++++++++++--------------------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/docs/compatible_configs.md b/docs/compatible_configs.md index a4f83ee472f..de81769f72d 100644 --- a/docs/compatible_configs.md +++ b/docs/compatible_configs.md @@ -19,7 +19,24 @@ Compatible configuration files can be _Black_ also formats imports, but in a different way from isort's defaults which leads to conflicting changes. -### Configuration +### Profile + +Since version 5.0.0, isort supports +[profiles](https://pycqa.github.io/isort/docs/configuration/profiles/) to allow easy +interoperability with common code styles. You can set the black profile in any of the +[config files](https://pycqa.github.io/isort/docs/configuration/config_files/) supported +by isort. Below, an example for `pyproject.toml`: + +```toml +[tool.isort] +profile = "black" +``` + +### Custom Configuration + +If you're using an isort version that is older than 5.0.0 or you have some custom +configuration for _Black_, you can tweak your isort configuration to make it compatible +with _Black_. Below, an example for `.isort.cfg`: ``` multi_line_output = 3 @@ -72,9 +89,6 @@ works the same as with _Black_. **Please note** `ensure_newline_before_comments = True` only works since isort >= 5 but does not break older versions so you can keep it if you are running previous versions. -If only isort >= 5 is used you can add `profile = black` instead of all the options -since [profiles](https://timothycrosley.github.io/isort/docs/configuration/profiles/) -are available and do the configuring for you. ### Formats @@ -83,12 +97,7 @@ are available and do the configuring for you. ```cfg [settings] -multi_line_output = 3 -include_trailing_comma = True -force_grid_wrap = 0 -use_parentheses = True -ensure_newline_before_comments = True -line_length = 88 +profile = black ``` @@ -98,12 +107,7 @@ line_length = 88 ```cfg [isort] -multi_line_output = 3 -include_trailing_comma = True -force_grid_wrap = 0 -use_parentheses = True -ensure_newline_before_comments = True -line_length = 88 +profile = black ``` @@ -113,12 +117,7 @@ line_length = 88 ```toml [tool.isort] -multi_line_output = 3 -include_trailing_comma = true -force_grid_wrap = 0 -use_parentheses = true -ensure_newline_before_comments = true -line_length = 88 +profile = 'black' ``` @@ -128,12 +127,7 @@ line_length = 88 ```ini [*.py] -multi_line_output = 3 -include_trailing_comma = True -force_grid_wrap = 0 -use_parentheses = True -ensure_newline_before_comments = True -line_length = 88 +profile = black ``` From 267bc5dde9f4e5e4b6dacdf79cf1688ffe9b7715 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Tue, 4 May 2021 14:41:04 +0300 Subject: [PATCH 10/19] Use pre-commit/action to simplify CI (#2191) --- .github/workflows/lint.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 2480a5ec131..51f6d02e2e6 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -22,8 +22,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install --upgrade pre-commit python -m pip install -e '.[d]' - name: Lint - run: pre-commit run --all-files --show-diff-on-failure + uses: pre-commit/action@v2.0.2 From d8a034f9b69b3a214e01985e741c4418ae562e2e Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Tue, 4 May 2021 11:07:08 -0700 Subject: [PATCH 11/19] Update CHANGES.md for 21.5b0 release (#2192) * Update CHANGES.md for 21.5b0 release * Make prettier happy --- CHANGES.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 00d6782dc6b..becf69904fb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,15 +1,17 @@ ## Change Log -### Unreleased +### 21.5b0 #### _Black_ - Set `--pyi` mode if `--stdin-filename` ends in `.pyi` (#2169) -- Add `--no-diff` to black-primer to suppress formatting changes (#2187) - - Stop detecting target version as Python 3.9+ with pre-PEP-614 decorators that are being called but with no arguments (#2182) +#### _Black-Primer_ + +- Add `--no-diff` to black-primer to suppress formatting changes (#2187) + ### 21.4b2 #### _Black_ From 14c76e89716b5b53c97ece80bb935ea956b7dd89 Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Tue, 4 May 2021 12:49:20 -0700 Subject: [PATCH 12/19] Disable pandas while we look into #2193 (#2195) --- src/black_primer/primer.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 137d075c68e..78c1e2acdef 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -53,6 +53,8 @@ "py_versions": ["all"] }, "pandas": { + "disabled_reason": "black-primer runs failing on Pandas - #2193", + "disabled": true, "cli_arguments": [], "expect_formatting_changes": true, "git_clone_url": "https://github.com/pandas-dev/pandas.git", From 07c8812937cf75ac5bc7ceac07ef5ea383f10f2f Mon Sep 17 00:00:00 2001 From: Cooper Lees Date: Wed, 5 May 2021 08:33:23 -0700 Subject: [PATCH 13/19] Enable ` --experimental-string-processing` on most primer projects (#2184) * Enable ` --experimental-string-processing` on all primer projects - We want to make this default so need to test it more - Fixed splat/star bug in extending black args for each project * Disable sqlalchemy due to crash --- src/black_primer/lib.py | 2 +- src/black_primer/primer.json | 47 ++++++++++++++++++------------------ 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/src/black_primer/lib.py b/src/black_primer/lib.py index aba694b0e60..3ce383f17ce 100644 --- a/src/black_primer/lib.py +++ b/src/black_primer/lib.py @@ -120,7 +120,7 @@ async def black_run( """Run Black and record failures""" cmd = [str(which(BLACK_BINARY))] if "cli_arguments" in project_config and project_config["cli_arguments"]: - cmd.extend(*project_config["cli_arguments"]) + cmd.extend(project_config["cli_arguments"]) cmd.append("--check") if no_diff: cmd.append(".") diff --git a/src/black_primer/primer.json b/src/black_primer/primer.json index 78c1e2acdef..76ed4820487 100644 --- a/src/black_primer/primer.json +++ b/src/black_primer/primer.json @@ -2,28 +2,28 @@ "configuration_format_version": 20200509, "projects": { "aioexabgp": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": false, "git_clone_url": "https://github.com/cooperlees/aioexabgp.git", "long_checkout": false, "py_versions": ["all"] }, "attrs": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/python-attrs/attrs.git", "long_checkout": false, "py_versions": ["all"] }, "bandersnatch": { - "cli_arguments": [], - "expect_formatting_changes": false, + "cli_arguments": ["--experimental-string-processing"], + "expect_formatting_changes": true, "git_clone_url": "https://github.com/pypa/bandersnatch.git", "long_checkout": false, "py_versions": ["all"] }, "channels": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/django/channels.git", "long_checkout": false, @@ -32,22 +32,22 @@ "django": { "disabled_reason": "black --check --diff returned 123 on tests_syntax_error.py", "disabled": true, - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/django/django.git", "long_checkout": false, "py_versions": ["all"] }, "flake8-bugbear": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": false, "git_clone_url": "https://github.com/PyCQA/flake8-bugbear.git", "long_checkout": false, "py_versions": ["all"] }, "hypothesis": { - "cli_arguments": [], - "expect_formatting_changes": false, + "cli_arguments": ["--experimental-string-processing"], + "expect_formatting_changes": true, "git_clone_url": "https://github.com/HypothesisWorks/hypothesis.git", "long_checkout": false, "py_versions": ["all"] @@ -55,55 +55,56 @@ "pandas": { "disabled_reason": "black-primer runs failing on Pandas - #2193", "disabled": true, - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/pandas-dev/pandas.git", "long_checkout": false, "py_versions": ["all"] }, "pillow": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/python-pillow/Pillow.git", "long_checkout": false, "py_versions": ["all"] }, "poetry": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/python-poetry/poetry.git", "long_checkout": false, "py_versions": ["all"] }, "pyanalyze": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": false, "git_clone_url": "https://github.com/quora/pyanalyze.git", "long_checkout": false, "py_versions": ["all"] }, "pyramid": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/Pylons/pyramid.git", "long_checkout": false, "py_versions": ["all"] }, "ptr": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/facebookincubator/ptr.git", "long_checkout": false, "py_versions": ["all"] }, "pytest": { - "cli_arguments": [], - "expect_formatting_changes": false, + "cli_arguments": ["--experimental-string-processing"], + "expect_formatting_changes": true, "git_clone_url": "https://github.com/pytest-dev/pytest.git", "long_checkout": false, "py_versions": ["all"] }, "sqlalchemy": { + "no_cli_args_reason": "breaks black with new string parsing - #2188", "cli_arguments": [], "expect_formatting_changes": true, "git_clone_url": "https://github.com/sqlalchemy/sqlalchemy.git", @@ -111,28 +112,28 @@ "py_versions": ["all"] }, "tox": { - "cli_arguments": [], - "expect_formatting_changes": false, + "cli_arguments": ["--experimental-string-processing"], + "expect_formatting_changes": true, "git_clone_url": "https://github.com/tox-dev/tox.git", "long_checkout": false, "py_versions": ["all"] }, "typeshed": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/python/typeshed.git", "long_checkout": false, "py_versions": ["all"] }, "virtualenv": { - "cli_arguments": [], - "expect_formatting_changes": false, + "cli_arguments": ["--experimental-string-processing"], + "expect_formatting_changes": true, "git_clone_url": "https://github.com/pypa/virtualenv.git", "long_checkout": false, "py_versions": ["all"] }, "warehouse": { - "cli_arguments": [], + "cli_arguments": ["--experimental-string-processing"], "expect_formatting_changes": true, "git_clone_url": "https://github.com/pypa/warehouse.git", "long_checkout": false, From 5316836393682c6ec6a05d69c549d8167f46d8f6 Mon Sep 17 00:00:00 2001 From: Shota Ray Imaki Date: Thu, 6 May 2021 11:25:43 +0900 Subject: [PATCH 14/19] Simplify GitHub Action entrypoint (#2119) This commit simplifies entrypoint.sh for GitHub Actions by removing duplication of args and black_args (cf. #1909). The reason why #1909 uses the input id black_args is to avoid an overlap with args, but this naming seems redundant. So let me suggest option and src, which are consistent with CLI. Backward compatibility is guaranteed; Users can still use black_args as well. Commit history pre-merge: * Simplify GitHub Action entrypoint (#1909) * Fix prettier * Emit a warning message when `black_args` is used This deprecation should be visible in GitHub Action's UI now. Co-authored-by: Shota Ray Imaki Co-authored-by: Richard Si <63936253+ichard26@users.noreply.github.com> --- README.md | 14 ++++++++------ action.yml | 12 +++++++++++- action/entrypoint.sh | 21 ++++++--------------- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 696bfa7e064..6443569d0d0 100644 --- a/README.md +++ b/README.md @@ -425,15 +425,17 @@ jobs: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 - uses: psf/black@stable - with: - args: ". --check" ``` -### Inputs +You may use `options` (Default is `'--check --diff'`) and `src` (Default is `'.'`) as +follows: -#### `black_args` - -**optional**: Black input arguments. Defaults to `. --check --diff`. +```yaml +- uses: psf/black@stable + with: + options: "--check --verbose" + src: "./src" +``` ## Ignoring unmodified files diff --git a/action.yml b/action.yml index 59b16a9fb6c..827e971801b 100644 --- a/action.yml +++ b/action.yml @@ -2,8 +2,18 @@ name: "Black" description: "The uncompromising Python code formatter." author: "Łukasz Langa and contributors to Black" inputs: + options: + description: + "Options passed to black. Use `black --help` to see available options. Default: + '--check'" + required: false + default: "--check --diff" + src: + description: "Source to run black. Default: '.'" + required: false + default: "." black_args: - description: "Black input arguments." + description: "[DEPRECATED] Black input arguments." required: false default: "" branding: diff --git a/action/entrypoint.sh b/action/entrypoint.sh index 50f9472cc5f..fc66e24f53a 100755 --- a/action/entrypoint.sh +++ b/action/entrypoint.sh @@ -1,17 +1,8 @@ -#!/bin/bash -set -e +#!/bin/bash -e -# If no arguments are given use current working directory -black_args=(".") -if [ "$#" -eq 0 ] && [ "${INPUT_BLACK_ARGS}" != "" ]; then - black_args+=(${INPUT_BLACK_ARGS}) -elif [ "$#" -ne 0 ] && [ "${INPUT_BLACK_ARGS}" != "" ]; then - black_args+=($* ${INPUT_BLACK_ARGS}) -elif [ "$#" -ne 0 ] && [ "${INPUT_BLACK_ARGS}" == "" ]; then - black_args+=($*) -else - # Default (if no args provided). - black_args+=("--check" "--diff") -fi +if [ -n $INPUT_BLACK_ARGS ]; then + echo '::warning::Input `with.black_args` is deprecated. Use `with.options` and `with.src` instead.' + black $INPUT_BLACK_ARGS + exit $? -sh -c "black . ${black_args[*]}" +black $INPUT_OPTIONS $INPUT_SRC From 4b7b5ed5b8630a906e2ef3a405134131651a251e Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Thu, 6 May 2021 15:21:13 -0400 Subject: [PATCH 15/19] Fix broken Action entrypoint (#2202) --- action/entrypoint.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/action/entrypoint.sh b/action/entrypoint.sh index fc66e24f53a..30bf4eb688f 100755 --- a/action/entrypoint.sh +++ b/action/entrypoint.sh @@ -1,8 +1,9 @@ #!/bin/bash -e -if [ -n $INPUT_BLACK_ARGS ]; then +if [ -n "$INPUT_BLACK_ARGS" ]; then echo '::warning::Input `with.black_args` is deprecated. Use `with.options` and `with.src` instead.' black $INPUT_BLACK_ARGS exit $? +fi black $INPUT_OPTIONS $INPUT_SRC From 1fe2efd8573a63ffc76c69320720d349b21897da Mon Sep 17 00:00:00 2001 From: Kaleb Barrett Date: Fri, 7 May 2021 07:54:21 -0500 Subject: [PATCH 16/19] Do not use gitignore if explicitly passing excludes (#2170) Closes #2164. Changes behavior of how .gitignore is handled. With this change, the rules in .gitignore are only used as a fallback if no exclusion rule is explicitly passed on the command line or in pyproject.toml. Previously they were used regardless if explicit exclusion rules were specified, preventing any overriding of .gitignore rules. Those that depend only on .gitignore for their exclusion rules will not be affected. Those that use both .gitignore and exclude will find that exclude will act more like actually specifying exclude and not just another extra-excludes. If the previous behavior was desired, they should move their rules from exclude to extra-excludes. --- CHANGES.md | 3 ++ src/black/__init__.py | 25 ++++++++++------- tests/data/include_exclude_tests/.gitignore | 1 + .../data/include_exclude_tests/pyproject.toml | 3 ++ tests/test_black.py | 28 +++++++++++++++++++ 5 files changed, 50 insertions(+), 10 deletions(-) create mode 100644 tests/data/include_exclude_tests/.gitignore create mode 100644 tests/data/include_exclude_tests/pyproject.toml diff --git a/CHANGES.md b/CHANGES.md index becf69904fb..b81e6eb4411 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -22,6 +22,9 @@ [circumstances](https://github.com/psf/black/blob/master/docs/the_black_code_style.md#pragmatism) in which _Black_ may change the AST (#2159) +- Allow `.gitignore` rules to be overridden by specifying `exclude` in `pyproject.toml` + or on the command line. (#2170) + #### _Packaging_ - Install `primer.json` (used by `black-primer` by default) with black. (#2154) diff --git a/src/black/__init__.py b/src/black/__init__.py index cf257876c03..e47aa21d435 100644 --- a/src/black/__init__.py +++ b/src/black/__init__.py @@ -492,15 +492,15 @@ def validate_regex( @click.option( "--exclude", type=str, - default=DEFAULT_EXCLUDES, callback=validate_regex, help=( "A regular expression that matches files and directories that should be" " excluded on recursive searches. An empty value means no paths are excluded." " Use forward slashes for directories on all platforms (Windows, too)." - " Exclusions are calculated first, inclusions later." + " Exclusions are calculated first, inclusions later. [default:" + f" {DEFAULT_EXCLUDES}]" ), - show_default=True, + show_default=False, ) @click.option( "--extend-exclude", @@ -587,7 +587,7 @@ def main( quiet: bool, verbose: bool, include: Pattern, - exclude: Pattern, + exclude: Optional[Pattern], extend_exclude: Optional[Pattern], force_exclude: Optional[Pattern], stdin_filename: Optional[str], @@ -662,7 +662,7 @@ def get_sources( quiet: bool, verbose: bool, include: Pattern[str], - exclude: Pattern[str], + exclude: Optional[Pattern[str]], extend_exclude: Optional[Pattern[str]], force_exclude: Optional[Pattern[str]], report: "Report", @@ -673,7 +673,12 @@ def get_sources( root = find_project_root(src) sources: Set[Path] = set() path_empty(src, "No Path provided. Nothing to do 😴", quiet, verbose, ctx) - gitignore = get_gitignore(root) + + if exclude is None: + exclude = re_compile_maybe_verbose(DEFAULT_EXCLUDES) + gitignore = get_gitignore(root) + else: + gitignore = None for s in src: if s == "-" and stdin_filename: @@ -6215,12 +6220,12 @@ def path_is_excluded( def gen_python_files( paths: Iterable[Path], root: Path, - include: Optional[Pattern[str]], + include: Pattern[str], exclude: Pattern[str], extend_exclude: Optional[Pattern[str]], force_exclude: Optional[Pattern[str]], report: "Report", - gitignore: PathSpec, + gitignore: Optional[PathSpec], ) -> Iterator[Path]: """Generate all files under `path` whose paths are not excluded by the `exclude_regex`, `extend_exclude`, or `force_exclude` regexes, @@ -6236,8 +6241,8 @@ def gen_python_files( if normalized_path is None: continue - # First ignore files matching .gitignore - if gitignore.match_file(normalized_path): + # First ignore files matching .gitignore, if passed + if gitignore is not None and gitignore.match_file(normalized_path): report.path_ignored(child, "matches the .gitignore file content") continue diff --git a/tests/data/include_exclude_tests/.gitignore b/tests/data/include_exclude_tests/.gitignore new file mode 100644 index 00000000000..91f34560522 --- /dev/null +++ b/tests/data/include_exclude_tests/.gitignore @@ -0,0 +1 @@ +dont_exclude/ diff --git a/tests/data/include_exclude_tests/pyproject.toml b/tests/data/include_exclude_tests/pyproject.toml new file mode 100644 index 00000000000..9ba7ec26980 --- /dev/null +++ b/tests/data/include_exclude_tests/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools>=41.0", "setuptools-scm", "wheel"] +build-backend = "setuptools.build_meta" diff --git a/tests/test_black.py b/tests/test_black.py index 7d855cab3d3..9b2bfcd740e 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -1418,6 +1418,32 @@ def test_include_exclude(self) -> None: ) self.assertEqual(sorted(expected), sorted(sources)) + def test_gitingore_used_as_default(self) -> None: + path = Path(THIS_DIR / "data" / "include_exclude_tests") + include = re.compile(r"\.pyi?$") + extend_exclude = re.compile(r"/exclude/") + src = str(path / "b/") + report = black.Report() + expected: List[Path] = [ + path / "b/.definitely_exclude/a.py", + path / "b/.definitely_exclude/a.pyi", + ] + sources = list( + black.get_sources( + ctx=FakeContext(), + src=(src,), + quiet=True, + verbose=False, + include=include, + exclude=None, + extend_exclude=extend_exclude, + force_exclude=None, + report=report, + stdin_filename=None, + ) + ) + self.assertEqual(sorted(expected), sorted(sources)) + @patch("black.find_project_root", lambda *args: THIS_DIR.resolve()) def test_exclude_for_issue_1572(self) -> None: # Exclude shouldn't touch files that were explicitly given to Black through the @@ -1705,6 +1731,8 @@ def test_empty_include(self) -> None: Path(path / "b/.definitely_exclude/a.pie"), Path(path / "b/.definitely_exclude/a.py"), Path(path / "b/.definitely_exclude/a.pyi"), + Path(path / ".gitignore"), + Path(path / "pyproject.toml"), ] this_abs = THIS_DIR.resolve() sources.extend( From e4b4fb02b91e0f5a60a9678604653aecedff513b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 7 May 2021 15:03:13 +0200 Subject: [PATCH 17/19] Use optional tests for "no_python2" to simplify local testing (#2203) --- pyproject.toml | 5 +- tests/conftest.py | 1 + tests/optional.py | 119 ++++++++++++++++++++++++++++++++++++++++++++ tests/test_black.py | 10 ++-- tox.ini | 4 +- 5 files changed, 133 insertions(+), 6 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/optional.py diff --git a/pyproject.toml b/pyproject.toml index ca75f8f92ef..e89cc7a6c9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,4 +27,7 @@ requires = ["setuptools>=41.0", "setuptools-scm", "wheel"] build-backend = "setuptools.build_meta" [tool.pytest.ini_options] -markers = ['python2', "without_python2"] \ No newline at end of file +# Option below requires `tests/optional.py` +optional-tests = [ + "no_python2: run when `python2` extra NOT installed", +] \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000000..67517268d1b --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1 @@ +pytest_plugins = ["tests.optional"] diff --git a/tests/optional.py b/tests/optional.py new file mode 100644 index 00000000000..e12b94cd29e --- /dev/null +++ b/tests/optional.py @@ -0,0 +1,119 @@ +""" +Allows configuring optional test markers in config, see pyproject.toml. + +Run optional tests with `pytest --run-optional=...`. + +Mark tests to run only if an optional test ISN'T selected by prepending the mark with +"no_". + +You can specify a "no_" prefix straight in config, in which case you can mark tests +to run when this tests ISN'T selected by omitting the "no_" prefix. + +Specifying the name of the default behavior in `--run-optional=` is harmless. + +Adapted from https://pypi.org/project/pytest-optional-tests/, (c) 2019 Reece Hart +""" + +from functools import lru_cache +import itertools +import logging +import re +from typing import FrozenSet, List, Set, TYPE_CHECKING + +import pytest +from _pytest.store import StoreKey + +log = logging.getLogger(__name__) + + +if TYPE_CHECKING: + from _pytest.config.argparsing import Parser + from _pytest.config import Config + from _pytest.mark.structures import MarkDecorator + from _pytest.nodes import Node + + +ALL_POSSIBLE_OPTIONAL_MARKERS = StoreKey[FrozenSet[str]]() +ENABLED_OPTIONAL_MARKERS = StoreKey[FrozenSet[str]]() + + +def pytest_addoption(parser: "Parser") -> None: + group = parser.getgroup("collect") + group.addoption( + "--run-optional", + action="append", + dest="run_optional", + default=None, + help="Optional test markers to run; comma-separated", + ) + parser.addini("optional-tests", "List of optional tests markers", "linelist") + + +def pytest_configure(config: "Config") -> None: + """Optional tests are markers. + + Use the syntax in https://docs.pytest.org/en/stable/mark.html#registering-marks. + """ + ot_ini = config.inicfg.get("optional-tests") or [] + ot_markers = set() + ot_run: Set[str] = set() + if isinstance(ot_ini, str): + ot_ini = ot_ini.strip().split("\n") + marker_re = re.compile(r"^\s*(?Pno_)?(?P\w+)(:\s*(?P.*))?") + for ot in ot_ini: + m = marker_re.match(ot) + if not m: + raise ValueError(f"{ot!r} doesn't match pytest marker syntax") + + marker = (m.group("no") or "") + m.group("marker") + description = m.group("description") + config.addinivalue_line("markers", f"{marker}: {description}") + config.addinivalue_line( + "markers", f"{no(marker)}: run when `{marker}` not passed" + ) + ot_markers.add(marker) + + # collect requested optional tests + passed_args = config.getoption("run_optional") + if passed_args: + ot_run.update(itertools.chain.from_iterable(a.split(",") for a in passed_args)) + ot_run |= {no(excluded) for excluded in ot_markers - ot_run} + ot_markers |= {no(m) for m in ot_markers} + + log.info("optional tests to run:", ot_run) + unknown_tests = ot_run - ot_markers + if unknown_tests: + raise ValueError(f"Unknown optional tests wanted: {unknown_tests!r}") + + store = config._store + store[ALL_POSSIBLE_OPTIONAL_MARKERS] = frozenset(ot_markers) + store[ENABLED_OPTIONAL_MARKERS] = frozenset(ot_run) + + +def pytest_collection_modifyitems(config: "Config", items: "List[Node]") -> None: + store = config._store + all_possible_optional_markers = store[ALL_POSSIBLE_OPTIONAL_MARKERS] + enabled_optional_markers = store[ENABLED_OPTIONAL_MARKERS] + + for item in items: + all_markers_on_test = set(m.name for m in item.iter_markers()) + optional_markers_on_test = all_markers_on_test & all_possible_optional_markers + if not optional_markers_on_test or ( + optional_markers_on_test & enabled_optional_markers + ): + continue + log.info("skipping non-requested optional", item) + item.add_marker(skip_mark(frozenset(optional_markers_on_test))) + + +@lru_cache() +def skip_mark(tests: FrozenSet[str]) -> "MarkDecorator": + names = ", ".join(sorted(tests)) + return pytest.mark.skip(reason=f"Marked with disabled optional tests ({names})") + + +@lru_cache() +def no(name: str) -> str: + if name.startswith("no_"): + return name[len("no_") :] + return "no_" + name diff --git a/tests/test_black.py b/tests/test_black.py index 9b2bfcd740e..b8e526a953e 100644 --- a/tests/test_black.py +++ b/tests/test_black.py @@ -460,11 +460,15 @@ def test_skip_magic_trailing_comma(self) -> None: ) self.assertEqual(expected, actual, msg) - @pytest.mark.without_python2 + @pytest.mark.no_python2 def test_python2_should_fail_without_optional_install(self) -> None: - # python 3.7 and below will install typed-ast and will be able to parse Python 2 if sys.version_info < (3, 8): - return + self.skipTest( + "Python 3.6 and 3.7 will install typed-ast to work and as such will be" + " able to parse Python 2 syntax without explicitly specifying the" + " python2 extra" + ) + source = "x = 1234l" tmp_file = Path(black.dump_to_file(source)) try: diff --git a/tox.ini b/tox.ini index cbb0f75d145..317bf485375 100644 --- a/tox.ini +++ b/tox.ini @@ -9,9 +9,9 @@ deps = commands = pip install -e .[d] coverage erase - coverage run -m pytest tests -m "not python2" {posargs} + coverage run -m pytest tests --run-optional=no_python2 {posargs} pip install -e .[d,python2] - coverage run -m pytest tests -m "not without_python2" {posargs} + coverage run -m pytest tests --run-optional=python2 {posargs} coverage report [testenv:fuzz] From d0e06b53b09248be34c1d5c0fa8f050bff1d201c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Langa?= Date: Fri, 7 May 2021 16:33:36 +0200 Subject: [PATCH 18/19] Mark blackd tests with the `blackd` optional marker (#2204) This is a follow-up of #2203 that uses a pytest marker instead of a bunch of `skipUnless`. Similarly to the Python 2 tests, they are running by default and will crash on an unsuspecting contributor with missing dependencies. This is by design, we WANT contributors to test everything. Unless we actually don't and then we can run: pytest --run-optional=no_blackd Relatedly, bump required aiohttp to 3.6.0 at least to get rid of expected failures on Python 3.8 (see 6b5eb7d4651c7333cc3f5df4bf7aa7a1f1ffb45b). --- Pipfile | 2 +- pyproject.toml | 1 + setup.py | 2 +- tests/test_blackd.py | 32 +++----------------------------- tests/util.py | 14 +------------- 5 files changed, 7 insertions(+), 44 deletions(-) diff --git a/Pipfile b/Pipfile index 40f3d607653..68562f5e53f 100644 --- a/Pipfile +++ b/Pipfile @@ -20,7 +20,7 @@ wheel = ">=0.31.1" black = {editable = true, extras = ["d"], path = "."} [packages] -aiohttp = ">=3.3.2" +aiohttp = ">=3.6.0" aiohttp-cors = "*" appdirs = "*" click = ">=7.1.2" diff --git a/pyproject.toml b/pyproject.toml index e89cc7a6c9b..4d6777ef6ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,4 +30,5 @@ build-backend = "setuptools.build_meta" # Option below requires `tests/optional.py` optional-tests = [ "no_python2: run when `python2` extra NOT installed", + "no_blackd: run when `d` extra NOT installed", ] \ No newline at end of file diff --git a/setup.py b/setup.py index f1792a46fe8..af93d0f453a 100644 --- a/setup.py +++ b/setup.py @@ -83,7 +83,7 @@ def get_long_description() -> str: "mypy_extensions>=0.4.3", ], extras_require={ - "d": ["aiohttp>=3.3.2", "aiohttp-cors"], + "d": ["aiohttp>=3.6.0", "aiohttp-cors"], "colorama": ["colorama>=0.4.3"], "python2": ["typed-ast>=1.4.2"], }, diff --git a/tests/test_blackd.py b/tests/test_blackd.py index 9127297c54f..9ca19d49dc6 100644 --- a/tests/test_blackd.py +++ b/tests/test_blackd.py @@ -1,10 +1,10 @@ import re -import unittest from unittest.mock import patch from click.testing import CliRunner +import pytest -from tests.util import read_data, DETERMINISTIC_HEADER, skip_if_exception +from tests.util import read_data, DETERMINISTIC_HEADER try: import blackd @@ -16,8 +16,8 @@ has_blackd_deps = True +@pytest.mark.blackd class BlackDTestCase(AioHTTPTestCase): - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") def test_blackd_main(self) -> None: with patch("blackd.web.run_app"): result = CliRunner().invoke(blackd.main, []) @@ -28,10 +28,6 @@ def test_blackd_main(self) -> None: async def get_application(self) -> web.Application: return blackd.make_app() - # TODO: remove these decorators once the below is released - # https://github.com/aio-libs/aiohttp/pull/3727 - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_request_needs_formatting(self) -> None: response = await self.client.post("/", data=b"print('hello world')") @@ -39,16 +35,12 @@ async def test_blackd_request_needs_formatting(self) -> None: self.assertEqual(response.charset, "utf8") self.assertEqual(await response.read(), b'print("hello world")\n') - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_request_no_change(self) -> None: response = await self.client.post("/", data=b'print("hello world")\n') self.assertEqual(response.status, 204) self.assertEqual(await response.read(), b"") - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_request_syntax_error(self) -> None: response = await self.client.post("/", data=b"what even ( is") @@ -59,8 +51,6 @@ async def test_blackd_request_syntax_error(self) -> None: msg=f"Expected error to start with 'Cannot parse', got {repr(content)}", ) - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_unsupported_version(self) -> None: response = await self.client.post( @@ -68,8 +58,6 @@ async def test_blackd_unsupported_version(self) -> None: ) self.assertEqual(response.status, 501) - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_supported_version(self) -> None: response = await self.client.post( @@ -77,8 +65,6 @@ async def test_blackd_supported_version(self) -> None: ) self.assertEqual(response.status, 200) - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_invalid_python_variant(self) -> None: async def check(header_value: str, expected_status: int = 400) -> None: @@ -97,8 +83,6 @@ async def check(header_value: str, expected_status: int = 400) -> None: await check("pypy3.0") await check("jython3.4") - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_pyi(self) -> None: source, expected = read_data("stub.pyi") @@ -108,8 +92,6 @@ async def test_blackd_pyi(self) -> None: self.assertEqual(response.status, 200) self.assertEqual(await response.text(), expected) - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_diff(self) -> None: diff_header = re.compile( @@ -128,8 +110,6 @@ async def test_blackd_diff(self) -> None: actual = diff_header.sub(DETERMINISTIC_HEADER, actual) self.assertEqual(actual, expected) - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_python_variant(self) -> None: code = ( @@ -166,8 +146,6 @@ async def check(header_value: str, expected_status: int) -> None: await check("py34,py36", 204) await check("34", 204) - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_line_length(self) -> None: response = await self.client.post( @@ -175,8 +153,6 @@ async def test_blackd_line_length(self) -> None: ) self.assertEqual(response.status, 200) - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_invalid_line_length(self) -> None: response = await self.client.post( @@ -184,8 +160,6 @@ async def test_blackd_invalid_line_length(self) -> None: ) self.assertEqual(response.status, 400) - @skip_if_exception("ClientOSError") - @unittest.skipUnless(has_blackd_deps, "blackd's dependencies are not installed") @unittest_run_loop async def test_blackd_response_black_version_header(self) -> None: response = await self.client.post("/") diff --git a/tests/util.py b/tests/util.py index ad9866975ac..3670952ba8c 100644 --- a/tests/util.py +++ b/tests/util.py @@ -1,8 +1,7 @@ import os import unittest -from contextlib import contextmanager from pathlib import Path -from typing import List, Tuple, Iterator, Any +from typing import List, Tuple, Any import black from functools import partial @@ -45,17 +44,6 @@ def assertFormatEqual(self, expected: str, actual: str) -> None: self.assertMultiLineEqual(expected, actual) -@contextmanager -def skip_if_exception(e: str) -> Iterator[None]: - try: - yield - except Exception as exc: - if exc.__class__.__name__ == e: - unittest.skip(f"Encountered expected exception {exc}, skipping") - else: - raise - - def read_data(name: str, data: bool = True) -> Tuple[str, str]: """read_data('test_name') -> 'input', 'output'""" if not name.endswith((".py", ".pyi", ".out", ".diff")): From 4fc1354aeb6b217cd18dbdb2a0c41373fa9d8056 Mon Sep 17 00:00:00 2001 From: Richard Si <63936253+ichard26@users.noreply.github.com> Date: Fri, 7 May 2021 10:41:55 -0400 Subject: [PATCH 19/19] Speed up test suite via distributed testing (#2196) * Speed up test suite via distributed testing Since we now run the test suite twice, one with Python 2 and another without, full test runs are getting pretty slow. Let's try to fix that with parallization. Also use verbose mode on CI since more logs is usually better since getting more is quite literally impossible. The main issue we'll face with this is we'll hit https://github.com/pytest-dev/pytest-xdist/issues/620 sometimes (although pretty rarely). I suppose we can test this and see if how bad this bug is for us, and revert if necessary down the line. Also let's have some colours :tada: --- .coveragerc | 1 + .github/workflows/test.yml | 2 +- test_requirements.txt | 2 ++ tox.ini | 4 ++-- 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.coveragerc b/.coveragerc index f05041ca1dc..5577e496a57 100644 --- a/.coveragerc +++ b/.coveragerc @@ -3,6 +3,7 @@ omit = src/blib2to3/* tests/data/* */site-packages/* + .tox/* [run] relative_files = True diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 03adc7f0bea..2cfbab67ce1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,7 @@ jobs: - name: Unit tests run: | - tox -e py + tox -e py -- -v --color=yes - name: Publish coverage to Coveralls # If pushed / is a pull request against main repo AND diff --git a/test_requirements.txt b/test_requirements.txt index a1464e90686..31ab2d05fea 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -3,5 +3,7 @@ pre-commit pytest >= 6.1.1 pytest-mock >= 3.3.1 pytest-cases >= 2.3.0 +pytest-xdist >= 2.2.1 +pytest-cov >= 2.11.1 parameterized >= 0.7.4 tox diff --git a/tox.ini b/tox.ini index 317bf485375..2379500f55a 100644 --- a/tox.ini +++ b/tox.ini @@ -9,9 +9,9 @@ deps = commands = pip install -e .[d] coverage erase - coverage run -m pytest tests --run-optional=no_python2 {posargs} + pytest tests --run-optional no_python2 --numprocesses auto --cov {posargs} pip install -e .[d,python2] - coverage run -m pytest tests --run-optional=python2 {posargs} + pytest tests --run-optional python2 --numprocesses auto --cov --cov-append {posargs} coverage report [testenv:fuzz]