Add --extend-exclude parameter (#2005)

Look ma! I contribute to open source!

Co-authored-by: Richard Si <63936253+ichard26@users.noreply.github.com>

README.md

@@ -135,11 +135,17 @@ Options:
                                   hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_bu
                                   ild|buck-out|build|dist)/]
 
+  --extend-exclude TEXT           Like --exclude, but adds additional files
+                                  and directories on top of the excluded
+                                  ones (useful if you simply want to add to
+                                  the default).
+
   --force-exclude TEXT            Like --exclude, but files and directories
                                   matching this regex will be excluded even
                                   when they are passed explicitly as
                                   arguments.
 
+
   --stdin-filename TEXT           The name of the file when passing it through
                                   stdin. Useful to make sure Black will
                                   respect --force-exclude option on some
@@ -151,7 +157,7 @@ Options:
 
   -v, --verbose                   Also emit messages to stderr about files
                                   that were not changed or were ignored due to
-                                  --exclude=.
+                                  exclusion patterns.
 
   --version                       Show the version and exit.
   --config FILE                   Read configuration from FILE path.
@@ -263,7 +269,7 @@ above. What seems like a bug might be intended behaviour.
 
 _Black_ is able to read project-specific default values for its command line options
 from a `pyproject.toml` file. This is especially useful for specifying custom
-`--include` and `--exclude` patterns for your project.
+`--include` and `--exclude`/`--extend-exclude` patterns for your project.
 
 **Pro-tip**: If you're asking yourself "Do I need to configure anything?" the answer is
 "No". _Black_ is all about sensible defaults.
@@ -313,25 +319,10 @@ expressions by Black. Use `[ ]` to denote a significant space character.
 line-length = 88
 target-version = ['py37']
 include = '\.pyi?$'
-exclude = '''
+extend-exclude = '''
 # A regex preceded with ^/ will apply only to files and directories
 # in the root of the project.
-^/(
-  (
-      \.eggs         # exclude a few common directories in the
-    | \.git          # root of the project
-    | \.hg
-    | \.mypy_cache
-    | \.tox
-    | \.venv
-    | _build
-    | buck-out
-    | build
-    | dist
-  )/
-  | foo.py           # also separately exclude a file named foo.py in
-                     # the root of the project
-)
+^/foo.py  # exclude a file named foo.py in the root of the project (in addition to the defaults)
 '''
 ```
 
@@ -616,6 +607,7 @@ Multiple contributions by:
 - [Joseph Larson](mailto:larson.joseph@gmail.com)
 - [Josh Bode](mailto:joshbode@fastmail.com)
 - [Josh Holland](mailto:anowlcalledjosh@gmail.com)
+- [Joshua Cannon](mailto:joshdcannon@gmail.com)
 - [José Padilla](mailto:jpadilla@webapplicate.com)
 - [Juan Luis Cano Rodríguez](mailto:hello@juanlu.space)
 - [kaiix](mailto:kvn.hou@gmail.com)

docs/change_log.md

@@ -28,6 +28,8 @@
 
 - use lowercase hex strings (#1692)
 
+- added `--extend-exclude` argument (#1571)
+
 #### _Packaging_
 
 - Self-contained native _Black_ binaries are now provided for releases via GitHub

docs/installation_and_usage.md

@@ -95,6 +95,11 @@ Options:
                                   when they are passed explicitly as
                                   arguments.
 
+  --extend-exclude TEXT           Like --exclude, but adds additional files
+                                  and directories on top of the excluded
+                                  ones. (useful if you simply want to add to
+                                  the default)
+
   --stdin-filename TEXT           The name of the file when passing it through
                                   stdin. Useful to make sure Black will
                                   respect --force-exclude option on some
@@ -106,7 +111,7 @@ Options:
 
   -v, --verbose                   Also emit messages to stderr about files
                                   that were not changed or were ignored due to
-                                  --exclude=.
+                                  exclusion patterns.
 
   --version                       Show the version and exit.
   --config FILE                   Read configuration from FILE path.

docs/pyproject_toml.md

@@ -4,7 +4,8 @@
 
 _Black_ is able to read project-specific default values for its command line options
 from a `pyproject.toml` file. This is especially useful for specifying custom
-`--include` and `--exclude` patterns for your project.
+`--include` and `--exclude`/`--force-exclude`/`--extend-exclude` patterns for your
+project.
 
 **Pro-tip**: If you're asking yourself "Do I need to configure anything?" the answer is
 "No". _Black_ is all about sensible defaults.
@@ -54,25 +55,10 @@ expressions by Black. Use `[ ]` to denote a significant space character.
 line-length = 88
 target-version = ['py37']
 include = '\.pyi?$'
-exclude = '''
+extend-exclude = '''
 # A regex preceded with ^/ will apply only to files and directories
 # in the root of the project.
-^/(
-  (
-      \.eggs         # exclude a few common directories in the
-    | \.git          # root of the project
-    | \.hg
-    | \.mypy_cache
-    | \.tox
-    | \.venv
-    | _build
-    | buck-out
-    | build
-    | dist
-  )/
-  | foo.py           # also separately exclude a file named foo.py in
-                     # the root of the project
-)
+^/foo.py  # exclude a file named foo.py in the root of the project (in addition to the defaults)
 '''
 ```
 

pyproject.toml

@@ -9,19 +9,8 @@
 line-length = 88
 target-version = ['py36', 'py37', 'py38']
 include = '\.pyi?$'
-exclude = '''
+extend-exclude = '''
 /(
-    \.eggs
-  | \.git
-  | \.hg
-  | \.mypy_cache
-  | \.tox
-  | \.venv
-  | _build
-  | buck-out
-  | build
-  | dist
-
   # The following are specific to Black, you probably don't want those.
   | blib2to3
   | tests/data

src/black/__init__.py

@@ -461,6 +461,14 @@ def target_version_option_callback(
     ),
     show_default=True,
 )
+@click.option(
+    "--extend-exclude",
+    type=str,
+    help=(
+        "Like --exclude, but adds additional files and directories on top of the"
+        " excluded ones. (Useful if you simply want to add to the default)"
+    ),
+)
 @click.option(
     "--force-exclude",
     type=str,
@@ -493,7 +501,7 @@ def target_version_option_callback(
     is_flag=True,
     help=(
         "Also emit messages to stderr about files that were not changed or were ignored"
-        " due to --exclude=."
+        " due to exclusion patterns."
     ),
 )
 @click.version_option(version=__version__)
@@ -537,6 +545,7 @@ def main(
     verbose: bool,
     include: str,
     exclude: str,
+    extend_exclude: Optional[str],
     force_exclude: Optional[str],
     stdin_filename: Optional[str],
     src: Tuple[str, ...],
@@ -570,6 +579,7 @@ def main(
         verbose=verbose,
         include=include,
         exclude=exclude,
+        extend_exclude=extend_exclude,
         force_exclude=force_exclude,
         report=report,
         stdin_filename=stdin_filename,
@@ -602,6 +612,18 @@ def main(
     ctx.exit(report.return_code)
 
 
+def test_regex(
+    ctx: click.Context,
+    regex_name: str,
+    regex: Optional[str],
+) -> Optional[Pattern]:
+    try:
+        return re_compile_maybe_verbose(regex) if regex is not None else None
+    except re.error:
+        err(f"Invalid regular expression for {regex_name} given: {regex!r}")
+        ctx.exit(2)
+
+
 def get_sources(
     *,
     ctx: click.Context,
@@ -610,28 +632,18 @@ def get_sources(
     verbose: bool,
     include: str,
     exclude: str,
+    extend_exclude: Optional[str],
     force_exclude: Optional[str],
     report: "Report",
     stdin_filename: Optional[str],
 ) -> Set[Path]:
     """Compute the set of files to be formatted."""
-    try:
-        include_regex = re_compile_maybe_verbose(include)
-    except re.error:
-        err(f"Invalid regular expression for include given: {include!r}")
-        ctx.exit(2)
-    try:
-        exclude_regex = re_compile_maybe_verbose(exclude)
-    except re.error:
-        err(f"Invalid regular expression for exclude given: {exclude!r}")
-        ctx.exit(2)
-    try:
-        force_exclude_regex = (
-            re_compile_maybe_verbose(force_exclude) if force_exclude else None
-        )
-    except re.error:
-        err(f"Invalid regular expression for force_exclude given: {force_exclude!r}")
-        ctx.exit(2)
+
+    include_regex = test_regex(ctx, "include", include)
+    exclude_regex = test_regex(ctx, "exclude", exclude)
+    assert exclude_regex is not None
+    extend_exclude_regex = test_regex(ctx, "extend_exclude", extend_exclude)
+    force_exclude_regex = test_regex(ctx, "force_exclude", force_exclude)
 
     root = find_project_root(src)
     sources: Set[Path] = set()
@@ -672,6 +684,7 @@ def get_sources(
                     root,
                     include_regex,
                     exclude_regex,
+                    extend_exclude_regex,
                     force_exclude_regex,
                     report,
                     gitignore,
@@ -6112,17 +6125,27 @@ def normalize_path_maybe_ignore(
     return normalized_path
 
 
+def path_is_excluded(
+    normalized_path: str,
+    pattern: Optional[Pattern[str]],
+) -> bool:
+    match = pattern.search(normalized_path) if pattern else None
+    return bool(match and match.group(0))
+
+
 def gen_python_files(
     paths: Iterable[Path],
     root: Path,
     include: Optional[Pattern[str]],
     exclude: Pattern[str],
+    extend_exclude: Optional[Pattern[str]],
     force_exclude: Optional[Pattern[str]],
     report: "Report",
     gitignore: PathSpec,
 ) -> Iterator[Path]:
     """Generate all files under `path` whose paths are not excluded by the
-    `exclude_regex` or `force_exclude` regexes, but are included by the `include` regex.
+    `exclude_regex`, `extend_exclude`, or `force_exclude` regexes,
+    but are included by the `include` regex.
 
     Symbolic links pointing outside of the `root` directory are ignored.
 
@@ -6139,20 +6162,22 @@ def gen_python_files(
             report.path_ignored(child, "matches the .gitignore file content")
             continue
 
-        # Then ignore with `--exclude` and `--force-exclude` options.
+        # Then ignore with `--exclude` `--extend-exclude` and `--force-exclude` options.
         normalized_path = "/" + normalized_path
         if child.is_dir():
             normalized_path += "/"
 
-        exclude_match = exclude.search(normalized_path) if exclude else None
-        if exclude_match and exclude_match.group(0):
+        if path_is_excluded(normalized_path, exclude):
             report.path_ignored(child, "matches the --exclude regular expression")
             continue
 
-        force_exclude_match = (
-            force_exclude.search(normalized_path) if force_exclude else None
-        )
-        if force_exclude_match and force_exclude_match.group(0):
+        if path_is_excluded(normalized_path, extend_exclude):
+            report.path_ignored(
+                child, "matches the --extend-exclude regular expression"
+            )
+            continue
+
+        if path_is_excluded(normalized_path, force_exclude):
             report.path_ignored(child, "matches the --force-exclude regular expression")
             continue
 
@@ -6162,6 +6187,7 @@ def gen_python_files(
                 root,
                 include,
                 exclude,
+                extend_exclude,
                 force_exclude,
                 report,
                 gitignore,