Merge branch 'trs/build/download-globs'

CHANGES.md

@@ -63,6 +63,15 @@ This release contains **a potentially-breaking change** for existing usages of
   ncov_cross-species_cat_tip-frequencies.json
   ```
 
+* Advanced globbing features are now supported in patterns for the `--download`
+  option of `nextstrain build`, including multi-part wildcards (`**`), extended
+  globbing (`@(…)`, `+(…)`, etc.), and negation (`!…`).  Basic globbing
+  features like single-part wildcards (`*`), character classes (`[…]`), and
+  brace expansion (`{…, …}`) are still supported.  Note that the `--download`
+  option continues to be applicable only to the AWS Batch runtime (e.g. the
+  `--aws-batch` option).
+
+
 # 4.2.0 (29 July 2022)
 
 ## Bug fixes

nextstrain/cli/command/build.py

@@ -79,13 +79,21 @@ def register_parser(subparser):
     parser.add_argument(
         "--download",
         metavar = "<pattern>",
-        help    = "Only download new or modified files matching <pattern> from the remote build. "
-                  "Basic shell-style globbing is supported, but be sure to escape wildcards "
-                  "or quote the whole pattern so your shell doesn't expand them. "
-                  "May be passed more than once. "
-                  "Currently only supported when also using --aws-batch. "
-                  "Default is to download every new or modified file."
-                  f"{SKIP_AUTO_DEFAULT_IN_HELP}",
+        help    = dedent(f"""\
+            Only download new or modified files matching <pattern> from the
+            remote build.  Shell-style advanced globbing is supported, but be
+            sure to escape wildcards or quote the whole pattern so your shell
+            doesn't expand them.  May be passed more than once.  Currently only
+            supported when also using --aws-batch.  Default is to download
+            every new or modified file.
+
+            Besides basic glob features like single-part wildcards (*),
+            character classes ([…]), and brace expansion ({{…, …}}), several
+            advanced globbing features are also supported: multi-part wildcards
+            (**), extended globbing (@(…), +(…), etc.), and negation (!…).
+
+            {SKIP_AUTO_DEFAULT_IN_HELP}
+            """),
         default = True,
         action  = AppendOverwriteDefault)
 

nextstrain/cli/runner/aws_batch/s3.py

@@ -15,6 +15,7 @@
 from urllib.parse import urlparse
 from zipfile import ZipFile, ZipInfo
 from ...types import S3Bucket, S3Object
+from ...util import glob_matcher
 
 
 PathMatcher = Callable[[Path], bool]
@@ -79,8 +80,8 @@ def download_workdir(remote_workdir: S3Object, workdir: Path, patterns: List[str
 
     The remote workdir's files **overwrite** local files!
 
-    An optional list of *patterns* (shell-style globs) can be passed to
-    selectively download only part of the remote workdir.
+    An optional list of *patterns* (shell-style advanced globs) can be passed
+    to selectively download only part of the remote workdir.
     """
 
     excluded = path_matcher([
@@ -102,7 +103,7 @@ def download_workdir(remote_workdir: S3Object, workdir: Path, patterns: List[str
     ])
 
     if patterns:
-        selected = path_matcher(patterns)
+        selected = glob_matcher(patterns)
     else:
         selected = lambda path: True