From f5c3b35f4a40e345faf87ccfe49003a914e58dee Mon Sep 17 00:00:00 2001 From: azhu Date: Wed, 7 Dec 2022 23:01:02 +1300 Subject: [PATCH 01/28] Initial support for argcomplete for KVArgParseConfigLoader After enabling argcomplete shell completion for a traitlets.Application based script (e.g. via activate-global-python-argcomplete), this commit sets up tab auto-completion for command-line flags and aliases. Argcomplete completers can be added for a trait by using an "argcompleter" metadata tag, which should have a function that takes keyword arguments (passed down from argcomplete) and returns a list of string completions. Completers are also set up for Bool ("true", "false", "1", "0") & Enum. This commit does *not* add general support for arbitrary class traits of the form --Class.trait=xyz, as these are currently not added to the ArgumentParser instance, but rather directly parsed. It is probably possible to add this support for the classes in Application.classes. Issue: #539 Example: ~/dev/traitlets$ python examples/myapp.py --[TAB] --debug --disable --enable --enabled --help --i --j --log_level --mode --name --running ~/dev/traitlets$ python examples/myapp.py --running [TAB] 0 1 false true ~/dev/traitlets$ python examples/myapp.py --running true --[TAB] --debug --disable --enable --enabled --help --i --j --log_level --mode --name --running ~/dev/traitlets$ python examples/myapp.py --running true --mode o[TAB] off on other --- examples/myapp.py | 16 +++++++++------- traitlets/config/loader.py | 22 ++++++++++++++++++++-- traitlets/traitlets.py | 11 +++++++++++ 3 files changed, 40 insertions(+), 9 deletions(-) mode change 100644 => 100755 examples/myapp.py diff --git a/examples/myapp.py b/examples/myapp.py old mode 100644 new mode 100755 index 77135ac6..973ff58d --- a/examples/myapp.py +++ b/examples/myapp.py @@ -1,3 +1,5 @@ +#!/usr/bin/env python +# PYTHON_ARGCOMPLETE_OK """A simple example of how to use traitlets.config.application.Application. This should serve as a simple example that shows how the traitlets config @@ -29,18 +31,17 @@ When the config attribute of an Application is updated, it will fire all of the trait's events for all of the config=True attributes. """ - -from traitlets import Bool, Dict, Int, List, Unicode +from traitlets import Bool, Dict, Enum, Int, List, Unicode from traitlets.config.application import Application from traitlets.config.configurable import Configurable - class Foo(Configurable): """A class that has configurable, typed attributes.""" i = Int(0, help="The integer i.").tag(config=True) j = Int(1, help="The integer j.").tag(config=True) name = Unicode("Brian", help="First name.").tag(config=True, shortname="B") + mode = Enum(values=["on", "off", "other"], default_value="on").tag(config=True) class Bar(Configurable): @@ -60,6 +61,7 @@ class MyApp(Application): i="Foo.i", j="Foo.j", name="Foo.name", + mode="Foo.mode", running="MyApp.running", enabled="Bar.enabled", log_level="MyApp.log_level", @@ -93,10 +95,10 @@ def start(self): print("app.config:") print(self.config) print("try running with --help-all to see all available flags") - self.log.info("Info Mesage") - self.log.debug("DebugMessage") - self.log.critical("Warning") - self.log.critical("Critical mesage") + self.log.debug("Debug Message") + self.log.info("Info Message") + self.log.warning("Warning Message") + self.log.critical("Critical Message") def main(): diff --git a/traitlets/config/loader.py b/traitlets/config/loader.py index 414912b7..6afa1b7c 100644 --- a/traitlets/config/loader.py +++ b/traitlets/config/loader.py @@ -5,6 +5,7 @@ import argparse import copy +import functools import json import os import re @@ -1016,8 +1017,10 @@ def _add_arguments(self, aliases, flags, classes): "dest": traitname.replace(".", _DOT_REPLACEMENT), "metavar": traitname, } + argcompleter = None if traitname in argparse_traits: - argparse_kwds.update(argparse_traits[traitname][1]) + trait, kwds = argparse_traits[traitname] + argparse_kwds.update(kwds) if "action" in argparse_kwds and traitname in alias_flags: # flag sets 'action', so can't have flag & alias with custom action # on the same name @@ -1025,6 +1028,11 @@ def _add_arguments(self, aliases, flags, classes): "The alias `%s` for the 'append' sequence " "config-trait `%s` cannot be also a flag!'" % (key, traitname) ) + # For argcomplete, check if any either an argcompleter metadata tag or method + # is available. If so, it should be a callable which takes the command-line key + # string as an argument and other kwargs passed by argcomplete, + # and returns the a list of string completions. + argcompleter = trait.metadata.get("argcompleter") or getattr(trait, "argcompleter", None) if traitname in alias_flags: # alias and flag. # when called with 0 args: flag @@ -1034,7 +1042,11 @@ def _add_arguments(self, aliases, flags, classes): argparse_kwds["flag"] = alias_flags[traitname] argparse_kwds["alias"] = traitname keys = ("-" + key, "--" + key) if len(key) == 1 else ("--" + key,) - paa(*keys, **argparse_kwds) + action = paa(*keys, **argparse_kwds) + if argcompleter is not None: + # argcomplete's completers are callables returning list of completion strings + action.completer = functools.partial(argcompleter, key=key) + self.argcomplete() def _convert_to_config(self): """self.parsed_data->self.config, parse unrecognized extra args via KVLoader.""" @@ -1084,6 +1096,12 @@ def _handle_unrecognized_alias(self, arg: str) -> None: """ self.log.warning("Unrecognized alias: '%s', it will have no effect.", arg) + def argcomplete(self): + try: + import argcomplete + argcomplete.autocomplete(self.parser) + except ImportError: + pass class KeyValueConfigLoader(KVArgParseConfigLoader): """Deprecated in traitlets 5.0 diff --git a/traitlets/traitlets.py b/traitlets/traitlets.py index 73133842..49ecb338 100644 --- a/traitlets/traitlets.py +++ b/traitlets/traitlets.py @@ -2619,6 +2619,13 @@ def from_string(self, s): def subclass_init(self, cls): pass # fully opt out of instance_init + def argcompleter(self, **kwargs): + """Completion hints for argcomplete""" + completions = ["true", "1", "false", "0"] + if self.allow_none: + completions.append("None") + return completions + class CBool(Bool): """A casting version of the boolean trait.""" @@ -2673,6 +2680,10 @@ def from_string(self, s): def subclass_init(self, cls): pass # fully opt out of instance_init + def argcompleter(self, **kwargs): + """Completion hints for argcomplete""" + return list(self.values) + class CaselessStrEnum(Enum): """An enum of strings where the case should be ignored.""" From 632a7b1fed3597525127f82c401a4a81ebd679c9 Mon Sep 17 00:00:00 2001 From: azhu Date: Fri, 9 Dec 2022 01:30:10 +1300 Subject: [PATCH 02/28] Custom argcomplete.CompletionFinder for class traits This custom finder mainly adds 2 functionalities: 1. When completing options, it will add --Class. to the list of completions, for each class in Application.classes that could complete the current option. 2. If it detects that we are currently trying to complete an option related to --Class., it will add the corresponding config traits of Class to the ArgumentParser instance, so that the traits' completers can be used. (This is currently done in a bit of a hacky manner.) Note that we are avoiding adding all config traits of all classes to the ArgumentParser, which would be easier but would add more runtime overhead and would also make completions appear more spammy. It also does not support nested class options like --Class1.Class2.trait. Example: ~/dev/traitlets$ examples/myapp.py --mode on --[TAB] --Application. --Foo. --debug --enable --help --j --mode --running --Bar. --MyApp. --disable --enabled --i --log_level --name ~/dev/traitlets$ examples/myapp.py --mode on --F[TAB] ~/dev/traitlets$ examples/myapp.py --mode on --Foo.[TAB] --Foo.i --Foo.j --Foo.mode --Foo.name ~/dev/traitlets$ examples/myapp.py --mode on --Foo.m[TAB] ~/dev/traitlets$ examples/myapp.py --mode on --Foo.mode [TAB] ~/dev/traitlets$ examples/myapp.py --mode on --Foo.mode o[TAB] off on other --- traitlets/config/argcomplete_config.py | 105 +++++++++++++++++++++++++ traitlets/config/loader.py | 13 +-- 2 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 traitlets/config/argcomplete_config.py diff --git a/traitlets/config/argcomplete_config.py b/traitlets/config/argcomplete_config.py new file mode 100644 index 00000000..ebb56d38 --- /dev/null +++ b/traitlets/config/argcomplete_config.py @@ -0,0 +1,105 @@ +import typing as t + +import argcomplete + +class ExtendedCompletionFinder(argcomplete.CompletionFinder): + """An extension of CompletionFinder which dynamically completes class-trait based options + + This finder mainly adds 2 functionalities: + + 1. When completing options, it will add --Class. to the list of completions, for each + class in Application.classes that could complete the current option. + 2. If it detects that we are currently trying to complete an option related to --Class., + it will add the corresponding config traits of Class to the ArgumentParser instance, + so that the traits' completers can be used. + + Note that we are avoiding adding all config traits of all classes to the ArgumentParser, + which would be easier but would add more runtime overhead and would also make completions + appear more spammy. + + These changes do require using the internals of argcomplete.CompletionFinder. + """ + def match_class_completions(self, cword_prefix: str) -> t.List[t.Tuple[t.Any, str]]: + """Match the word to be completed against our Configurable classes + + Check if cword_prefix could potentially match against --{class}. for any class + in Application.classes. + """ + class_completions = [(cls, f"--{cls.__name__}.") for cls in self.config_classes] + matched_completions = class_completions + if "." in cword_prefix: + cword_prefix = cword_prefix[:cword_prefix.index(".") + 1] + matched_completions = [(cls, c) for (cls, c) in class_completions if c == cword_prefix] + elif len(cword_prefix) > 0: + matched_completions = [(cls, c) for (cls, c) in class_completions if c.startswith(cword_prefix)] + return matched_completions + + def inject_class_to_parser(self, cls): + """Add dummy arguments to our ArgumentParser for the traits of this class + + The argparse-based loader currently does not actually add any class traits to + the constructed ArgumentParser, only the flags & aliaes. In order to work nicely + with argcomplete's completers functionality, this method adds dummy arguments + of the form --Class.trait to the ArgumentParser instance. + + This method should be called selectively to reduce runtime overhead and to avoid + spamming options across all of Application.classes. + """ + try: + for traitname, trait in cls.class_traits(config=True).items(): + completer = trait.metadata.get("argcompleter") or getattr(trait, "argcompleter", None) + self._parser.add_argument( + f"--{cls.__name__}.{traitname}", + type=str, + help=trait.help, + # metavar=traitname, + ).completer = completer + argcomplete.debug(f"added --{cls.__name__}.{traitname}") + except AttributeError: + pass + + def _get_completions(self, comp_words, cword_prefix, *args): + """Overriden to dynamically append --Class.trait arguments if appropriate + + Warning: + This does not (currently) support completions of the form + --Class1.Class2.<...>.trait, although this is valid for traitlets. + Part of the reason is that we don't currently have a way to identify + which classes may be used with Class1 as a parent. + """ + prefix_chars = self._parser.prefix_chars + is_option = len(cword_prefix) > 0 and cword_prefix[0] in prefix_chars + if is_option: + # If we are currently completing an option, check if it could + # match with any of the --Class. completions. If there's exactly + # one matched class, then expand out the --Class.trait options. + matched_completions = self.match_class_completions(cword_prefix) + if len(matched_completions) == 1: + matched_cls = matched_completions[0][0] + self.inject_class_to_parser(matched_cls) + elif len(comp_words) > 0 and "." in comp_words[-1] and not is_option: + # If not an option, perform a hacky check to see if we are completing + # an argument for a --Class.trait option. + # TODO: the branch condition is wrong here for multiplicity="+", need to fix + matched_completions = self.match_class_completions(comp_words[-1]) + if matched_completions: + matched_cls = matched_completions[0][0] + self.inject_class_to_parser(matched_cls) + + return super()._get_completions(comp_words, cword_prefix, *args) + + def _get_option_completions(self, parser, cword_prefix): + """Overriden to add --Class. completions when appropriate""" + completions = super()._get_option_completions(parser, cword_prefix) + if cword_prefix.endswith("."): + return completions + + matched_completions = self.match_class_completions(cword_prefix) + if len(matched_completions) > 1: + completions.extend(opt for cls, opt in matched_completions) + # If there is exactly one match, we would expect it to have aleady + # been handled by the options dynamically added in _get_completions(). + # However, there could be edge cases, for example if the matched class + # has no configurable traits. + return completions + diff --git a/traitlets/config/loader.py b/traitlets/config/loader.py index 6afa1b7c..361bd2d0 100644 --- a/traitlets/config/loader.py +++ b/traitlets/config/loader.py @@ -997,7 +997,7 @@ def _add_arguments(self, aliases, flags, classes): argparse_kwds["nargs"] = multiplicity argparse_traits[argname] = (trait, argparse_kwds) - for keys, (value, _) in flags.items(): + for keys, (value, fhelp) in flags.items(): if not isinstance(keys, tuple): keys = (keys,) for key in keys: @@ -1005,7 +1005,7 @@ def _add_arguments(self, aliases, flags, classes): alias_flags[aliases[key]] = value continue keys = ("-" + key, "--" + key) if len(key) == 1 else ("--" + key,) - paa(*keys, action=_FlagAction, flag=value) + paa(*keys, action=_FlagAction, flag=value, help=fhelp) for keys, traitname in aliases.items(): if not isinstance(keys, tuple): @@ -1046,7 +1046,7 @@ def _add_arguments(self, aliases, flags, classes): if argcompleter is not None: # argcomplete's completers are callables returning list of completion strings action.completer = functools.partial(argcompleter, key=key) - self.argcomplete() + self.argcomplete(classes) def _convert_to_config(self): """self.parsed_data->self.config, parse unrecognized extra args via KVLoader.""" @@ -1096,10 +1096,13 @@ def _handle_unrecognized_alias(self, arg: str) -> None: """ self.log.warning("Unrecognized alias: '%s', it will have no effect.", arg) - def argcomplete(self): + def argcomplete(self, classes: t.List[t.Any]): try: import argcomplete - argcomplete.autocomplete(self.parser) + from . import argcomplete_config + finder = argcomplete_config.ExtendedCompletionFinder() + finder.config_classes = classes # type: ignore + finder(self.parser) except ImportError: pass From e36f12c68cb85316912e6e7b17bbd17c7dc9f7a3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 8 Dec 2022 12:56:52 +0000 Subject: [PATCH 03/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- examples/myapp.py | 1 + traitlets/config/argcomplete_config.py | 13 +++++++++---- traitlets/config/loader.py | 7 ++++++- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/examples/myapp.py b/examples/myapp.py index 973ff58d..e54c2bf9 100755 --- a/examples/myapp.py +++ b/examples/myapp.py @@ -35,6 +35,7 @@ from traitlets.config.application import Application from traitlets.config.configurable import Configurable + class Foo(Configurable): """A class that has configurable, typed attributes.""" diff --git a/traitlets/config/argcomplete_config.py b/traitlets/config/argcomplete_config.py index ebb56d38..7c6298b0 100644 --- a/traitlets/config/argcomplete_config.py +++ b/traitlets/config/argcomplete_config.py @@ -2,6 +2,7 @@ import argcomplete + class ExtendedCompletionFinder(argcomplete.CompletionFinder): """An extension of CompletionFinder which dynamically completes class-trait based options @@ -19,6 +20,7 @@ class in Application.classes that could complete the current option. These changes do require using the internals of argcomplete.CompletionFinder. """ + def match_class_completions(self, cword_prefix: str) -> t.List[t.Tuple[t.Any, str]]: """Match the word to be completed against our Configurable classes @@ -28,10 +30,12 @@ def match_class_completions(self, cword_prefix: str) -> t.List[t.Tuple[t.Any, st class_completions = [(cls, f"--{cls.__name__}.") for cls in self.config_classes] matched_completions = class_completions if "." in cword_prefix: - cword_prefix = cword_prefix[:cword_prefix.index(".") + 1] + cword_prefix = cword_prefix[: cword_prefix.index(".") + 1] matched_completions = [(cls, c) for (cls, c) in class_completions if c == cword_prefix] elif len(cword_prefix) > 0: - matched_completions = [(cls, c) for (cls, c) in class_completions if c.startswith(cword_prefix)] + matched_completions = [ + (cls, c) for (cls, c) in class_completions if c.startswith(cword_prefix) + ] return matched_completions def inject_class_to_parser(self, cls): @@ -47,7 +51,9 @@ def inject_class_to_parser(self, cls): """ try: for traitname, trait in cls.class_traits(config=True).items(): - completer = trait.metadata.get("argcompleter") or getattr(trait, "argcompleter", None) + completer = trait.metadata.get("argcompleter") or getattr( + trait, "argcompleter", None + ) self._parser.add_argument( f"--{cls.__name__}.{traitname}", type=str, @@ -102,4 +108,3 @@ def _get_option_completions(self, parser, cword_prefix): # However, there could be edge cases, for example if the matched class # has no configurable traits. return completions - diff --git a/traitlets/config/loader.py b/traitlets/config/loader.py index 361bd2d0..ed133ddd 100644 --- a/traitlets/config/loader.py +++ b/traitlets/config/loader.py @@ -1032,7 +1032,9 @@ def _add_arguments(self, aliases, flags, classes): # is available. If so, it should be a callable which takes the command-line key # string as an argument and other kwargs passed by argcomplete, # and returns the a list of string completions. - argcompleter = trait.metadata.get("argcompleter") or getattr(trait, "argcompleter", None) + argcompleter = trait.metadata.get("argcompleter") or getattr( + trait, "argcompleter", None + ) if traitname in alias_flags: # alias and flag. # when called with 0 args: flag @@ -1099,13 +1101,16 @@ def _handle_unrecognized_alias(self, arg: str) -> None: def argcomplete(self, classes: t.List[t.Any]): try: import argcomplete + from . import argcomplete_config + finder = argcomplete_config.ExtendedCompletionFinder() finder.config_classes = classes # type: ignore finder(self.parser) except ImportError: pass + class KeyValueConfigLoader(KVArgParseConfigLoader): """Deprecated in traitlets 5.0 From c0c6aab9dd4c20a39a31fdc926a0f6dd89c6ea1d Mon Sep 17 00:00:00 2001 From: azhu Date: Fri, 9 Dec 2022 23:27:20 +1300 Subject: [PATCH 04/28] Example application for testing argcomplete Added an example application for testing argcomplete, under examples/argcomplete_app.py, with several examples of completions provided in the docstring. Fixed using completers --Class.trait arg1 arg2 [TAB] for config traits with nargs/multiplicity="+". Note that currently traitlets does not support multiplicity even though it is used in the code; refer to issue GH#690 for discussion. Add more comments since we're using argcomplete internals, and add argcomplete as dependency for coverage/mypy tests. Some other minor fixes such as minor mypy annotations fixes. Another example: add # PYTHON_ARGCOMPLETE_OK to bin/ipython, and tab-complete away: $ ipython --[TAB] --Application. --help --BaseFormatter. --i --BaseIPythonApplication. --ignore-cwd --Completer. --init --HistoryAccessor. --ipython-dir --HistoryManager. --log-level --IPCompleter. --logappend --InteractiveShell. --logfile --InteractiveShellApp. --m ... $ ipython --gui=[TAB] asyncio gtk gtk3 pyglet qt4 tk glut gtk2 osx qt qt5 wx To-do still: support subcommands. This may still take some work as traitlets does subcommand parsing independently of argparse. --- examples/argcomplete_app.py | 154 +++++++++++++++++++++ pyproject.toml | 4 +- traitlets/config/application.py | 1 + traitlets/config/argcomplete_config.py | 38 +++-- traitlets/config/loader.py | 6 +- traitlets/config/tests/test_application.py | 34 ++++- 6 files changed, 220 insertions(+), 17 deletions(-) create mode 100755 examples/argcomplete_app.py diff --git a/examples/argcomplete_app.py b/examples/argcomplete_app.py new file mode 100755 index 00000000..cca25201 --- /dev/null +++ b/examples/argcomplete_app.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python +# PYTHON_ARGCOMPLETE_OK +"""A simple example to test CLI completion with `traitlets.Application` + +Follow the installation instructions in +https://github.com/kislyuk/argcomplete#installation +to install argcomplete. For example for bash, you can set up global completion +via something like:: + + $ activate-global-python-argcomplete --dest=~/.bash_completion.d + +and ``source ~/.bash_completion.d`` in your ``~/.bashrc``. To use the +`global-python-argcomplete`, your `traitlets.Application`-based script should +have the string ``PYTHON_ARGCOMPLETE_OK`` in the first few lines of the script. + +Afterwards, try tab completing options to this script:: + + # Option completion, show flags, aliases and --Class. + $ examples/argcomplete_app.py --[TAB] + --Application. --JsonPrinter. --env-vars --s + --ArgcompleteApp. --e --help --skip-if-missing + --EnvironPrinter. --env-var --json-indent --style + + $ examples/argcomplete_app.py --A[TAB] + --Application. --ArgcompleteApp. + + # Complete class config traits + $ examples/argcomplete_app.py --EnvironPrinter.[TAB] + --EnvironPrinter.no_complete --EnvironPrinter.style + --EnvironPrinter.skip_if_missing --EnvironPrinter.vars + + # Using argcomplete's provided EnvironCompleter + $ examples/argcomplete_app.py --EnvironPrinter.vars=[TAB] + APPDATA LS_COLORS + COMP_LINE NAME + COMP_POINT OLDPWD + COMP_TYPE PATH + ... + + $ examples/argcomplete_app.py --EnvironPrinter.vars USER [TAB] + APPDATA LS_COLORS + COMP_LINE NAME + COMP_POINT OLDPWD + COMP_TYPE PATH + ... + + # Alias for --EnvironPrinter.vars + $ examples/argcomplete_app.py --env-vars P[TAB] + PATH PWD PYTHONPATH + + # Custom completer example + $ examples/argcomplete_app.py --env-vars PWD --json-indent [TAB] + 2 4 8 + + # Enum completer example + $ examples/argcomplete_app.py --style [TAB] + ndjson posix verbose + + # Bool completer example + $ examples/argcomplete_app.py --Application.show_config_json [TAB] + 0 1 false true + +If completions are not showing, you can set the environment variable ``_ARC_DEBUG=1`` +to assist in debugging argcomplete. This was last checked with ``argcomplete==1.12.3``. +""" +import json +import os + +from argcomplete.completers import EnvironCompleter, SuppressCompleter + +from traitlets import Bool, Dict, Enum, Int, List, Unicode +from traitlets.config.application import Application +from traitlets.config.configurable import Configurable + + +def _indent_completions(**kwargs): + """Example of a custom completer, which could be dynamic""" + return ["2", "4", "8"] + + +class JsonPrinter(Configurable): + indent = Int(None, allow_none=True).tag(config=True, argcompleter=_indent_completions) + + def print(self, obj): + print(json.dumps(obj, indent=self.indent)) + + +class EnvironPrinter(Configurable): + """A class that has configurable, typed attributes.""" + + vars = List(trait=Unicode(), help="Environment variable").tag( + # NOTE: currently multiplicity is ignored by the traitlets CLI. + # Refer to issue GH#690 for discussion + config=True, + multiplicity="+", + argcompleter=EnvironCompleter, + ) + no_complete = Unicode().tag(config=True, argcompleter=SuppressCompleter) + style = Enum(values=["posix", "ndjson", "verbose"], default_value="posix").tag(config=True) + skip_if_missing = Bool(False, help="Skip variable if not set").tag(config=True) + + def print(self): + for env_var in self.vars: + if env_var not in os.environ: + if self.skip_if_missing: + continue + else: + raise KeyError(f"Environment variable not set: {env_var}") + + value = os.environ[env_var] + if self.style == "posix": + print(f"{env_var}={value}") + elif self.style == "verbose": + print(f">> key: {env_var} value:\n{value}\n") + elif self.style == "ndjson": + JsonPrinter(parent=self).print({"key": env_var, "value": value}) + + +def bool_flag(trait, value=True): + return ({trait.this_class.__name__: {trait.name: value}}, trait.help) + + +class ArgcompleteApp(Application): + name = Unicode("argcomplete-example-app") + description = Unicode("prints requested environment variables") + classes = List([JsonPrinter, EnvironPrinter]) + + config_file = Unicode("", help="Load this config file").tag(config=True) + + aliases = Dict( + { # type:ignore[assignment] + ("e", "env-var", "env-vars"): "EnvironPrinter.vars", + ("s", "style"): "EnvironPrinter.style", + ("json-indent"): "JsonPrinter.indent", + } + ) + + flags = Dict( + { # type:ignore[assignment] + "skip-if-missing": bool_flag(EnvironPrinter.skip_if_missing), + } + ) + + def initialize(self, argv=None): + self.parse_command_line(argv) + if self.config_file: + self.load_config_file(self.config_file) + + def start(self): + EnvironPrinter(parent=self).print() + + +if __name__ == "__main__": + ArgcompleteApp.launch_instance() diff --git a/pyproject.toml b/pyproject.toml index 6afe1e49..9c885852 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,14 +47,14 @@ nowarn = "test -W default {args}" [tool.hatch.envs.cov] features = ["test"] -dependencies = ["coverage", "pytest-cov"] +dependencies = ["coverage", "pytest-cov", "argcomplete"] [tool.hatch.envs.cov.scripts] test = "python -m pytest -vv --cov traitlets --cov-branch --cov-report term-missing:skip-covered {args}" nowarn = "test -W default {args}" [tool.hatch.envs.typing] features = ["test", "typing"] -dependencies = ["mypy>=0.990"] +dependencies = ["mypy>=0.990", "argcomplete"] [tool.hatch.envs.typing.scripts] test = "mypy --install-types --non-interactive {args:.}" diff --git a/traitlets/config/application.py b/traitlets/config/application.py index 66f56f72..23e3aec8 100644 --- a/traitlets/config/application.py +++ b/traitlets/config/application.py @@ -525,6 +525,7 @@ def emit_alias_help(self): for c in cls.mro()[:-3]: classdict[c.__name__] = c + fhelp: t.Optional[str] for alias, longname in self.aliases.items(): try: if isinstance(longname, tuple): diff --git a/traitlets/config/argcomplete_config.py b/traitlets/config/argcomplete_config.py index 7c6298b0..fa352c7e 100644 --- a/traitlets/config/argcomplete_config.py +++ b/traitlets/config/argcomplete_config.py @@ -1,3 +1,4 @@ +import argparse import typing as t import argcomplete @@ -20,6 +21,8 @@ class in Application.classes that could complete the current option. These changes do require using the internals of argcomplete.CompletionFinder. """ + _parser: argparse.ArgumentParser + config_classes: t.List[t.Any] # Configurables def match_class_completions(self, cword_prefix: str) -> t.List[t.Tuple[t.Any, str]]: """Match the word to be completed against our Configurable classes @@ -54,17 +57,19 @@ def inject_class_to_parser(self, cls): completer = trait.metadata.get("argcompleter") or getattr( trait, "argcompleter", None ) + multiplicity = trait.metadata.get("multiplicity") self._parser.add_argument( f"--{cls.__name__}.{traitname}", type=str, help=trait.help, + nargs=multiplicity, # metavar=traitname, - ).completer = completer + ).completer = completer # type: ignore argcomplete.debug(f"added --{cls.__name__}.{traitname}") except AttributeError: pass - def _get_completions(self, comp_words, cword_prefix, *args): + def _get_completions(self, comp_words: t.List[str], cword_prefix: str, *args) -> t.List[str]: """Overriden to dynamically append --Class.trait arguments if appropriate Warning: @@ -72,7 +77,13 @@ def _get_completions(self, comp_words, cword_prefix, *args): --Class1.Class2.<...>.trait, although this is valid for traitlets. Part of the reason is that we don't currently have a way to identify which classes may be used with Class1 as a parent. + + Warning: + This is an internal method in CompletionFinder and so the API might + be subject to drift. """ + # Try to identify if we are completing something related to --Class. for + # a known Class, if we are then add the Class config traits to our ArgumentParser. prefix_chars = self._parser.prefix_chars is_option = len(cword_prefix) > 0 and cword_prefix[0] in prefix_chars if is_option: @@ -85,16 +96,21 @@ def _get_completions(self, comp_words, cword_prefix, *args): self.inject_class_to_parser(matched_cls) elif len(comp_words) > 0 and "." in comp_words[-1] and not is_option: # If not an option, perform a hacky check to see if we are completing - # an argument for a --Class.trait option. - # TODO: the branch condition is wrong here for multiplicity="+", need to fix - matched_completions = self.match_class_completions(comp_words[-1]) - if matched_completions: - matched_cls = matched_completions[0][0] - self.inject_class_to_parser(matched_cls) + # an argument for an already present --Class.trait option. Search backwards + # for last option (based on last word starting with prefix_chars), and see + # if it is of the form --Class.trait. Note that if multiplicity="+", these + # arguments might conflict with positional arguments. + for prev_word in comp_words[::-1]: + if len(prev_word) > 0 and prev_word[0] in prefix_chars: + matched_completions = self.match_class_completions(prev_word) + if matched_completions: + matched_cls = matched_completions[0][0] + self.inject_class_to_parser(matched_cls) + break return super()._get_completions(comp_words, cword_prefix, *args) - def _get_option_completions(self, parser, cword_prefix): + def _get_option_completions(self, parser: argparse.ArgumentParser, cword_prefix: str) -> t.List[str]: """Overriden to add --Class. completions when appropriate""" completions = super()._get_option_completions(parser, cword_prefix) if cword_prefix.endswith("."): @@ -105,6 +121,6 @@ def _get_option_completions(self, parser, cword_prefix): completions.extend(opt for cls, opt in matched_completions) # If there is exactly one match, we would expect it to have aleady # been handled by the options dynamically added in _get_completions(). - # However, there could be edge cases, for example if the matched class - # has no configurable traits. + # However, maybe there's an edge cases missed here, for example if the + # matched class has no configurable traits. return completions diff --git a/traitlets/config/loader.py b/traitlets/config/loader.py index ed133ddd..0e5af6b8 100644 --- a/traitlets/config/loader.py +++ b/traitlets/config/loader.py @@ -1047,7 +1047,7 @@ def _add_arguments(self, aliases, flags, classes): action = paa(*keys, **argparse_kwds) if argcompleter is not None: # argcomplete's completers are callables returning list of completion strings - action.completer = functools.partial(argcompleter, key=key) + action.completer = functools.partial(argcompleter, key=key) # type: ignore self.argcomplete(classes) def _convert_to_config(self): @@ -1098,14 +1098,14 @@ def _handle_unrecognized_alias(self, arg: str) -> None: """ self.log.warning("Unrecognized alias: '%s', it will have no effect.", arg) - def argcomplete(self, classes: t.List[t.Any]): + def argcomplete(self, classes: t.List[t.Any]) -> None: try: import argcomplete from . import argcomplete_config finder = argcomplete_config.ExtendedCompletionFinder() - finder.config_classes = classes # type: ignore + finder.config_classes = classes finder(self.parser) except ImportError: pass diff --git a/traitlets/config/tests/test_application.py b/traitlets/config/tests/test_application.py index 16224683..1f52f10e 100644 --- a/traitlets/config/tests/test_application.py +++ b/traitlets/config/tests/test_application.py @@ -13,7 +13,7 @@ import sys import typing as t from io import StringIO -from tempfile import TemporaryDirectory +from tempfile import TemporaryFile, TemporaryDirectory from unittest import TestCase import pytest @@ -672,6 +672,38 @@ def test_loaded_config_files(self): self.assertEqual(app.running, False) +class TestArgcomplete: + IFS = "\013" + COMP_WORDBREAKS = " \t\n\"'><=;|&(:" + + @pytest.fixture + def argcomplete_on(self): + _old_environ = os.environ + os.environ = os.environ.copy() + os.environ["_ARGCOMPLETE"] = "1" + os.environ["_ARC_DEBUG"] = "yes" + os.environ["IFS"] = self.IFS + os.environ["_ARGCOMPLETE_COMP_WORDBREAKS"] = self.COMP_WORDBREAKS + os.environ["_ARGCOMPLETE"] = "1" + yield + os.environ = _old_environ + + def run_completer(self, app, command, point=None, **kwargs): + if point is None: + point = str(len(command)) + with TemporaryFile(mode="w+") as t: + os.environ["COMP_LINE"] = command + os.environ["COMP_POINT"] = point + with pytest.raises(SystemExit) as cm: + app.initialize(command, output_stream=t, exit_method=sys.exit, **kwargs) + if cm.exception.code != 0: + raise Exception("Unexpected exit code %d" % cm.exception.code) + t.seek(0) + return t.read().split(self.IFS) + + # TODO: need to pass through output_stream to argcomplete to create unit tests + + def test_cli_multi_scalar(caplog): class App(Application): aliases = {"opt": "App.opt"} From 6522562c1d03fb2a27c5190fd55aff076390cb50 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 9 Dec 2022 10:41:02 +0000 Subject: [PATCH 05/28] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- traitlets/config/argcomplete_config.py | 5 ++++- traitlets/config/loader.py | 1 - traitlets/config/tests/test_application.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/traitlets/config/argcomplete_config.py b/traitlets/config/argcomplete_config.py index fa352c7e..72b45ac9 100644 --- a/traitlets/config/argcomplete_config.py +++ b/traitlets/config/argcomplete_config.py @@ -21,6 +21,7 @@ class in Application.classes that could complete the current option. These changes do require using the internals of argcomplete.CompletionFinder. """ + _parser: argparse.ArgumentParser config_classes: t.List[t.Any] # Configurables @@ -110,7 +111,9 @@ def _get_completions(self, comp_words: t.List[str], cword_prefix: str, *args) -> return super()._get_completions(comp_words, cword_prefix, *args) - def _get_option_completions(self, parser: argparse.ArgumentParser, cword_prefix: str) -> t.List[str]: + def _get_option_completions( + self, parser: argparse.ArgumentParser, cword_prefix: str + ) -> t.List[str]: """Overriden to add --Class. completions when appropriate""" completions = super()._get_option_completions(parser, cword_prefix) if cword_prefix.endswith("."): diff --git a/traitlets/config/loader.py b/traitlets/config/loader.py index 0e5af6b8..601001c6 100644 --- a/traitlets/config/loader.py +++ b/traitlets/config/loader.py @@ -1100,7 +1100,6 @@ def _handle_unrecognized_alias(self, arg: str) -> None: def argcomplete(self, classes: t.List[t.Any]) -> None: try: - import argcomplete from . import argcomplete_config diff --git a/traitlets/config/tests/test_application.py b/traitlets/config/tests/test_application.py index 1f52f10e..b42899a5 100644 --- a/traitlets/config/tests/test_application.py +++ b/traitlets/config/tests/test_application.py @@ -13,7 +13,7 @@ import sys import typing as t from io import StringIO -from tempfile import TemporaryFile, TemporaryDirectory +from tempfile import TemporaryDirectory, TemporaryFile from unittest import TestCase import pytest From 032eabc9bf3fe34e548bd09dc5f35ba19ce09bd1 Mon Sep 17 00:00:00 2001 From: azhu Date: Mon, 12 Dec 2022 09:40:51 +1300 Subject: [PATCH 06/28] Initial traitlets subcommand support for argcomplete argcomplete's strategy is to call the python script with no arguments e.g. len(sys.argv) == 1, run until the ArgumentParser is constructed and determine what completions are available. On the other hand, traitlet's subcommand-handling strategy is to check sys.argv[1] and see if it matches a subcommand, and if so then dynamically load the subcommand app and initialize it with sys.argv[1:]. Write a couple of helper functions to reconcile this by: 1. retrieving tokens from $COMP_LINES, etc, and setting it to argv 2. if traitlets descends into a subcommand, increment index passed via env var to argcomplete to mark where command starts There's quite a few caveats to this approach. For example, it only is evaluated currently when `App.initialize()` is passed with `argv=None` (the default). If `argv` is explicitly passed, then the `argcomplete`-specific handling is skipped currently. More details in: https://github.com/ipython/traitlets/pull/811#issuecomment-1345535450 Some additional minor cleanup with respect to subcommands typing, examples, and documentation. --- docs/source/config.rst | 4 +- examples/argcomplete_app.py | 5 +- examples/subcommands_app.py | 74 ++++++++++++++++++++++++++ traitlets/config/application.py | 50 ++++++++++++++++- traitlets/config/argcomplete_config.py | 66 +++++++++++++++++++++-- traitlets/config/configurable.py | 2 +- 6 files changed, 191 insertions(+), 10 deletions(-) create mode 100755 examples/subcommands_app.py diff --git a/docs/source/config.rst b/docs/source/config.rst index 72dc0089..d8e4ad34 100644 --- a/docs/source/config.rst +++ b/docs/source/config.rst @@ -429,10 +429,10 @@ instances, mapping *subcommand names* to two-tuples containing these: .. note:: The return value of the factory above is an *instance*, not a class, - son the :meth:`SingletonConfigurable.instance()` is not invoked + so the :meth:`SingletonConfigurable.instance()` is not invoked in this case. - In all cases, the instanciated app is stored in :attr:`Application.subapp` + In all cases, the instantiated app is stored in :attr:`Application.subapp` and its :meth:`Application.initialize()` is invoked. 2. A short description of the subcommand for use in help output. diff --git a/examples/argcomplete_app.py b/examples/argcomplete_app.py index cca25201..f3c6592b 100755 --- a/examples/argcomplete_app.py +++ b/examples/argcomplete_app.py @@ -66,7 +66,10 @@ import json import os -from argcomplete.completers import EnvironCompleter, SuppressCompleter +try: + from argcomplete.completers import EnvironCompleter, SuppressCompleter +except ImportError: + EnvironCompleter = SuppressCompleter = None from traitlets import Bool, Dict, Enum, Int, List, Unicode from traitlets.config.application import Application diff --git a/examples/subcommands_app.py b/examples/subcommands_app.py new file mode 100755 index 00000000..bd383b5e --- /dev/null +++ b/examples/subcommands_app.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python +# PYTHON_ARGCOMPLETE_OK +"""A simple example to demonstrate subcommands with traitlets.Application + +Example: + + $ examples/subcommands_app.py foo --print-name alice + foo + hello alice + + $ examples/subcommands_app.py bar --print-name bob + bar + hello bob +""" + +from traitlets import Enum, List, Unicode +from traitlets.config.application import Application +from traitlets.config.configurable import Configurable + +class PrintHello(Configurable): + greet_name = Unicode("world").tag(config=True) + greeting = Enum(values=["hello", "hi", "bye"], default_value="hello").tag(config=True) + + def run(self): + print(f"{self.greeting} {self.greet_name}") + +class FooApp(Application): + name = Unicode("foo") + classes = List([PrintHello]) + aliases = { + "print-name": "PrintHello.greet_name", + } + + config_file = Unicode("", help="Load this config file").tag(config=True) + + def start(self): + print(self.name) + PrintHello(parent=self).run() + +class BarApp(Application): + name = Unicode("bar") + classes = List([PrintHello]) + aliases = { + "print-name": "PrintHello.greet_name", + } + + config_file = Unicode("", help="Load this config file").tag(config=True) + + def start(self): + print(self.name) + PrintHello(parent=self).run() + + @classmethod + def get_subapp(cls, main_app: Application) -> Application: + main_app.clear_instance() + return cls.instance(parent=main_app) + +class MainApp(Application): + name = Unicode("subcommand-example-app") + description = Unicode("demonstrates app with subcommands") + subcommands = { + # Subcommands should be a dictionary mapping from the subcommand name + # to one of the following: + # 1. The Application class to be instantiated e.g. FooApp + # 2. A string e.g. "traitlets.examples.subcommands_app.FooApp" + # which will be lazily evaluated + # 3. A callable which takes this Application and returns an instance + # (not class) of the subcommmand Application + "foo": (FooApp, "run foo"), + "bar": (BarApp.get_subapp, "run bar"), + } + +if __name__ == "__main__": + MainApp.launch_instance() \ No newline at end of file diff --git a/traitlets/config/application.py b/traitlets/config/application.py index 23e3aec8..aa3efc72 100644 --- a/traitlets/config/application.py +++ b/traitlets/config/application.py @@ -400,7 +400,7 @@ def _log_default(self): # this must be a dict of two-tuples, # the first element being the application class/import string # and the second being the help string for the subcommand - subcommands: t.Union[t.Dict[str, t.Tuple[str, str]], Dict] = Dict() + subcommands: t.Union[t.Dict[str, t.Tuple[t.Any, str]], Dict] = Dict() # parse_command_line will initialize a subapp, if requested subapp = Instance("traitlets.config.application.Application", allow_none=True) @@ -787,11 +787,56 @@ def flatten_flags(self): def _create_loader(self, argv, aliases, flags, classes): return KVArgParseConfigLoader(argv, aliases, flags, classes=classes, log=self.log) + @classmethod + def _get_sys_argv(cls, check_argcomplete=False) -> t.List[str]: + """Get `sys.argv` or equivalent from `argcomplete` + + `argcomplete`'s strategy is to call the python script with no arguments, + so ``len(sys.argv) == 1``, and run until the `ArgumentParser` is constructed + and determine what completions are available. + + On the other hand, `traitlet`'s subcommand-handling strategy is to check + ``sys.argv[1]`` and see if it matches a subcommand, and if so then dynamically + load the subcommand app and initialize it with ``sys.argv[1:]``. + + This helper method helps to take the current tokens for `argcomplete` and pass + them through as `argv`. + """ + if check_argcomplete and "_ARGCOMPLETE" in os.environ: + try: + from traitlets.config.argcomplete_config import get_argcomplete_cwords + cwords = get_argcomplete_cwords() + assert cwords is not None + return cwords + except (ImportError, ModuleNotFoundError): + pass + return sys.argv + + @classmethod + def _handle_argcomplete_for_subcommand(cls): + """Helper for `argcomplete` to recognize `traitlets` subcommands + + `argcomplete` does not know that `traitlets` has already consumed subcommands, + as it only "sees" the final `argparse.ArgumentParser` that is constructed. + (Indeed `KVArgParseConfigLoader` does not get passed subcommands at all currently.) + We explicitly manipulate the environment variables used internally by `argcomplete` + to get it to skip over the subcommand tokens. + """ + if "_ARGCOMPLETE" not in os.environ: + return + + try: + from traitlets.config.argcomplete_config import increment_argcomplete_index + increment_argcomplete_index() + except (ImportError, ModuleNotFoundError): + pass + @catch_config_error def parse_command_line(self, argv=None): """Parse the command line arguments.""" assert not isinstance(argv, str) - argv = sys.argv[1:] if argv is None else argv + if argv is None: + argv = self._get_sys_argv(check_argcomplete=bool(self.subcommands))[1:] self.argv = [cast_unicode(arg) for arg in argv] if argv and argv[0] == "help": @@ -803,6 +848,7 @@ def parse_command_line(self, argv=None): subc, subargv = argv[0], argv[1:] if re.match(r"^\w(\-?\w)*$", subc) and subc in self.subcommands: # it's a subcommand, and *not* a flag or class parameter + self._handle_argcomplete_for_subcommand() return self.initialize_subcommand(subc, subargv) # Arguments after a '--' argument are for the script IPython may be diff --git a/traitlets/config/argcomplete_config.py b/traitlets/config/argcomplete_config.py index 72b45ac9..7c522095 100644 --- a/traitlets/config/argcomplete_config.py +++ b/traitlets/config/argcomplete_config.py @@ -1,10 +1,68 @@ +"""Helper utilities for integrating argcomplete with traitlets""" import argparse +import os import typing as t -import argcomplete - +try: + import argcomplete + from argcomplete import CompletionFinder +except ImportError: + # This module and its utility methods are written to not crash even + # if argcomplete is not installed. + class StubModule: + def __getattr__(self, attr): + raise ModuleNotFoundError("No module named 'argcomplete'") + + argcomplete = StubModule() + CompletionFinder = object + +def get_argcomplete_cwords() -> t.Optional[t.List[str]]: + """Get current words prior to completion point + + This is normally done in the `argcomplete.CompletionFinder` constructor, + but is exposed here to allow `traitlets` to follow dynamic code-paths such + as determining whether to evaluate a subcommand. + """ + if "_ARGCOMPLETE" not in os.environ: + return None + + comp_line = os.environ["COMP_LINE"] + comp_point = int(os.environ["COMP_POINT"]) + try: + comp_line = argcomplete.ensure_str(comp_line) + except ModuleNotFoundError: + return None + # argcomplete.debug("splitting COMP_LINE for:", comp_line, comp_point) + cword_prequote, cword_prefix, cword_suffix, comp_words, last_wordbreak_pos = argcomplete.split_line(comp_line, comp_point) + + # _ARGCOMPLETE is set by the shell script to tell us where comp_words + # should start, based on what we're completing. + # 1: