-
-
Notifications
You must be signed in to change notification settings - Fork 199
/
loader.py
1124 lines (897 loc) · 35.5 KB
/
loader.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""A simple configuration system."""
# Copyright (c) IPython Development Team.
# Distributed under the terms of the Modified BSD License.
import argparse
import copy
import json
import os
import re
import sys
import typing as t
import warnings
from traitlets.traitlets import Any, Container, Dict, HasTraits, List, Undefined
from ..utils import cast_unicode, filefind
# -----------------------------------------------------------------------------
# Exceptions
# -----------------------------------------------------------------------------
class ConfigError(Exception):
pass
class ConfigLoaderError(ConfigError):
pass
class ConfigFileNotFound(ConfigError): # noqa
pass
class ArgumentError(ConfigLoaderError):
pass
# -----------------------------------------------------------------------------
# Argparse fix
# -----------------------------------------------------------------------------
# Unfortunately argparse by default prints help messages to stderr instead of
# stdout. This makes it annoying to capture long help screens at the command
# line, since one must know how to pipe stderr, which many users don't know how
# to do. So we override the print_help method with one that defaults to
# stdout and use our class instead.
class _Sentinel:
def __repr__(self):
return "<Sentinel deprecated>"
def __str__(self):
return "<deprecated>"
_deprecated = _Sentinel()
class ArgumentParser(argparse.ArgumentParser):
"""Simple argparse subclass that prints help to stdout by default."""
def print_help(self, file=None):
if file is None:
file = sys.stdout
return super().print_help(file)
print_help.__doc__ = argparse.ArgumentParser.print_help.__doc__
# -----------------------------------------------------------------------------
# Config class for holding config information
# -----------------------------------------------------------------------------
def execfile(fname, glob):
with open(fname, "rb") as f:
exec(compile(f.read(), fname, "exec"), glob, glob) # noqa
class LazyConfigValue(HasTraits):
"""Proxy object for exposing methods on configurable containers
These methods allow appending/extending/updating
to add to non-empty defaults instead of clobbering them.
Exposes:
- append, extend, insert on lists
- update on dicts
- update, add on sets
"""
_value = None
# list methods
_extend = List()
_prepend = List()
_inserts = List()
def append(self, obj):
"""Append an item to a List"""
self._extend.append(obj)
def extend(self, other):
"""Extend a list"""
self._extend.extend(other)
def prepend(self, other):
"""like list.extend, but for the front"""
self._prepend[:0] = other
def merge_into(self, other):
"""
Merge with another earlier LazyConfigValue or an earlier container.
This is useful when having global system-wide configuration files.
Self is expected to have higher precedence.
Parameters
----------
other : LazyConfigValue or container
Returns
-------
LazyConfigValue
if ``other`` is also lazy, a reified container otherwise.
"""
if isinstance(other, LazyConfigValue):
other._extend.extend(self._extend)
self._extend = other._extend
self._prepend.extend(other._prepend)
other._inserts.extend(self._inserts)
self._inserts = other._inserts
if self._update:
other.update(self._update)
self._update = other._update
return self
else:
# other is a container, reify now.
return self.get_value(other)
def insert(self, index, other):
if not isinstance(index, int):
raise TypeError("An integer is required")
self._inserts.append((index, other))
# dict methods
# update is used for both dict and set
_update = Any()
def update(self, other):
"""Update either a set or dict"""
if self._update is None:
if isinstance(other, dict):
self._update = {}
else:
self._update = set()
self._update.update(other)
# set methods
def add(self, obj):
"""Add an item to a set"""
self.update({obj})
def get_value(self, initial):
"""construct the value from the initial one
after applying any insert / extend / update changes
"""
if self._value is not None:
return self._value
value = copy.deepcopy(initial)
if isinstance(value, list):
for idx, obj in self._inserts:
value.insert(idx, obj)
value[:0] = self._prepend
value.extend(self._extend)
elif isinstance(value, dict):
if self._update:
value.update(self._update)
elif isinstance(value, set):
if self._update:
value.update(self._update)
self._value = value
return value
def to_dict(self):
"""return JSONable dict form of my data
Currently update as dict or set, extend, prepend as lists, and inserts as list of tuples.
"""
d = {}
if self._update:
d["update"] = self._update
if self._extend:
d["extend"] = self._extend
if self._prepend:
d["prepend"] = self._prepend
elif self._inserts:
d["inserts"] = self._inserts
return d
def __repr__(self):
if self._value is not None:
return f"<{self.__class__.__name__} value={self._value!r}>"
else:
return f"<{self.__class__.__name__} {self.to_dict()!r}>"
def _is_section_key(key):
"""Is a Config key a section name (does it start with a capital)?"""
if key and key[0].upper() == key[0] and not key.startswith("_"):
return True
else:
return False
class Config(dict): # type:ignore[type-arg]
"""An attribute-based dict that can do smart merges.
Accessing a field on a config object for the first time populates the key
with either a nested Config object for keys starting with capitals
or :class:`.LazyConfigValue` for lowercase keys,
allowing quick assignments such as::
c = Config()
c.Class.int_trait = 5
c.Class.list_trait.append("x")
"""
def __init__(self, *args, **kwds):
dict.__init__(self, *args, **kwds)
self._ensure_subconfig()
def _ensure_subconfig(self):
"""ensure that sub-dicts that should be Config objects are
casts dicts that are under section keys to Config objects,
which is necessary for constructing Config objects from dict literals.
"""
for key in self:
obj = self[key]
if _is_section_key(key) and isinstance(obj, dict) and not isinstance(obj, Config):
setattr(self, key, Config(obj))
def _merge(self, other):
"""deprecated alias, use Config.merge()"""
self.merge(other)
def merge(self, other):
"""merge another config object into this one"""
to_update = {}
for k, v in other.items():
if k not in self:
to_update[k] = v
else: # I have this key
if isinstance(v, Config) and isinstance(self[k], Config):
# Recursively merge common sub Configs
self[k].merge(v)
elif isinstance(v, LazyConfigValue):
self[k] = v.merge_into(self[k])
else:
# Plain updates for non-Configs
to_update[k] = v
self.update(to_update)
def collisions(self, other: "Config") -> t.Dict[str, t.Any]:
"""Check for collisions between two config objects.
Returns a dict of the form {"Class": {"trait": "collision message"}}`,
indicating which values have been ignored.
An empty dict indicates no collisions.
"""
collisions: t.Dict[str, t.Any] = {}
for section in self:
if section not in other:
continue
mine = self[section]
theirs = other[section]
for key in mine:
if key in theirs and mine[key] != theirs[key]:
collisions.setdefault(section, {})
collisions[section][key] = f"{mine[key]!r} ignored, using {theirs[key]!r}"
return collisions
def __contains__(self, key):
# allow nested contains of the form `"Section.key" in config`
if "." in key:
first, remainder = key.split(".", 1)
if first not in self:
return False
return remainder in self[first]
return super().__contains__(key)
# .has_key is deprecated for dictionaries.
has_key = __contains__
def _has_section(self, key):
return _is_section_key(key) and key in self
def copy(self):
return type(self)(dict.copy(self))
def __copy__(self):
return self.copy()
def __deepcopy__(self, memo):
new_config = type(self)()
for key, value in self.items():
if isinstance(value, (Config, LazyConfigValue)):
# deep copy config objects
value = copy.deepcopy(value, memo)
elif type(value) in {dict, list, set, tuple}:
# shallow copy plain container traits
value = copy.copy(value)
new_config[key] = value
return new_config
def __getitem__(self, key):
try:
return dict.__getitem__(self, key)
except KeyError:
if _is_section_key(key):
c = Config()
dict.__setitem__(self, key, c)
return c
elif not key.startswith("_"):
# undefined, create lazy value, used for container methods
v = LazyConfigValue()
dict.__setitem__(self, key, v)
return v
else:
raise KeyError
def __setitem__(self, key, value):
if _is_section_key(key):
if not isinstance(value, Config):
raise ValueError(
"values whose keys begin with an uppercase "
"char must be Config instances: %r, %r" % (key, value)
)
dict.__setitem__(self, key, value)
def __getattr__(self, key):
if key.startswith("__"):
return dict.__getattr__(self, key) # type:ignore[attr-defined]
try:
return self.__getitem__(key)
except KeyError as e:
raise AttributeError(e) from e
def __setattr__(self, key, value):
if key.startswith("__"):
return dict.__setattr__(self, key, value)
try:
self.__setitem__(key, value)
except KeyError as e:
raise AttributeError(e) from e
def __delattr__(self, key):
if key.startswith("__"):
return dict.__delattr__(self, key)
try:
dict.__delitem__(self, key)
except KeyError as e:
raise AttributeError(e) from e
class DeferredConfig:
"""Class for deferred-evaluation of config from CLI"""
pass
def get_value(self, trait):
raise NotImplementedError("Implement in subclasses")
def _super_repr(self):
# explicitly call super on direct parent
return super(self.__class__, self).__repr__()
class DeferredConfigString(str, DeferredConfig):
"""Config value for loading config from a string
Interpretation is deferred until it is loaded into the trait.
Subclass of str for backward compatibility.
This class is only used for values that are not listed
in the configurable classes.
When config is loaded, `trait.from_string` will be used.
If an error is raised in `.from_string`,
the original string is returned.
.. versionadded:: 5.0
"""
def get_value(self, trait):
"""Get the value stored in this string"""
s = str(self)
try:
return trait.from_string(s)
except Exception:
# exception casting from string,
# let the original string lie.
# this will raise a more informative error when config is loaded.
return s
def __repr__(self):
return f"{self.__class__.__name__}({self._super_repr()})"
class DeferredConfigList(list, DeferredConfig): # type:ignore[type-arg]
"""Config value for loading config from a list of strings
Interpretation is deferred until it is loaded into the trait.
This class is only used for values that are not listed
in the configurable classes.
When config is loaded, `trait.from_string_list` will be used.
If an error is raised in `.from_string_list`,
the original string list is returned.
.. versionadded:: 5.0
"""
def get_value(self, trait):
"""Get the value stored in this string"""
if hasattr(trait, "from_string_list"):
src = list(self)
cast = trait.from_string_list
else:
# only allow one item
if len(self) > 1:
raise ValueError(
f"{trait.name} only accepts one value, got {len(self)}: {list(self)}"
)
src = self[0]
cast = trait.from_string
try:
return cast(src)
except Exception:
# exception casting from string,
# let the original value lie.
# this will raise a more informative error when config is loaded.
return src
def __repr__(self):
return f"{self.__class__.__name__}({self._super_repr()})"
# -----------------------------------------------------------------------------
# Config loading classes
# -----------------------------------------------------------------------------
class ConfigLoader:
"""A object for loading configurations from just about anywhere.
The resulting configuration is packaged as a :class:`Config`.
Notes
-----
A :class:`ConfigLoader` does one thing: load a config from a source
(file, command line arguments) and returns the data as a :class:`Config` object.
There are lots of things that :class:`ConfigLoader` does not do. It does
not implement complex logic for finding config files. It does not handle
default values or merge multiple configs. These things need to be
handled elsewhere.
"""
def _log_default(self):
from traitlets.log import get_logger
return get_logger()
def __init__(self, log=None):
"""A base class for config loaders.
log : instance of :class:`logging.Logger` to use.
By default logger of :meth:`traitlets.config.application.Application.instance()`
will be used
Examples
--------
>>> cl = ConfigLoader()
>>> config = cl.load_config()
>>> config
{}
"""
self.clear()
if log is None:
self.log = self._log_default()
self.log.debug("Using default logger")
else:
self.log = log
def clear(self):
self.config = Config()
def load_config(self):
"""Load a config from somewhere, return a :class:`Config` instance.
Usually, this will cause self.config to be set and then returned.
However, in most cases, :meth:`ConfigLoader.clear` should be called
to erase any previous state.
"""
self.clear()
return self.config
class FileConfigLoader(ConfigLoader):
"""A base class for file based configurations.
As we add more file based config loaders, the common logic should go
here.
"""
def __init__(self, filename, path=None, **kw):
"""Build a config loader for a filename and path.
Parameters
----------
filename : str
The file name of the config file.
path : str, list, tuple
The path to search for the config file on, or a sequence of
paths to try in order.
"""
super().__init__(**kw)
self.filename = filename
self.path = path
self.full_filename = ""
def _find_file(self):
"""Try to find the file by searching the paths."""
self.full_filename = filefind(self.filename, self.path)
class JSONFileConfigLoader(FileConfigLoader):
"""A JSON file loader for config
Can also act as a context manager that rewrite the configuration file to disk on exit.
Example::
with JSONFileConfigLoader('myapp.json','/home/jupyter/configurations/') as c:
c.MyNewConfigurable.new_value = 'Updated'
"""
def load_config(self):
"""Load the config from a file and return it as a Config object."""
self.clear()
try:
self._find_file()
except OSError as e:
raise ConfigFileNotFound(str(e)) from e
dct = self._read_file_as_dict()
self.config = self._convert_to_config(dct)
return self.config
def _read_file_as_dict(self):
with open(self.full_filename) as f:
return json.load(f)
def _convert_to_config(self, dictionary):
if "version" in dictionary:
version = dictionary.pop("version")
else:
version = 1
if version == 1:
return Config(dictionary)
else:
raise ValueError(f"Unknown version of JSON config file: {version}")
def __enter__(self):
self.load_config()
return self.config
def __exit__(self, exc_type, exc_value, traceback):
"""
Exit the context manager but do not handle any errors.
In case of any error, we do not want to write the potentially broken
configuration to disk.
"""
self.config.version = 1
json_config = json.dumps(self.config, indent=2)
with open(self.full_filename, "w") as f:
f.write(json_config)
class PyFileConfigLoader(FileConfigLoader):
"""A config loader for pure python files.
This is responsible for locating a Python config file by filename and
path, then executing it to construct a Config object.
"""
def load_config(self):
"""Load the config from a file and return it as a Config object."""
self.clear()
try:
self._find_file()
except OSError as e:
raise ConfigFileNotFound(str(e)) from e
self._read_file_as_dict()
return self.config
def load_subconfig(self, fname, path=None):
"""Injected into config file namespace as load_subconfig"""
if path is None:
path = self.path
loader = self.__class__(fname, path)
try:
sub_config = loader.load_config()
except ConfigFileNotFound:
# Pass silently if the sub config is not there,
# treat it as an empty config file.
pass
else:
self.config.merge(sub_config)
def _read_file_as_dict(self):
"""Load the config file into self.config, with recursive loading."""
def get_config():
"""Unnecessary now, but a deprecation warning is more trouble than it's worth."""
return self.config
namespace = dict(
c=self.config,
load_subconfig=self.load_subconfig,
get_config=get_config,
__file__=self.full_filename,
)
conf_filename = self.full_filename
with open(conf_filename, "rb") as f:
exec(compile(f.read(), conf_filename, "exec"), namespace, namespace) # noqa
class CommandLineConfigLoader(ConfigLoader):
"""A config loader for command line arguments.
As we add more command line based loaders, the common logic should go
here.
"""
def _exec_config_str(self, lhs, rhs, trait=None):
"""execute self.config.<lhs> = <rhs>
* expands ~ with expanduser
* interprets value with trait if available
"""
value = rhs
if isinstance(value, DeferredConfig):
if trait:
# trait available, reify config immediately
value = value.get_value(trait)
elif isinstance(rhs, DeferredConfigList) and len(rhs) == 1:
# single item, make it a deferred str
value = DeferredConfigString(os.path.expanduser(rhs[0]))
else:
if trait:
value = trait.from_string(value)
else:
value = DeferredConfigString(value)
*path, key = lhs.split(".")
section = self.config
for part in path:
section = section[part]
section[key] = value
return
def _load_flag(self, cfg):
"""update self.config from a flag, which can be a dict or Config"""
if isinstance(cfg, (dict, Config)):
# don't clobber whole config sections, update
# each section from config:
for sec, c in cfg.items():
self.config[sec].update(c)
else:
raise TypeError("Invalid flag: %r" % cfg)
# match --Class.trait keys for argparse
# matches:
# --Class.trait
# --x
# -x
class_trait_opt_pattern = re.compile(r"^\-?\-[A-Za-z][\w]*(\.[\w]+)*$")
_DOT_REPLACEMENT = "__DOT__"
_DASH_REPLACEMENT = "__DASH__"
class _KVAction(argparse.Action):
"""Custom argparse action for handling --Class.trait=x
Always
"""
def __call__(self, parser, namespace, values, option_string=None):
if isinstance(values, str):
values = [values]
values = ["-" if v is _DASH_REPLACEMENT else v for v in values]
items = getattr(namespace, self.dest, None)
if items is None:
items = DeferredConfigList()
else:
items = DeferredConfigList(items)
items.extend(values)
setattr(namespace, self.dest, items)
class _DefaultOptionDict(dict): # type:ignore[type-arg]
"""Like the default options dict
but acts as if all --Class.trait options are predefined
"""
def _add_kv_action(self, key):
self[key] = _KVAction(
option_strings=[key],
dest=key.lstrip("-").replace(".", _DOT_REPLACEMENT),
# use metavar for display purposes
metavar=key.lstrip("-"),
)
def __contains__(self, key):
if "=" in key:
return False
if super().__contains__(key):
return True
if key.startswith("-") and class_trait_opt_pattern.match(key):
self._add_kv_action(key)
return True
return False
def __getitem__(self, key):
if key in self:
return super().__getitem__(key)
else:
raise KeyError(key)
def get(self, key, default=None):
try:
return self[key]
except KeyError:
return default
class _KVArgParser(argparse.ArgumentParser):
"""subclass of ArgumentParser where any --Class.trait option is implicitly defined"""
def parse_known_args(self, args=None, namespace=None):
# must be done immediately prior to parsing because if we do it in init,
# registration of explicit actions via parser.add_option will fail during setup
for container in (self, self._optionals):
container._option_string_actions = _DefaultOptionDict(container._option_string_actions)
return super().parse_known_args(args, namespace)
class ArgParseConfigLoader(CommandLineConfigLoader):
"""A loader that uses the argparse module to load from the command line."""
parser_class = ArgumentParser
Flags = t.Union[str, t.Tuple[str, ...]]
def __init__(
self,
argv: t.Optional[t.List[str]] = None,
aliases: t.Optional[t.Dict[Flags, str]] = None,
flags: t.Optional[t.Dict[Flags, str]] = None,
log: t.Any = None,
classes: t.Optional[t.List[t.Type[t.Any]]] = None,
*parser_args: t.Any,
**parser_kw: t.Any,
) -> None:
"""Create a config loader for use with argparse.
Parameters
----------
classes : optional, list
The classes to scan for *container* config-traits and decide
for their "multiplicity" when adding them as *argparse* arguments.
argv : optional, list
If given, used to read command-line arguments from, otherwise
sys.argv[1:] is used.
*parser_args : tuple
A tuple of positional arguments that will be passed to the
constructor of :class:`argparse.ArgumentParser`.
**parser_kw : dict
A tuple of keyword arguments that will be passed to the
constructor of :class:`argparse.ArgumentParser`.
aliases : dict of str to str
Dict of aliases to full traitlets names for CLI parsing
flags : dict of str to str
Dict of flags to full traitlets names for CLI parsing
log
Passed to `ConfigLoader`
Returns
-------
config : Config
The resulting Config object.
"""
classes = classes or []
super(CommandLineConfigLoader, self).__init__(log=log)
self.clear()
if argv is None:
argv = sys.argv[1:]
self.argv = argv
self.aliases = aliases or {}
self.flags = flags or {}
self.classes = classes
self.parser_args = parser_args
self.version = parser_kw.pop("version", None)
kwargs = dict(argument_default=argparse.SUPPRESS)
kwargs.update(parser_kw)
self.parser_kw = kwargs
def load_config(self, argv=None, aliases=None, flags=_deprecated, classes=None):
"""Parse command line arguments and return as a Config object.
Parameters
----------
argv : optional, list
If given, a list with the structure of sys.argv[1:] to parse
arguments from. If not given, the instance's self.argv attribute
(given at construction time) is used.
flags
Deprecated in traitlets 5.0, instanciate the config loader with the flags.
"""
if flags is not _deprecated:
warnings.warn(
"The `flag` argument to load_config is deprecated since Traitlets "
f"5.0 and will be ignored, pass flags the `{type(self)}` constructor.",
DeprecationWarning,
stacklevel=2,
)
self.clear()
if argv is None:
argv = self.argv
if aliases is not None:
self.aliases = aliases
if classes is not None:
self.classes = classes
self._create_parser()
self._parse_args(argv)
self._convert_to_config()
return self.config
def get_extra_args(self):
if hasattr(self, "extra_args"):
return self.extra_args
else:
return []
def _create_parser(self):
self.parser = self.parser_class(
*self.parser_args, **self.parser_kw # type:ignore[arg-type]
)
self._add_arguments(self.aliases, self.flags, self.classes)
def _add_arguments(self, aliases, flags, classes):
raise NotImplementedError("subclasses must implement _add_arguments")
def _parse_args(self, args):
"""self.parser->self.parsed_data"""
uargs = [cast_unicode(a) for a in args]
unpacked_aliases: t.Dict[str, str] = {}
if self.aliases:
unpacked_aliases = {}
for alias, alias_target in self.aliases.items():
if alias in self.flags:
continue
if not isinstance(alias, tuple):
alias = (alias,)
for al in alias:
if len(al) == 1:
unpacked_aliases["-" + al] = "--" + alias_target
unpacked_aliases["--" + al] = "--" + alias_target
def _replace(arg):
if arg == "-":
return _DASH_REPLACEMENT
for k, v in unpacked_aliases.items():
if arg == k:
return v
if arg.startswith(k + "="):
return v + "=" + arg[len(k) + 1 :]
return arg
if "--" in uargs:
idx = uargs.index("--")
extra_args = uargs[idx + 1 :]
to_parse = uargs[:idx]
else:
extra_args = []
to_parse = uargs
to_parse = [_replace(a) for a in to_parse]
self.parsed_data = self.parser.parse_args(to_parse)
self.extra_args = extra_args
def _convert_to_config(self):
"""self.parsed_data->self.config"""
for k, v in vars(self.parsed_data).items():
*path, key = k.split(".")
section = self.config
for p in path:
section = section[p]
setattr(section, key, v)
class _FlagAction(argparse.Action):
"""ArgParse action to handle a flag"""
def __init__(self, *args, **kwargs):
self.flag = kwargs.pop("flag")
self.alias = kwargs.pop("alias", None)
kwargs["const"] = Undefined
if not self.alias:
kwargs["nargs"] = 0
super().__init__(*args, **kwargs)
def __call__(self, parser, namespace, values, option_string=None):
if self.nargs == 0 or values is Undefined:
if not hasattr(namespace, "_flags"):
namespace._flags = []
namespace._flags.append(self.flag)
else:
setattr(namespace, self.alias, values)
class KVArgParseConfigLoader(ArgParseConfigLoader):
"""A config loader that loads aliases and flags with argparse,
as well as arbitrary --Class.trait value
"""
parser_class = _KVArgParser # type:ignore[assignment]
def _add_arguments(self, aliases, flags, classes):
alias_flags: t.Dict[str, t.Any] = {}
argparse_kwds: t.Dict[str, t.Any]
paa = self.parser.add_argument
self.parser.set_defaults(_flags=[])
paa("extra_args", nargs="*")
# An index of all container traits collected::
#
# { <traitname>: (<trait>, <argparse-kwds>) }
#
# Used to add the correct type into the `config` tree.
# Used also for aliases, not to re-collect them.
self.argparse_traits = argparse_traits = {}
for cls in classes:
for traitname, trait in cls.class_traits(config=True).items():
argname = f"{cls.__name__}.{traitname}"
argparse_kwds = {"type": str}
if isinstance(trait, (Container, Dict)):
multiplicity = trait.metadata.get("multiplicity", "append")
if multiplicity == "append":
argparse_kwds["action"] = multiplicity
else:
argparse_kwds["nargs"] = multiplicity
argparse_traits[argname] = (trait, argparse_kwds)
for keys, (value, _) in flags.items():
if not isinstance(keys, tuple):
keys = (keys,)