-
Notifications
You must be signed in to change notification settings - Fork 2.4k
/
trans.py
2064 lines (1701 loc) · 74.7 KB
/
trans.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
String transformers that can split and merge strings.
"""
from abc import ABC, abstractmethod
from collections import defaultdict
from dataclasses import dataclass
import regex as re # We need recursive patterns here (?R)
from typing import (
Any,
Callable,
ClassVar,
Collection,
Dict,
Iterable,
Iterator,
List,
Optional,
Sequence,
Set,
Tuple,
TypeVar,
Union,
)
import sys
if sys.version_info < (3, 8):
from typing_extensions import Final
else:
from typing import Final
from mypy_extensions import trait
from black.rusty import Result, Ok, Err
from black.mode import Feature
from black.nodes import syms, replace_child, parent_type
from black.nodes import is_empty_par, is_empty_lpar, is_empty_rpar
from black.nodes import OPENING_BRACKETS, CLOSING_BRACKETS, STANDALONE_COMMENT
from black.lines import Line, append_leaves
from black.brackets import BracketMatchError
from black.comments import contains_pragma_comment
from black.strings import has_triple_quotes, get_string_prefix, assert_is_leaf_string
from black.strings import normalize_string_quotes
from blib2to3.pytree import Leaf, Node
from blib2to3.pgen2 import token
class CannotTransform(Exception):
"""Base class for errors raised by Transformers."""
# types
T = TypeVar("T")
LN = Union[Leaf, Node]
Transformer = Callable[[Line, Collection[Feature]], Iterator[Line]]
Index = int
NodeType = int
ParserState = int
StringID = int
TResult = Result[T, CannotTransform] # (T)ransform Result
TMatchResult = TResult[Index]
def TErr(err_msg: str) -> Err[CannotTransform]:
"""(T)ransform Err
Convenience function used when working with the TResult type.
"""
cant_transform = CannotTransform(err_msg)
return Err(cant_transform)
class StringTransformer(ABC):
"""
An implementation of the Transformer protocol that relies on its
subclasses overriding the template methods `do_match(...)` and
`do_transform(...)`.
This Transformer works exclusively on strings (for example, by merging
or splitting them).
The following sections can be found among the docstrings of each concrete
StringTransformer subclass.
Requirements:
Which requirements must be met of the given Line for this
StringTransformer to be applied?
Transformations:
If the given Line meets all of the above requirements, which string
transformations can you expect to be applied to it by this
StringTransformer?
Collaborations:
What contractual agreements does this StringTransformer have with other
StringTransfomers? Such collaborations should be eliminated/minimized
as much as possible.
"""
__name__: Final = "StringTransformer"
# Ideally this would be a dataclass, but unfortunately mypyc breaks when used with
# `abc.ABC`.
def __init__(self, line_length: int, normalize_strings: bool) -> None:
self.line_length = line_length
self.normalize_strings = normalize_strings
@abstractmethod
def do_match(self, line: Line) -> TMatchResult:
"""
Returns:
* Ok(string_idx) such that `line.leaves[string_idx]` is our target
string, if a match was able to be made.
OR
* Err(CannotTransform), if a match was not able to be made.
"""
@abstractmethod
def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]:
"""
Yields:
* Ok(new_line) where new_line is the new transformed line.
OR
* Err(CannotTransform) if the transformation failed for some reason. The
`do_match(...)` template method should usually be used to reject
the form of the given Line, but in some cases it is difficult to
know whether or not a Line meets the StringTransformer's
requirements until the transformation is already midway.
Side Effects:
This method should NOT mutate @line directly, but it MAY mutate the
Line's underlying Node structure. (WARNING: If the underlying Node
structure IS altered, then this method should NOT be allowed to
yield an CannotTransform after that point.)
"""
def __call__(self, line: Line, _features: Collection[Feature]) -> Iterator[Line]:
"""
StringTransformer instances have a call signature that mirrors that of
the Transformer type.
Raises:
CannotTransform(...) if the concrete StringTransformer class is unable
to transform @line.
"""
# Optimization to avoid calling `self.do_match(...)` when the line does
# not contain any string.
if not any(leaf.type == token.STRING for leaf in line.leaves):
raise CannotTransform("There are no strings in this line.")
match_result = self.do_match(line)
if isinstance(match_result, Err):
cant_transform = match_result.err()
raise CannotTransform(
f"The string transformer {self.__class__.__name__} does not recognize"
" this line as one that it can transform."
) from cant_transform
string_idx = match_result.ok()
for line_result in self.do_transform(line, string_idx):
if isinstance(line_result, Err):
cant_transform = line_result.err()
raise CannotTransform(
"StringTransformer failed while attempting to transform string."
) from cant_transform
line = line_result.ok()
yield line
@dataclass
class CustomSplit:
"""A custom (i.e. manual) string split.
A single CustomSplit instance represents a single substring.
Examples:
Consider the following string:
```
"Hi there friend."
" This is a custom"
f" string {split}."
```
This string will correspond to the following three CustomSplit instances:
```
CustomSplit(False, 16)
CustomSplit(False, 17)
CustomSplit(True, 16)
```
"""
has_prefix: bool
break_idx: int
@trait
class CustomSplitMapMixin:
"""
This mixin class is used to map merged strings to a sequence of
CustomSplits, which will then be used to re-split the strings iff none of
the resultant substrings go over the configured max line length.
"""
_Key: ClassVar = Tuple[StringID, str]
_CUSTOM_SPLIT_MAP: ClassVar[Dict[_Key, Tuple[CustomSplit, ...]]] = defaultdict(
tuple
)
@staticmethod
def _get_key(string: str) -> "CustomSplitMapMixin._Key":
"""
Returns:
A unique identifier that is used internally to map @string to a
group of custom splits.
"""
return (id(string), string)
def add_custom_splits(
self, string: str, custom_splits: Iterable[CustomSplit]
) -> None:
"""Custom Split Map Setter Method
Side Effects:
Adds a mapping from @string to the custom splits @custom_splits.
"""
key = self._get_key(string)
self._CUSTOM_SPLIT_MAP[key] = tuple(custom_splits)
def pop_custom_splits(self, string: str) -> List[CustomSplit]:
"""Custom Split Map Getter Method
Returns:
* A list of the custom splits that are mapped to @string, if any
exist.
OR
* [], otherwise.
Side Effects:
Deletes the mapping between @string and its associated custom
splits (which are returned to the caller).
"""
key = self._get_key(string)
custom_splits = self._CUSTOM_SPLIT_MAP[key]
del self._CUSTOM_SPLIT_MAP[key]
return list(custom_splits)
def has_custom_splits(self, string: str) -> bool:
"""
Returns:
True iff @string is associated with a set of custom splits.
"""
key = self._get_key(string)
return key in self._CUSTOM_SPLIT_MAP
class StringMerger(StringTransformer, CustomSplitMapMixin):
"""StringTransformer that merges strings together.
Requirements:
(A) The line contains adjacent strings such that ALL of the validation checks
listed in StringMerger.__validate_msg(...)'s docstring pass.
OR
(B) The line contains a string which uses line continuation backslashes.
Transformations:
Depending on which of the two requirements above where met, either:
(A) The string group associated with the target string is merged.
OR
(B) All line-continuation backslashes are removed from the target string.
Collaborations:
StringMerger provides custom split information to StringSplitter.
"""
def do_match(self, line: Line) -> TMatchResult:
LL = line.leaves
is_valid_index = is_valid_index_factory(LL)
for (i, leaf) in enumerate(LL):
if (
leaf.type == token.STRING
and is_valid_index(i + 1)
and LL[i + 1].type == token.STRING
):
return Ok(i)
if leaf.type == token.STRING and "\\\n" in leaf.value:
return Ok(i)
return TErr("This line has no strings that need merging.")
def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]:
new_line = line
rblc_result = self._remove_backslash_line_continuation_chars(
new_line, string_idx
)
if isinstance(rblc_result, Ok):
new_line = rblc_result.ok()
msg_result = self._merge_string_group(new_line, string_idx)
if isinstance(msg_result, Ok):
new_line = msg_result.ok()
if isinstance(rblc_result, Err) and isinstance(msg_result, Err):
msg_cant_transform = msg_result.err()
rblc_cant_transform = rblc_result.err()
cant_transform = CannotTransform(
"StringMerger failed to merge any strings in this line."
)
# Chain the errors together using `__cause__`.
msg_cant_transform.__cause__ = rblc_cant_transform
cant_transform.__cause__ = msg_cant_transform
yield Err(cant_transform)
else:
yield Ok(new_line)
@staticmethod
def _remove_backslash_line_continuation_chars(
line: Line, string_idx: int
) -> TResult[Line]:
"""
Merge strings that were split across multiple lines using
line-continuation backslashes.
Returns:
Ok(new_line), if @line contains backslash line-continuation
characters.
OR
Err(CannotTransform), otherwise.
"""
LL = line.leaves
string_leaf = LL[string_idx]
if not (
string_leaf.type == token.STRING
and "\\\n" in string_leaf.value
and not has_triple_quotes(string_leaf.value)
):
return TErr(
f"String leaf {string_leaf} does not contain any backslash line"
" continuation characters."
)
new_line = line.clone()
new_line.comments = line.comments.copy()
append_leaves(new_line, line, LL)
new_string_leaf = new_line.leaves[string_idx]
new_string_leaf.value = new_string_leaf.value.replace("\\\n", "")
return Ok(new_line)
def _merge_string_group(self, line: Line, string_idx: int) -> TResult[Line]:
"""
Merges string group (i.e. set of adjacent strings) where the first
string in the group is `line.leaves[string_idx]`.
Returns:
Ok(new_line), if ALL of the validation checks found in
__validate_msg(...) pass.
OR
Err(CannotTransform), otherwise.
"""
LL = line.leaves
is_valid_index = is_valid_index_factory(LL)
vresult = self._validate_msg(line, string_idx)
if isinstance(vresult, Err):
return vresult
# If the string group is wrapped inside an Atom node, we must make sure
# to later replace that Atom with our new (merged) string leaf.
atom_node = LL[string_idx].parent
# We will place BREAK_MARK in between every two substrings that we
# merge. We will then later go through our final result and use the
# various instances of BREAK_MARK we find to add the right values to
# the custom split map.
BREAK_MARK = "@@@@@ BLACK BREAKPOINT MARKER @@@@@"
QUOTE = LL[string_idx].value[-1]
def make_naked(string: str, string_prefix: str) -> str:
"""Strip @string (i.e. make it a "naked" string)
Pre-conditions:
* assert_is_leaf_string(@string)
Returns:
A string that is identical to @string except that
@string_prefix has been stripped, the surrounding QUOTE
characters have been removed, and any remaining QUOTE
characters have been escaped.
"""
assert_is_leaf_string(string)
RE_EVEN_BACKSLASHES = r"(?:(?<!\\)(?:\\\\)*)"
naked_string = string[len(string_prefix) + 1 : -1]
naked_string = re.sub(
"(" + RE_EVEN_BACKSLASHES + ")" + QUOTE, r"\1\\" + QUOTE, naked_string
)
return naked_string
# Holds the CustomSplit objects that will later be added to the custom
# split map.
custom_splits = []
# Temporary storage for the 'has_prefix' part of the CustomSplit objects.
prefix_tracker = []
# Sets the 'prefix' variable. This is the prefix that the final merged
# string will have.
next_str_idx = string_idx
prefix = ""
while (
not prefix
and is_valid_index(next_str_idx)
and LL[next_str_idx].type == token.STRING
):
prefix = get_string_prefix(LL[next_str_idx].value).lower()
next_str_idx += 1
# The next loop merges the string group. The final string will be
# contained in 'S'.
#
# The following convenience variables are used:
#
# S: string
# NS: naked string
# SS: next string
# NSS: naked next string
S = ""
NS = ""
num_of_strings = 0
next_str_idx = string_idx
while is_valid_index(next_str_idx) and LL[next_str_idx].type == token.STRING:
num_of_strings += 1
SS = LL[next_str_idx].value
next_prefix = get_string_prefix(SS).lower()
# If this is an f-string group but this substring is not prefixed
# with 'f'...
if "f" in prefix and "f" not in next_prefix:
# Then we must escape any braces contained in this substring.
SS = re.subf(r"(\{|\})", "{1}{1}", SS)
NSS = make_naked(SS, next_prefix)
has_prefix = bool(next_prefix)
prefix_tracker.append(has_prefix)
S = prefix + QUOTE + NS + NSS + BREAK_MARK + QUOTE
NS = make_naked(S, prefix)
next_str_idx += 1
S_leaf = Leaf(token.STRING, S)
if self.normalize_strings:
S_leaf.value = normalize_string_quotes(S_leaf.value)
# Fill the 'custom_splits' list with the appropriate CustomSplit objects.
temp_string = S_leaf.value[len(prefix) + 1 : -1]
for has_prefix in prefix_tracker:
mark_idx = temp_string.find(BREAK_MARK)
assert (
mark_idx >= 0
), "Logic error while filling the custom string breakpoint cache."
temp_string = temp_string[mark_idx + len(BREAK_MARK) :]
breakpoint_idx = mark_idx + (len(prefix) if has_prefix else 0) + 1
custom_splits.append(CustomSplit(has_prefix, breakpoint_idx))
string_leaf = Leaf(token.STRING, S_leaf.value.replace(BREAK_MARK, ""))
if atom_node is not None:
replace_child(atom_node, string_leaf)
# Build the final line ('new_line') that this method will later return.
new_line = line.clone()
for (i, leaf) in enumerate(LL):
if i == string_idx:
new_line.append(string_leaf)
if string_idx <= i < string_idx + num_of_strings:
for comment_leaf in line.comments_after(LL[i]):
new_line.append(comment_leaf, preformatted=True)
continue
append_leaves(new_line, line, [leaf])
self.add_custom_splits(string_leaf.value, custom_splits)
return Ok(new_line)
@staticmethod
def _validate_msg(line: Line, string_idx: int) -> TResult[None]:
"""Validate (M)erge (S)tring (G)roup
Transform-time string validation logic for __merge_string_group(...).
Returns:
* Ok(None), if ALL validation checks (listed below) pass.
OR
* Err(CannotTransform), if any of the following are true:
- The target string group does not contain ANY stand-alone comments.
- The target string is not in a string group (i.e. it has no
adjacent strings).
- The string group has more than one inline comment.
- The string group has an inline comment that appears to be a pragma.
- The set of all string prefixes in the string group is of
length greater than one and is not equal to {"", "f"}.
- The string group consists of raw strings.
"""
# We first check for "inner" stand-alone comments (i.e. stand-alone
# comments that have a string leaf before them AND after them).
for inc in [1, -1]:
i = string_idx
found_sa_comment = False
is_valid_index = is_valid_index_factory(line.leaves)
while is_valid_index(i) and line.leaves[i].type in [
token.STRING,
STANDALONE_COMMENT,
]:
if line.leaves[i].type == STANDALONE_COMMENT:
found_sa_comment = True
elif found_sa_comment:
return TErr(
"StringMerger does NOT merge string groups which contain "
"stand-alone comments."
)
i += inc
num_of_inline_string_comments = 0
set_of_prefixes = set()
num_of_strings = 0
for leaf in line.leaves[string_idx:]:
if leaf.type != token.STRING:
# If the string group is trailed by a comma, we count the
# comments trailing the comma to be one of the string group's
# comments.
if leaf.type == token.COMMA and id(leaf) in line.comments:
num_of_inline_string_comments += 1
break
if has_triple_quotes(leaf.value):
return TErr("StringMerger does NOT merge multiline strings.")
num_of_strings += 1
prefix = get_string_prefix(leaf.value).lower()
if "r" in prefix:
return TErr("StringMerger does NOT merge raw strings.")
set_of_prefixes.add(prefix)
if id(leaf) in line.comments:
num_of_inline_string_comments += 1
if contains_pragma_comment(line.comments[id(leaf)]):
return TErr("Cannot merge strings which have pragma comments.")
if num_of_strings < 2:
return TErr(
f"Not enough strings to merge (num_of_strings={num_of_strings})."
)
if num_of_inline_string_comments > 1:
return TErr(
f"Too many inline string comments ({num_of_inline_string_comments})."
)
if len(set_of_prefixes) > 1 and set_of_prefixes != {"", "f"}:
return TErr(f"Too many different prefixes ({set_of_prefixes}).")
return Ok(None)
class StringParenStripper(StringTransformer):
"""StringTransformer that strips surrounding parentheses from strings.
Requirements:
The line contains a string which is surrounded by parentheses and:
- The target string is NOT the only argument to a function call.
- The target string is NOT a "pointless" string.
- If the target string contains a PERCENT, the brackets are not
preceded or followed by an operator with higher precedence than
PERCENT.
Transformations:
The parentheses mentioned in the 'Requirements' section are stripped.
Collaborations:
StringParenStripper has its own inherent usefulness, but it is also
relied on to clean up the parentheses created by StringParenWrapper (in
the event that they are no longer needed).
"""
def do_match(self, line: Line) -> TMatchResult:
LL = line.leaves
is_valid_index = is_valid_index_factory(LL)
for (idx, leaf) in enumerate(LL):
# Should be a string...
if leaf.type != token.STRING:
continue
# If this is a "pointless" string...
if (
leaf.parent
and leaf.parent.parent
and leaf.parent.parent.type == syms.simple_stmt
):
continue
# Should be preceded by a non-empty LPAR...
if (
not is_valid_index(idx - 1)
or LL[idx - 1].type != token.LPAR
or is_empty_lpar(LL[idx - 1])
):
continue
# That LPAR should NOT be preceded by a function name or a closing
# bracket (which could be a function which returns a function or a
# list/dictionary that contains a function)...
if is_valid_index(idx - 2) and (
LL[idx - 2].type == token.NAME or LL[idx - 2].type in CLOSING_BRACKETS
):
continue
string_idx = idx
# Skip the string trailer, if one exists.
string_parser = StringParser()
next_idx = string_parser.parse(LL, string_idx)
# if the leaves in the parsed string include a PERCENT, we need to
# make sure the initial LPAR is NOT preceded by an operator with
# higher or equal precedence to PERCENT
if is_valid_index(idx - 2):
# mypy can't quite follow unless we name this
before_lpar = LL[idx - 2]
if token.PERCENT in {leaf.type for leaf in LL[idx - 1 : next_idx]} and (
(
before_lpar.type
in {
token.STAR,
token.AT,
token.SLASH,
token.DOUBLESLASH,
token.PERCENT,
token.TILDE,
token.DOUBLESTAR,
token.AWAIT,
token.LSQB,
token.LPAR,
}
)
or (
# only unary PLUS/MINUS
before_lpar.parent
and before_lpar.parent.type == syms.factor
and (before_lpar.type in {token.PLUS, token.MINUS})
)
):
continue
# Should be followed by a non-empty RPAR...
if (
is_valid_index(next_idx)
and LL[next_idx].type == token.RPAR
and not is_empty_rpar(LL[next_idx])
):
# That RPAR should NOT be followed by anything with higher
# precedence than PERCENT
if is_valid_index(next_idx + 1) and LL[next_idx + 1].type in {
token.DOUBLESTAR,
token.LSQB,
token.LPAR,
token.DOT,
}:
continue
return Ok(string_idx)
return TErr("This line has no strings wrapped in parens.")
def do_transform(self, line: Line, string_idx: int) -> Iterator[TResult[Line]]:
LL = line.leaves
string_parser = StringParser()
rpar_idx = string_parser.parse(LL, string_idx)
for leaf in (LL[string_idx - 1], LL[rpar_idx]):
if line.comments_after(leaf):
yield TErr(
"Will not strip parentheses which have comments attached to them."
)
return
new_line = line.clone()
new_line.comments = line.comments.copy()
try:
append_leaves(new_line, line, LL[: string_idx - 1])
except BracketMatchError:
# HACK: I believe there is currently a bug somewhere in
# right_hand_split() that is causing brackets to not be tracked
# properly by a shared BracketTracker.
append_leaves(new_line, line, LL[: string_idx - 1], preformatted=True)
string_leaf = Leaf(token.STRING, LL[string_idx].value)
LL[string_idx - 1].remove()
replace_child(LL[string_idx], string_leaf)
new_line.append(string_leaf)
append_leaves(
new_line, line, LL[string_idx + 1 : rpar_idx] + LL[rpar_idx + 1 :]
)
LL[rpar_idx].remove()
yield Ok(new_line)
class BaseStringSplitter(StringTransformer):
"""
Abstract class for StringTransformers which transform a Line's strings by splitting
them or placing them on their own lines where necessary to avoid going over
the configured line length.
Requirements:
* The target string value is responsible for the line going over the
line length limit. It follows that after all of black's other line
split methods have been exhausted, this line (or one of the resulting
lines after all line splits are performed) would still be over the
line_length limit unless we split this string.
AND
* The target string is NOT a "pointless" string (i.e. a string that has
no parent or siblings).
AND
* The target string is not followed by an inline comment that appears
to be a pragma.
AND
* The target string is not a multiline (i.e. triple-quote) string.
"""
STRING_OPERATORS: Final = [
token.EQEQUAL,
token.GREATER,
token.GREATEREQUAL,
token.LESS,
token.LESSEQUAL,
token.NOTEQUAL,
token.PERCENT,
token.PLUS,
token.STAR,
]
@abstractmethod
def do_splitter_match(self, line: Line) -> TMatchResult:
"""
BaseStringSplitter asks its clients to override this method instead of
`StringTransformer.do_match(...)`.
Follows the same protocol as `StringTransformer.do_match(...)`.
Refer to `help(StringTransformer.do_match)` for more information.
"""
def do_match(self, line: Line) -> TMatchResult:
match_result = self.do_splitter_match(line)
if isinstance(match_result, Err):
return match_result
string_idx = match_result.ok()
vresult = self._validate(line, string_idx)
if isinstance(vresult, Err):
return vresult
return match_result
def _validate(self, line: Line, string_idx: int) -> TResult[None]:
"""
Checks that @line meets all of the requirements listed in this classes'
docstring. Refer to `help(BaseStringSplitter)` for a detailed
description of those requirements.
Returns:
* Ok(None), if ALL of the requirements are met.
OR
* Err(CannotTransform), if ANY of the requirements are NOT met.
"""
LL = line.leaves
string_leaf = LL[string_idx]
max_string_length = self._get_max_string_length(line, string_idx)
if len(string_leaf.value) <= max_string_length:
return TErr(
"The string itself is not what is causing this line to be too long."
)
if not string_leaf.parent or [L.type for L in string_leaf.parent.children] == [
token.STRING,
token.NEWLINE,
]:
return TErr(
f"This string ({string_leaf.value}) appears to be pointless (i.e. has"
" no parent)."
)
if id(line.leaves[string_idx]) in line.comments and contains_pragma_comment(
line.comments[id(line.leaves[string_idx])]
):
return TErr(
"Line appears to end with an inline pragma comment. Splitting the line"
" could modify the pragma's behavior."
)
if has_triple_quotes(string_leaf.value):
return TErr("We cannot split multiline strings.")
return Ok(None)
def _get_max_string_length(self, line: Line, string_idx: int) -> int:
"""
Calculates the max string length used when attempting to determine
whether or not the target string is responsible for causing the line to
go over the line length limit.
WARNING: This method is tightly coupled to both StringSplitter and
(especially) StringParenWrapper. There is probably a better way to
accomplish what is being done here.
Returns:
max_string_length: such that `line.leaves[string_idx].value >
max_string_length` implies that the target string IS responsible
for causing this line to exceed the line length limit.
"""
LL = line.leaves
is_valid_index = is_valid_index_factory(LL)
# We use the shorthand "WMA4" in comments to abbreviate "We must
# account for". When giving examples, we use STRING to mean some/any
# valid string.
#
# Finally, we use the following convenience variables:
#
# P: The leaf that is before the target string leaf.
# N: The leaf that is after the target string leaf.
# NN: The leaf that is after N.
# WMA4 the whitespace at the beginning of the line.
offset = line.depth * 4
if is_valid_index(string_idx - 1):
p_idx = string_idx - 1
if (
LL[string_idx - 1].type == token.LPAR
and LL[string_idx - 1].value == ""
and string_idx >= 2
):
# If the previous leaf is an empty LPAR placeholder, we should skip it.
p_idx -= 1
P = LL[p_idx]
if P.type in self.STRING_OPERATORS:
# WMA4 a space and a string operator (e.g. `+ STRING` or `== STRING`).
offset += len(str(P)) + 1
if P.type == token.COMMA:
# WMA4 a space, a comma, and a closing bracket [e.g. `), STRING`].
offset += 3
if P.type in [token.COLON, token.EQUAL, token.PLUSEQUAL, token.NAME]:
# This conditional branch is meant to handle dictionary keys,
# variable assignments, 'return STRING' statement lines, and
# 'else STRING' ternary expression lines.
# WMA4 a single space.
offset += 1
# WMA4 the lengths of any leaves that came before that space,
# but after any closing bracket before that space.
for leaf in reversed(LL[: p_idx + 1]):
offset += len(str(leaf))
if leaf.type in CLOSING_BRACKETS:
break
if is_valid_index(string_idx + 1):
N = LL[string_idx + 1]
if N.type == token.RPAR and N.value == "" and len(LL) > string_idx + 2:
# If the next leaf is an empty RPAR placeholder, we should skip it.
N = LL[string_idx + 2]
if N.type == token.COMMA:
# WMA4 a single comma at the end of the string (e.g `STRING,`).
offset += 1
if is_valid_index(string_idx + 2):
NN = LL[string_idx + 2]
if N.type == token.DOT and NN.type == token.NAME:
# This conditional branch is meant to handle method calls invoked
# off of a string literal up to and including the LPAR character.
# WMA4 the '.' character.
offset += 1
if (
is_valid_index(string_idx + 3)
and LL[string_idx + 3].type == token.LPAR
):
# WMA4 the left parenthesis character.
offset += 1
# WMA4 the length of the method's name.
offset += len(NN.value)
has_comments = False
for comment_leaf in line.comments_after(LL[string_idx]):
if not has_comments:
has_comments = True
# WMA4 two spaces before the '#' character.
offset += 2
# WMA4 the length of the inline comment.
offset += len(comment_leaf.value)
max_string_length = self.line_length - offset
return max_string_length
def iter_fexpr_spans(s: str) -> Iterator[Tuple[int, int]]:
"""
Yields spans corresponding to expressions in a given f-string.
Spans are half-open ranges (left inclusive, right exclusive).
Assumes the input string is a valid f-string, but will not crash if the input
string is invalid.
"""
stack: List[int] = [] # our curly paren stack
i = 0
while i < len(s):
if s[i] == "{":
# if we're in a string part of the f-string, ignore escaped curly braces
if not stack and i + 1 < len(s) and s[i + 1] == "{":
i += 2
continue
stack.append(i)
i += 1
continue
if s[i] == "}":
if not stack:
i += 1
continue
j = stack.pop()
# we've made it back out of the expression! yield the span
if not stack:
yield (j, i + 1)
i += 1
continue
# if we're in an expression part of the f-string, fast forward through strings
# note that backslashes are not legal in the expression portion of f-strings
if stack:
delim = None
if s[i : i + 3] in ("'''", '"""'):
delim = s[i : i + 3]
elif s[i] in ("'", '"'):
delim = s[i]
if delim:
i += len(delim)
while i < len(s) and s[i : i + len(delim)] != delim:
i += 1
i += len(delim)
continue
i += 1
def fstring_contains_expr(s: str) -> bool:
return any(iter_fexpr_spans(s))
class StringSplitter(BaseStringSplitter, CustomSplitMapMixin):
"""
StringTransformer that splits "atom" strings (i.e. strings which exist on
lines by themselves).