From 355147e89334a32f4a08e6691010be9bab00c665 Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Tue, 30 Nov 2021 18:33:49 +0100 Subject: [PATCH 1/4] C++, fix typo in internal function --- sphinx/domains/cpp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/domains/cpp.py b/sphinx/domains/cpp.py index b332ae422e..a7f9807414 100644 --- a/sphinx/domains/cpp.py +++ b/sphinx/domains/cpp.py @@ -6541,7 +6541,7 @@ def parser() -> ASTExpression: # ========================================================================== - def _parse_template_paramter(self) -> ASTTemplateParam: + def _parse_template_parameter(self) -> ASTTemplateParam: self.skip_ws() if self.skip_word('template'): # declare a tenplate template parameter @@ -6613,7 +6613,7 @@ def _parse_template_parameter_list(self) -> ASTTemplateParams: pos = self.pos err = None try: - param = self._parse_template_paramter() + param = self._parse_template_parameter() templateParams.append(param) except DefinitionError as eParam: self.pos = pos From 67d673406f2b96988fc578c631c7a9696ae6ab1d Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Tue, 30 Nov 2021 21:32:46 +0100 Subject: [PATCH 2/4] C++, fix parsing of fundamental types When multiple simple type specifiers are part of the type, then they may appear in any order. --- CHANGES | 3 + sphinx/domains/cpp.py | 142 +++++++++++++++++++++++++++++++-------- tests/test_domain_cpp.py | 11 ++- 3 files changed, 126 insertions(+), 30 deletions(-) diff --git a/CHANGES b/CHANGES index c00765178c..ee8bb59be5 100644 --- a/CHANGES +++ b/CHANGES @@ -16,6 +16,9 @@ Features added Bugs fixed ---------- +* #9917: C++, parse fundamental types no matter the order of simple type + specifiers. + Testing -------- diff --git a/sphinx/domains/cpp.py b/sphinx/domains/cpp.py index a7f9807414..b22ee0b13a 100644 --- a/sphinx/domains/cpp.py +++ b/sphinx/domains/cpp.py @@ -338,24 +338,14 @@ _simple_type_specifiers_re = re.compile(r"""(?x) \b( auto|void|bool - # Integer - # ------- - |((signed|unsigned)\s+)?(char|__int128|( - ((long\s+long|long|short)\s+)?int - )) - |wchar_t|char(8|16|32)_t - # extensions - |((signed|unsigned)\s+)?__int(64|128) - # Floating-point - # -------------- - |(float|double|long\s+double)(\s+(_Complex|_Imaginary))? - |(_Complex|_Imaginary)\s+(float|double|long\s+double) - # extensions - |__float80|_Float64x|__float128|_Float128 - # Integer types that could be prefixes of the previous ones - # --------------------------------------------------------- - |((signed|unsigned)\s+)?(long\s+long|long|short) |signed|unsigned + |short|long + |char|wchar_t|char(8|16|32)_t + |int + |__int(64|128) # extension + |float|double + |__float80|_Float64x|__float128|_Float128 # extension + |_Complex|_Imaginary # extension )\b """) @@ -485,12 +475,12 @@ 'long double': 'e', '__float80': 'e', '_Float64x': 'e', '__float128': 'g', '_Float128': 'g', - 'float _Complex': 'Cf', '_Complex float': 'Cf', - 'double _Complex': 'Cd', '_Complex double': 'Cd', - 'long double _Complex': 'Ce', '_Complex long double': 'Ce', - 'float _Imaginary': 'f', '_Imaginary float': 'f', - 'double _Imaginary': 'd', '_Imaginary double': 'd', - 'long double _Imaginary': 'e', '_Imaginary long double': 'e', + '_Complex float': 'Cf', + '_Complex double': 'Cd', + '_Complex long double': 'Ce', + '_Imaginary float': 'f', + '_Imaginary double': 'd', + '_Imaginary long double': 'e', 'auto': 'Da', 'decltype(auto)': 'Dc', 'std::nullptr_t': 'Dn' @@ -1853,8 +1843,12 @@ def describe_signature(self, signode: TextElement, mode: str, class ASTTrailingTypeSpecFundamental(ASTTrailingTypeSpec): - def __init__(self, name: str) -> None: - self.names = name.split() + def __init__(self, names: List[str], canonNames: List[str]) -> None: + assert len(names) != 0 + assert len(names) == len(canonNames), (names, canonNames) + self.names = names + # the canonical name list is for ID lookup + self.canonNames = canonNames def _stringify(self, transform: StringifyTransform) -> str: return ' '.join(self.names) @@ -1862,14 +1856,14 @@ def _stringify(self, transform: StringifyTransform) -> str: def get_id(self, version: int) -> str: if version == 1: res = [] - for a in self.names: + for a in self.canonNames: if a in _id_fundamental_v1: res.append(_id_fundamental_v1[a]) else: res.append(a) return '-'.join(res) - txt = str(self) + txt = ' '.join(self.canonNames) if txt not in _id_fundamental_v2: raise Exception( 'Semi-internal error: Fundamental type "%s" can not be mapped ' @@ -5855,12 +5849,102 @@ def _parse_nested_name(self, memberPointer: bool = False) -> ASTNestedName: # ========================================================================== + def _parse_simple_type_specifiers(self) -> ASTTrailingTypeSpecFundamental: + modifier: Optional[str] = None + signedness: Optional[str] = None + width: List[str] = [] + typ: Optional[str] = None + names: List[str] = [] # the parsed sequence + + self.skip_ws() + while self.match(_simple_type_specifiers_re): + t = self.matched_text + names.append(t) + if t in ('auto', 'void', 'bool', + 'char', 'wchar_t', 'char8_t', 'char16_t', 'char32_t', + 'int', '__int64', '__int128', + 'float', 'double', + '__float80', '_Float64x', '__float128', '_Float128'): + if typ is not None: + self.fail("Can not have both {} and {}.".format(t, typ)) + typ = t + elif t in ('signed', 'unsigned'): + if signedness is not None: + self.fail("Can not have both {} and {}.".format(t, signedness)) + signedness = t + elif t == 'short': + if len(width) != 0: + self.fail("Can not have both {} and {}.".format(t, width[0])) + width.append(t) + elif t == 'long': + if len(width) != 0 and width[0] != 'long': + self.fail("Can not have both {} and {}.".format(t, width[0])) + width.append(t) + elif t in ('_Imaginary', '_Complex'): + if modifier is not None: + self.fail("Can not have both {} and {}.".format(t, modifier)) + modifier = t + self.skip_ws() + if len(names) == 0: + return None + + if typ in ('auto', 'void', 'bool', + 'wchar_t', 'char8_t', 'char16_t', 'char32_t', + '__float80', '_Float64x', '__float128', '_Float128'): + if modifier is not None: + self.fail("Can not have both {} and {}.".format(typ, modifier)) + if signedness is not None: + self.fail("Can not have both {} and {}.".format(typ, signedness)) + if len(width) != 0: + self.fail("Can not have both {} and {}.".format(typ, ' '.join(width))) + elif typ == 'char': + if modifier is not None: + self.fail("Can not have both {} and {}.".format(typ, modifier)) + if len(width) != 0: + self.fail("Can not have both {} and {}.".format(typ, ' '.join(width))) + elif typ == 'int': + if modifier is not None: + self.fail("Can not have both {} and {}.".format(typ, modifier)) + elif typ in ('__int64', '__int128'): + if modifier is not None: + self.fail("Can not have both {} and {}.".format(typ, modifier)) + if len(width) != 0: + self.fail("Can not have both {} and {}.".format(typ, ' '.join(width))) + elif typ == 'float': + if signedness is not None: + self.fail("Can not have both {} and {}.".format(typ, signedness)) + if len(width) != 0: + self.fail("Can not have both {} and {}.".format(typ, ' '.join(width))) + elif typ == 'double': + if signedness is not None: + self.fail("Can not have both {} and {}.".format(typ, signedness)) + if len(width) > 1: + self.fail("Can not have both {} and {}.".format(typ, ' '.join(width))) + if len(width) == 1 and width[0] != 'long': + self.fail("Can not have both {} and {}.".format(typ, ' '.join(width))) + elif typ is None: + if modifier is not None: + self.fail("Can not have {} without a floating point type.".format(modifier)) + else: + assert False, "Unhandled type {}".format(typ) + + canonNames: List[str] = [] + if modifier is not None: + canonNames.append(modifier) + if signedness is not None: + canonNames.append(signedness) + canonNames.extend(width) + if typ is not None: + canonNames.append(typ) + return ASTTrailingTypeSpecFundamental(names, canonNames) + def _parse_trailing_type_spec(self) -> ASTTrailingTypeSpec: # fundamental types, https://en.cppreference.com/w/cpp/language/type # and extensions self.skip_ws() - if self.match(_simple_type_specifiers_re): - return ASTTrailingTypeSpecFundamental(self.matched_text) + res = self._parse_simple_type_specifiers() + if res is not None: + return res # decltype self.skip_ws() diff --git a/tests/test_domain_cpp.py b/tests/test_domain_cpp.py index ede4a25310..881d911c3c 100644 --- a/tests/test_domain_cpp.py +++ b/tests/test_domain_cpp.py @@ -8,6 +8,7 @@ :license: BSD, see LICENSE for details. """ +import itertools import re import zlib @@ -137,9 +138,17 @@ def makeIdV2(): if t == "std::nullptr_t": id = "NSt9nullptr_tE" return "1f%s" % id + id1 = makeIdV1() + id2 = makeIdV2() input = "void f(%s arg)" % t.replace(' ', ' ') output = "void f(%s arg)" % t - check("function", input, {1: makeIdV1(), 2: makeIdV2()}, output=output) + check("function", input, {1: id1, 2: id2}, output=output) + if ' ' in t: + # try permutations of all commponents + tcs = t.split() + for p in itertools.permutations(tcs): + input = "void f(%s arg)" % ' '.join(p) + check("function", input, {1: id1, 2: id2}) def test_domain_cpp_ast_expressions(): From c09643c21fa007cde0f2bfa4fa889e7024d183c4 Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Tue, 30 Nov 2021 21:57:30 +0100 Subject: [PATCH 3/4] C, fix parsing of fundamental types When multiple simple type specifiers are part of the type, then they may appear in any order. --- CHANGES | 2 +- sphinx/domains/c.py | 51 +++++++++++++++++++++------------------- tests/test_domain_c.py | 8 +++++++ tests/test_domain_cpp.py | 2 +- 4 files changed, 37 insertions(+), 26 deletions(-) diff --git a/CHANGES b/CHANGES index ee8bb59be5..f642906c55 100644 --- a/CHANGES +++ b/CHANGES @@ -16,7 +16,7 @@ Features added Bugs fixed ---------- -* #9917: C++, parse fundamental types no matter the order of simple type +* #9917: C and C++, parse fundamental types no matter the order of simple type specifiers. Testing diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py index 84878cd05f..6f8a21af5c 100644 --- a/sphinx/domains/c.py +++ b/sphinx/domains/c.py @@ -95,28 +95,18 @@ _simple_type_specifiers_re = re.compile(r"""(?x) \b( void|_Bool|bool - # Integer - # ------- - |((signed|unsigned)\s+)?(char|( - ((long\s+long|long|short)\s+)?int - )) + |signed|unsigned + |short|long + |char + |int |__uint128|__int128 - # extensions - |((signed|unsigned)\s+)?__int(8|16|32|64|128) - # Floating-point - # -------------- - |(float|double|long\s+double)(\s+(_Complex|complex|_Imaginary|imaginary))? - |(_Complex|complex|_Imaginary|imaginary)\s+(float|double|long\s+double) + |__int(8|16|32|64|128) # extension + |float|double |_Decimal(32|64|128) - # extensions - |__float80|_Float64x|__float128|_Float128|__ibm128 - |__fp16 - # Fixed-point, extension - |(_Sat\s+)?((signed|unsigned)\s+)?((short|long|long\s+long)\s+)?(_Fract|fract|_Accum|accum) - # Integer types that could be prefixes of the previous ones - # --------------------------------------------------------- - |((signed|unsigned)\s+)?(long\s+long|long|short) - |signed|unsigned + |_Complex|complex|_Imaginary|imaginary + |__float80|_Float64x|__float128|_Float128|__ibm128 # extension + |__fp16 # extension + |_Sat|_Fract|fract|_Accum|accum # extension )\b """) @@ -636,8 +626,9 @@ class ASTTrailingTypeSpec(ASTBase): class ASTTrailingTypeSpecFundamental(ASTTrailingTypeSpec): - def __init__(self, name: str) -> None: - self.names = name.split() + def __init__(self, names: List[str]) -> None: + assert len(names) != 0 + self.names = names def _stringify(self, transform: StringifyTransform) -> str: return ' '.join(self.names) @@ -2580,12 +2571,24 @@ def _parse_nested_name(self) -> ASTNestedName: break return ASTNestedName(names, rooted) + def _parse_simple_type_specifiers(self) -> ASTTrailingTypeSpecFundamental: + names: List[str] = [] + + self.skip_ws() + while self.match(_simple_type_specifiers_re): + names.append(self.matched_text) + self.skip_ws() + if len(names) == 0: + return None + return ASTTrailingTypeSpecFundamental(names) + def _parse_trailing_type_spec(self) -> ASTTrailingTypeSpec: # fundamental types, https://en.cppreference.com/w/c/language/type # and extensions self.skip_ws() - if self.match(_simple_type_specifiers_re): - return ASTTrailingTypeSpecFundamental(self.matched_text) + res = self._parse_simple_type_specifiers() + if res is not None: + return res # prefixed prefix = None diff --git a/tests/test_domain_c.py b/tests/test_domain_c.py index 0800c50596..60d3fa488e 100644 --- a/tests/test_domain_c.py +++ b/tests/test_domain_c.py @@ -8,6 +8,7 @@ :license: BSD, see LICENSE for details. """ +import itertools import zlib from xml.etree import ElementTree @@ -329,6 +330,13 @@ def signed(t): input = "{key}%s foo" % t output = ' '.join(input.split()) check('type', input, {1: 'foo'}, key='typedef', output=output) + if ' ' in t: + # try permutations of all components + tcs = t.split() + for p in itertools.permutations(tcs): + input = "{key}%s foo" % ' '.join(p) + output = ' '.join(input.split()) + check("type", input, {1: 'foo'}, key='typedef', output=output) def test_domain_c_ast_type_definitions(): diff --git a/tests/test_domain_cpp.py b/tests/test_domain_cpp.py index 881d911c3c..5ff8f8e129 100644 --- a/tests/test_domain_cpp.py +++ b/tests/test_domain_cpp.py @@ -144,7 +144,7 @@ def makeIdV2(): output = "void f(%s arg)" % t check("function", input, {1: id1, 2: id2}, output=output) if ' ' in t: - # try permutations of all commponents + # try permutations of all components tcs = t.split() for p in itertools.permutations(tcs): input = "void f(%s arg)" % ' '.join(p) From f8f26286ad74cd33981c07b59bdaa50b7f4cc7f9 Mon Sep 17 00:00:00 2001 From: Jakob Lykke Andersen Date: Wed, 1 Dec 2021 19:33:23 +0100 Subject: [PATCH 4/4] C, simple type specifier parsing, handling macro-keywords --- sphinx/domains/c.py | 21 +++++++++++++++++---- tests/test_domain_c.py | 5 +---- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/sphinx/domains/c.py b/sphinx/domains/c.py index 6f8a21af5c..dff8bef00e 100644 --- a/sphinx/domains/c.py +++ b/sphinx/domains/c.py @@ -92,9 +92,10 @@ _string_re = re.compile(r"[LuU8]?('([^'\\]*(?:\\.[^'\\]*)*)'" r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S) +# bool, complex, and imaginary are macro "keywords", so they are handled seperately _simple_type_specifiers_re = re.compile(r"""(?x) \b( - void|_Bool|bool + void|_Bool |signed|unsigned |short|long |char @@ -103,7 +104,7 @@ |__int(8|16|32|64|128) # extension |float|double |_Decimal(32|64|128) - |_Complex|complex|_Imaginary|imaginary + |_Complex|_Imaginary |__float80|_Float64x|__float128|_Float128|__ibm128 # extension |__fp16 # extension |_Sat|_Fract|fract|_Accum|accum # extension @@ -2571,12 +2572,24 @@ def _parse_nested_name(self) -> ASTNestedName: break return ASTNestedName(names, rooted) + def _parse_simple_type_specifier(self) -> Optional[str]: + if self.match(_simple_type_specifiers_re): + return self.matched_text + for t in ('bool', 'complex', 'imaginary'): + if t in self.config.c_extra_keywords: + if self.skip_word(t): + return t + return None + def _parse_simple_type_specifiers(self) -> ASTTrailingTypeSpecFundamental: names: List[str] = [] self.skip_ws() - while self.match(_simple_type_specifiers_re): - names.append(self.matched_text) + while True: + t = self._parse_simple_type_specifier() + if t is None: + break + names.append(t) self.skip_ws() if len(names) == 0: return None diff --git a/tests/test_domain_c.py b/tests/test_domain_c.py index 60d3fa488e..d3198dcc03 100644 --- a/tests/test_domain_c.py +++ b/tests/test_domain_c.py @@ -595,10 +595,7 @@ def test_domain_c_ast_attributes(): def test_extra_keywords(): with pytest.raises(DefinitionError, - match='Expected identifier, got user-defined keyword: complex.'): - parse('function', 'void f(int complex)') - with pytest.raises(DefinitionError, - match='Expected identifier, got user-defined keyword: complex.'): + match='Expected identifier in nested name'): parse('function', 'void complex(void)')