Skip to content

Commit

Permalink
Merge pull request #9918 from jakobandersen/c-cpp-fundamental-type-pe…
Browse files Browse the repository at this point in the history 
…rmutation

C, C++: fundamental type parsing
  • Loading branch information
@jakobandersen
jakobandersen committed Dec 5, 2021
2 parents 95969a4 + f8f2628 commit 3ae124f
Show file tree
Hide file tree
Showing 5 changed files with 178 additions and 61 deletions.
3 changes: 3 additions & 0 deletions CHANGES
View file
Expand Up @@ -16,6 +16,9 @@ Features added
Bugs fixed
----------

* #9917: C and C++, parse fundamental types no matter the order of simple type
specifiers.

Testing
--------

Expand Down
66 changes: 41 additions & 25 deletions sphinx/domains/c.py
View file
Expand Up @@ -92,31 +92,22 @@
_string_re = re.compile(r"[LuU8]?('([^'\\]*(?:\\.[^'\\]*)*)'"
r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S)

# bool, complex, and imaginary are macro "keywords", so they are handled seperately
_simple_type_specifiers_re = re.compile(r"""(?x)
\b(
void|_Bool|bool
# Integer
# -------
|((signed|unsigned)\s+)?(char|(
((long\s+long|long|short)\s+)?int
))
void|_Bool
|signed|unsigned
|short|long
|char
|int
|__uint128|__int128
# extensions
|((signed|unsigned)\s+)?__int(8|16|32|64|128)
# Floating-point
# --------------
|(float|double|long\s+double)(\s+(_Complex|complex|_Imaginary|imaginary))?
|(_Complex|complex|_Imaginary|imaginary)\s+(float|double|long\s+double)
|__int(8|16|32|64|128) # extension
|float|double
|_Decimal(32|64|128)
# extensions
|__float80|_Float64x|__float128|_Float128|__ibm128
|__fp16
# Fixed-point, extension
|(_Sat\s+)?((signed|unsigned)\s+)?((short|long|long\s+long)\s+)?(_Fract|fract|_Accum|accum)
# Integer types that could be prefixes of the previous ones
# ---------------------------------------------------------
|((signed|unsigned)\s+)?(long\s+long|long|short)
|signed|unsigned
|_Complex|_Imaginary
|__float80|_Float64x|__float128|_Float128|__ibm128 # extension
|__fp16 # extension
|_Sat|_Fract|fract|_Accum|accum # extension
)\b
""")

Expand Down Expand Up @@ -636,8 +627,9 @@ class ASTTrailingTypeSpec(ASTBase):


class ASTTrailingTypeSpecFundamental(ASTTrailingTypeSpec):
def __init__(self, name: str) -> None:
self.names = name.split()
def __init__(self, names: List[str]) -> None:
assert len(names) != 0
self.names = names

def _stringify(self, transform: StringifyTransform) -> str:
return ' '.join(self.names)
Expand Down Expand Up @@ -2580,12 +2572,36 @@ def _parse_nested_name(self) -> ASTNestedName:
break
return ASTNestedName(names, rooted)

def _parse_simple_type_specifier(self) -> Optional[str]:
if self.match(_simple_type_specifiers_re):
return self.matched_text
for t in ('bool', 'complex', 'imaginary'):
if t in self.config.c_extra_keywords:
if self.skip_word(t):
return t
return None

def _parse_simple_type_specifiers(self) -> ASTTrailingTypeSpecFundamental:
names: List[str] = []

self.skip_ws()
while True:
t = self._parse_simple_type_specifier()
if t is None:
break
names.append(t)
self.skip_ws()
if len(names) == 0:
return None
return ASTTrailingTypeSpecFundamental(names)

def _parse_trailing_type_spec(self) -> ASTTrailingTypeSpec:
# fundamental types, https://en.cppreference.com/w/c/language/type
# and extensions
self.skip_ws()
if self.match(_simple_type_specifiers_re):
return ASTTrailingTypeSpecFundamental(self.matched_text)
res = self._parse_simple_type_specifiers()
if res is not None:
return res

# prefixed
prefix = None
Expand Down
146 changes: 115 additions & 31 deletions sphinx/domains/cpp.py
View file
Expand Up @@ -338,24 +338,14 @@
_simple_type_specifiers_re = re.compile(r"""(?x)
\b(
auto|void|bool
# Integer
# -------
|((signed|unsigned)\s+)?(char|__int128|(
((long\s+long|long|short)\s+)?int
))
|wchar_t|char(8|16|32)_t
# extensions
|((signed|unsigned)\s+)?__int(64|128)
# Floating-point
# --------------
|(float|double|long\s+double)(\s+(_Complex|_Imaginary))?
|(_Complex|_Imaginary)\s+(float|double|long\s+double)
# extensions
|__float80|_Float64x|__float128|_Float128
# Integer types that could be prefixes of the previous ones
# ---------------------------------------------------------
|((signed|unsigned)\s+)?(long\s+long|long|short)
|signed|unsigned
|short|long
|char|wchar_t|char(8|16|32)_t
|int
|__int(64|128) # extension
|float|double
|__float80|_Float64x|__float128|_Float128 # extension
|_Complex|_Imaginary # extension
)\b
""")

Expand Down Expand Up @@ -485,12 +475,12 @@
'long double': 'e',
'__float80': 'e', '_Float64x': 'e',
'__float128': 'g', '_Float128': 'g',
'float _Complex': 'Cf', '_Complex float': 'Cf',
'double _Complex': 'Cd', '_Complex double': 'Cd',
'long double _Complex': 'Ce', '_Complex long double': 'Ce',
'float _Imaginary': 'f', '_Imaginary float': 'f',
'double _Imaginary': 'd', '_Imaginary double': 'd',
'long double _Imaginary': 'e', '_Imaginary long double': 'e',
'_Complex float': 'Cf',
'_Complex double': 'Cd',
'_Complex long double': 'Ce',
'_Imaginary float': 'f',
'_Imaginary double': 'd',
'_Imaginary long double': 'e',
'auto': 'Da',
'decltype(auto)': 'Dc',
'std::nullptr_t': 'Dn'
Expand Down Expand Up @@ -1853,23 +1843,27 @@ def describe_signature(self, signode: TextElement, mode: str,


class ASTTrailingTypeSpecFundamental(ASTTrailingTypeSpec):
def __init__(self, name: str) -> None:
self.names = name.split()
def __init__(self, names: List[str], canonNames: List[str]) -> None:
assert len(names) != 0
assert len(names) == len(canonNames), (names, canonNames)
self.names = names
# the canonical name list is for ID lookup
self.canonNames = canonNames

def _stringify(self, transform: StringifyTransform) -> str:
return ' '.join(self.names)

def get_id(self, version: int) -> str:
if version == 1:
res = []
for a in self.names:
for a in self.canonNames:
if a in _id_fundamental_v1:
res.append(_id_fundamental_v1[a])
else:
res.append(a)
return '-'.join(res)

txt = str(self)
txt = ' '.join(self.canonNames)
if txt not in _id_fundamental_v2:
raise Exception(
'Semi-internal error: Fundamental type "%s" can not be mapped '
Expand Down Expand Up @@ -5855,12 +5849,102 @@ def _parse_nested_name(self, memberPointer: bool = False) -> ASTNestedName:

# ==========================================================================

def _parse_simple_type_specifiers(self) -> ASTTrailingTypeSpecFundamental:
modifier: Optional[str] = None
signedness: Optional[str] = None
width: List[str] = []
typ: Optional[str] = None
names: List[str] = [] # the parsed sequence

self.skip_ws()
while self.match(_simple_type_specifiers_re):
t = self.matched_text
names.append(t)
if t in ('auto', 'void', 'bool',
'char', 'wchar_t', 'char8_t', 'char16_t', 'char32_t',
'int', '__int64', '__int128',
'float', 'double',
'__float80', '_Float64x', '__float128', '_Float128'):
if typ is not None:
self.fail("Can not have both {} and {}.".format(t, typ))
typ = t
elif t in ('signed', 'unsigned'):
if signedness is not None:
self.fail("Can not have both {} and {}.".format(t, signedness))
signedness = t
elif t == 'short':
if len(width) != 0:
self.fail("Can not have both {} and {}.".format(t, width[0]))
width.append(t)
elif t == 'long':
if len(width) != 0 and width[0] != 'long':
self.fail("Can not have both {} and {}.".format(t, width[0]))
width.append(t)
elif t in ('_Imaginary', '_Complex'):
if modifier is not None:
self.fail("Can not have both {} and {}.".format(t, modifier))
modifier = t
self.skip_ws()
if len(names) == 0:
return None

if typ in ('auto', 'void', 'bool',
'wchar_t', 'char8_t', 'char16_t', 'char32_t',
'__float80', '_Float64x', '__float128', '_Float128'):
if modifier is not None:
self.fail("Can not have both {} and {}.".format(typ, modifier))
if signedness is not None:
self.fail("Can not have both {} and {}.".format(typ, signedness))
if len(width) != 0:
self.fail("Can not have both {} and {}.".format(typ, ' '.join(width)))
elif typ == 'char':
if modifier is not None:
self.fail("Can not have both {} and {}.".format(typ, modifier))
if len(width) != 0:
self.fail("Can not have both {} and {}.".format(typ, ' '.join(width)))
elif typ == 'int':
if modifier is not None:
self.fail("Can not have both {} and {}.".format(typ, modifier))
elif typ in ('__int64', '__int128'):
if modifier is not None:
self.fail("Can not have both {} and {}.".format(typ, modifier))
if len(width) != 0:
self.fail("Can not have both {} and {}.".format(typ, ' '.join(width)))
elif typ == 'float':
if signedness is not None:
self.fail("Can not have both {} and {}.".format(typ, signedness))
if len(width) != 0:
self.fail("Can not have both {} and {}.".format(typ, ' '.join(width)))
elif typ == 'double':
if signedness is not None:
self.fail("Can not have both {} and {}.".format(typ, signedness))
if len(width) > 1:
self.fail("Can not have both {} and {}.".format(typ, ' '.join(width)))
if len(width) == 1 and width[0] != 'long':
self.fail("Can not have both {} and {}.".format(typ, ' '.join(width)))
elif typ is None:
if modifier is not None:
self.fail("Can not have {} without a floating point type.".format(modifier))
else:
assert False, "Unhandled type {}".format(typ)

canonNames: List[str] = []
if modifier is not None:
canonNames.append(modifier)
if signedness is not None:
canonNames.append(signedness)
canonNames.extend(width)
if typ is not None:
canonNames.append(typ)
return ASTTrailingTypeSpecFundamental(names, canonNames)

def _parse_trailing_type_spec(self) -> ASTTrailingTypeSpec:
# fundamental types, https://en.cppreference.com/w/cpp/language/type
# and extensions
self.skip_ws()
if self.match(_simple_type_specifiers_re):
return ASTTrailingTypeSpecFundamental(self.matched_text)
res = self._parse_simple_type_specifiers()
if res is not None:
return res

# decltype
self.skip_ws()
Expand Down Expand Up @@ -6541,7 +6625,7 @@ def parser() -> ASTExpression:

# ==========================================================================

def _parse_template_paramter(self) -> ASTTemplateParam:
def _parse_template_parameter(self) -> ASTTemplateParam:
self.skip_ws()
if self.skip_word('template'):
# declare a tenplate template parameter
Expand Down Expand Up @@ -6613,7 +6697,7 @@ def _parse_template_parameter_list(self) -> ASTTemplateParams:
pos = self.pos
err = None
try:
param = self._parse_template_paramter()
param = self._parse_template_parameter()
templateParams.append(param)
except DefinitionError as eParam:
self.pos = pos
Expand Down
13 changes: 9 additions & 4 deletions tests/test_domain_c.py
View file
Expand Up @@ -8,6 +8,7 @@
:license: BSD, see LICENSE for details.
"""

import itertools
import zlib
from xml.etree import ElementTree

Expand Down Expand Up @@ -329,6 +330,13 @@ def signed(t):
input = "{key}%s foo" % t
output = ' '.join(input.split())
check('type', input, {1: 'foo'}, key='typedef', output=output)
if ' ' in t:
# try permutations of all components
tcs = t.split()
for p in itertools.permutations(tcs):
input = "{key}%s foo" % ' '.join(p)
output = ' '.join(input.split())
check("type", input, {1: 'foo'}, key='typedef', output=output)


def test_domain_c_ast_type_definitions():
Expand Down Expand Up @@ -587,10 +595,7 @@ def test_domain_c_ast_attributes():

def test_extra_keywords():
with pytest.raises(DefinitionError,
match='Expected identifier, got user-defined keyword: complex.'):
parse('function', 'void f(int complex)')
with pytest.raises(DefinitionError,
match='Expected identifier, got user-defined keyword: complex.'):
match='Expected identifier in nested name'):
parse('function', 'void complex(void)')


Expand Down
11 changes: 10 additions & 1 deletion tests/test_domain_cpp.py
View file
Expand Up @@ -8,6 +8,7 @@
:license: BSD, see LICENSE for details.
"""

import itertools
import re
import zlib

Expand Down Expand Up @@ -137,9 +138,17 @@ def makeIdV2():
if t == "std::nullptr_t":
id = "NSt9nullptr_tE"
return "1f%s" % id
id1 = makeIdV1()
id2 = makeIdV2()
input = "void f(%s arg)" % t.replace(' ', ' ')
output = "void f(%s arg)" % t
check("function", input, {1: makeIdV1(), 2: makeIdV2()}, output=output)
check("function", input, {1: id1, 2: id2}, output=output)
if ' ' in t:
# try permutations of all components
tcs = t.split()
for p in itertools.permutations(tcs):
input = "void f(%s arg)" % ' '.join(p)
check("function", input, {1: id1, 2: id2})


def test_domain_cpp_ast_expressions():
Expand Down

0 comments on commit 3ae124f

Please sign in to comment.