Skip to content

Commit

Permalink
Output warnings for non-UTF-8 tokens.
Browse files Browse the repository at this point in the history
  • Loading branch information
reitermarkus committed Feb 29, 2024
1 parent 11b0789 commit 2bac4dd
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 5 deletions.
9 changes: 8 additions & 1 deletion bindgen/ir/context.rs
Expand Up @@ -2162,7 +2162,14 @@ If you encounter an error missing from this list, please file an issue or a PR!"
let spelling = token.spelling();
let name = match spelling.to_str() {
Ok(name) => Cow::Borrowed(name),
Err(_) => spelling.to_string_lossy(),
Err(_) => {
let name = spelling.to_string_lossy();
warn!(
"Lossy conversion of non-UTF8 token {:?} to {:?}.",
spelling, name
);
name
}
};
match name.as_ref() {
"inline" => {
Expand Down
12 changes: 8 additions & 4 deletions bindgen/ir/var.rs
Expand Up @@ -167,17 +167,21 @@ fn handle_function_macro(
t.kind == clang_sys::CXToken_Punctuation &&
t.spelling().to_bytes() == b")"
};
let tokens: Vec<_> = cursor.tokens().iter().collect();
if let Some(boundary) = tokens.iter().position(is_closing_paren) {
let tokens: Result<Vec<_>, _> = tokens
let mut raw_tokens: Vec<_> = cursor.tokens().iter().collect();
if let Some(boundary) = raw_tokens.iter().position(is_closing_paren) {
let tokens: Result<Vec<_>, _> = raw_tokens
.iter()
.map(|token| token.spelling().to_str())
.collect();

let mut tokens = if let Ok(tokens) = tokens {
tokens
} else {
// Skip macros containing invalid UTF-8.
let raw_name = raw_tokens.remove(0);
warn!(
"Ignoring macro {:?} containing invalid UTF-8 tokens.",
raw_name.spelling()
);
return;
};

Expand Down

0 comments on commit 2bac4dd

Please sign in to comment.