Skip to content

Commit

Permalink
generator: Upgrade nom to 0.7 where parser macros have been remov…
Browse files Browse the repository at this point in the history 
…ed (#610)
  • Loading branch information
@pollend @MarijnS95
pollend authored and MarijnS95 committed May 11, 2022
1 parent 90179c8 commit 28a4868
Show file tree
Hide file tree
Showing 2 changed files with 94 additions and 78 deletions.
2 changes: 1 addition & 1 deletion generator/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ edition = "2018"
bindgen = "0.58"
heck = "0.3"
itertools = "0.10"
nom = "6.0"
nom = "7.1"
once_cell = "1.7"
proc-macro2 = "1.0"
quote = "1.0"
Expand Down
170 changes: 93 additions & 77 deletions generator/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,26 @@

use heck::{CamelCase, ShoutySnakeCase, SnakeCase};
use itertools::Itertools;
use nom::sequence::pair;
use nom::{
alt, char,
character::complete::{digit1, hex_digit1, multispace1},
complete, delimited, do_parse, many1, map, named, none_of, one_of, opt, pair, preceded, tag,
terminated, value,
branch::alt,
bytes::streaming::tag,
character::complete::{char, digit1, hex_digit1, multispace1, none_of, one_of},
combinator::{complete, map, opt, value},
error::{ParseError, VerboseError},
multi::many1,
sequence::{delimited, preceded, terminated},
IResult, Parser,
};
use once_cell::sync::Lazy;
use proc_macro2::{Delimiter, Group, Literal, Span, TokenStream, TokenTree};
use quote::*;
use regex::Regex;
use std::collections::{BTreeMap, HashMap, HashSet};
use std::fmt::Display;
use std::path::Path;
use std::{
collections::{BTreeMap, HashMap, HashSet},
fmt::Display,
path::Path,
};
use syn::Ident;

macro_rules! get_variant {
Expand Down Expand Up @@ -63,83 +70,90 @@ impl quote::ToTokens for CType {
}
}

named!(ctype<&str, CType>,
alt!(
value!(CType::U64, complete!(tag!("ULL"))) |
value!(CType::U32, complete!(tag!("U")))
)
);

named!(cexpr<&str, (CType, String)>,
alt!(
map!(cfloat, |f| (CType::Float, format!("{:.2}", f))) |
inverse_number |
decimal_number |
hexadecimal_number
)
);
fn parse_ctype<'a, E: ParseError<&'a str>>(i: &'a str) -> IResult<&'a str, CType, E> {
(alt((
value(CType::U64, complete(tag("ULL"))),
value(CType::U32, complete(tag("U"))),
)))(i)
}

named!(decimal_number<&str, (CType, String)>,
do_parse!(
num: digit1 >>
typ: ctype >>
((typ, num.to_string()))
)
);

named!(hexadecimal_number<&str, (CType, String)>,
preceded!(
alt!(tag!("0x") | tag!("0X")),
map!(
pair!(hex_digit1, ctype),
|(num, typ)| (typ, format!("0x{}{}", num.to_ascii_lowercase(), typ.to_string())
)
)
)
);

named!(inverse_number<&str, (CType, String)>,
map!(
delimited!(
tag!("("),
pair!(
preceded!(tag!("~"), decimal_number),
opt!(preceded!(tag!("-"), digit1))
fn parse_cexpr<'a, E: ParseError<&'a str>>(i: &'a str) -> IResult<&'a str, (CType, String), E> {
(alt((
map(parse_cfloat, |f| (CType::Float, format!("{:.2}", f))),
parse_inverse_number,
parse_decimal_number,
parse_hexadecimal_number,
)))(i)
}

fn parse_cfloat<'a, E: ParseError<&'a str>>(i: &'a str) -> IResult<&'a str, f32, E> {
(terminated(nom::number::complete::float, one_of("fF")))(i)
}

fn parse_inverse_number<'a, E: ParseError<&'a str>>(
i: &'a str,
) -> IResult<&'a str, (CType, String), E> {
(map(
delimited(
char('('),
pair(
preceded(char('~'), parse_decimal_number),
opt(preceded(char('-'), digit1)),
),
tag!(")")
char(')'),
),
|((ctyp, num), minus_num)| {
let expr = if let Some(minus) = minus_num {
format!("!{}-{}", num, minus)
}
else{
} else {
format!("!{}", num)
};
(ctyp, expr)
}
)
);

named!(cfloat<&str, f32>,
terminated!(nom::number::complete::float, one_of!("fF"))
);
},
))(i)
}

// Like a C string, but does not support quote escaping and expects at least one character.
// If needed, use https://github.com/Geal/nom/blob/8e09f0c3029d32421b5b69fb798cef6855d0c8df/tests/json.rs#L61-L81
named!(c_include_string<&str, String>,
delimited!(
char!('"'),
map!(
many1!(none_of!("\"")),
|chars| chars.iter().map(char::to_string).join("")
),
char!('"')
)
);

named!(c_include<&str, String>,
preceded!(tag!("#include"), preceded!(multispace1, c_include_string))
);
fn parse_c_include_string<'a, E: ParseError<&'a str>>(i: &'a str) -> IResult<&'a str, String, E> {
(delimited(
char('"'),
map(many1(none_of("\"")), |c| {
c.iter().map(char::to_string).join("")
}),
char('"'),
))(i)
}

fn parse_c_include<'a, E: ParseError<&'a str>>(i: &'a str) -> IResult<&'a str, String, E> {
(preceded(
tag("#include"),
preceded(multispace1, parse_c_include_string),
))(i)
}

fn parse_decimal_number<'a, E: ParseError<&'a str>>(
i: &'a str,
) -> IResult<&'a str, (CType, String), E> {
(map(
pair(digit1.map(str::to_string), parse_ctype),
|(dig, ctype)| (ctype, dig),
))(i)
}

fn parse_hexadecimal_number<'a, E: ParseError<&'a str>>(
i: &'a str,
) -> IResult<&'a str, (CType, String), E> {
(preceded(
alt((tag("0x"), tag("0X"))),
map(pair(hex_digit1, parse_ctype), |(num, typ)| {
(
typ,
format!("0x{}{}", num.to_ascii_lowercase(), typ.to_string()),
)
}),
))(i)
}

fn khronos_link<S: Display + ?Sized>(name: &S) -> Literal {
Literal::string(&format!(
Expand Down Expand Up @@ -265,7 +279,8 @@ impl quote::ToTokens for Constant {
}
Constant::Text(ref text) => text.to_tokens(tokens),
Constant::CExpr(ref expr) => {
let (_, (_, rexpr)) = cexpr(expr).expect("Unable to parse cexpr");
let (_, (_, rexpr)) =
parse_cexpr::<VerboseError<&str>>(expr).expect("Unable to parse cexpr");
tokens.extend(rexpr.parse::<TokenStream>());
}
Constant::BitPos(pos) => {
Expand Down Expand Up @@ -318,7 +333,8 @@ impl Constant {
match self {
Constant::Number(_) | Constant::Hex(_) => CType::USize,
Constant::CExpr(expr) => {
let (_, (ty, _)) = cexpr(expr).expect("Unable to parse cexpr");
let (_, (ty, _)) =
parse_cexpr::<VerboseError<&str>>(expr).expect("Unable to parse cexpr");
ty
}
_ => unimplemented!(),
Expand Down Expand Up @@ -549,12 +565,12 @@ fn name_to_tokens(type_name: &str) -> Ident {
/// Parses and rewrites a C literal into Rust
///
/// If no special pattern is recognized the original literal is returned.
/// Any new conversions need to be added to the [`cexpr()`] [`nom`] parser.
/// Any new conversions need to be added to the [`parse_cexpr()`] [`nom`] parser.
///
/// Examples:
/// - `0x3FFU` -> `0x3ffu32`
fn convert_c_literal(lit: Literal) -> Literal {
if let Ok((_, (_, rexpr))) = cexpr(&lit.to_string()) {
if let Ok((_, (_, rexpr))) = parse_cexpr::<VerboseError<&str>>(&lit.to_string()) {
// lit::SynInt uses the same `.parse` method to create hexadecimal
// literals because there is no `Literal` constructor for it.
let mut stream = rexpr.parse::<TokenStream>().unwrap().into_iter();
Expand Down Expand Up @@ -2289,7 +2305,7 @@ pub fn extract_native_types(registry: &vk_parse::Registry) -> (Vec<(String, Stri
name
);

let (rem, path) = c_include(&code.code)
let (rem, path) = parse_c_include::<VerboseError<&str>>(&code.code)
.expect("Failed to parse `#include` from `category=\"include\"` directive");
assert!(rem.is_empty());
header_includes.push((name, path));
Expand Down

0 comments on commit 28a4868

Please sign in to comment.