protoparse: lexer and grammar updates to better match protoc (#262)

* allow float literals to have trailing dot * allow exotic multi-line identifiers * allow groups in oneofs * also fixes issue with spurious newlines in protoparse related to oneofs
jhump · Aug 2, 2019 · 009ea28 · 009ea28
1 parent b1090fc
commit 009ea28
Show file tree

Hide file tree

Showing 47 changed files with 5,384 additions and 925 deletions.
diff --git a/desc/protoparse/ast.go b/desc/protoparse/ast.go
@@ -528,6 +528,10 @@ type groupNode struct {
 }
 
 func (n *groupNode) fieldLabel() node {
+	if n.label == nil {
+		// return nil interface to indicate absence, not a typed nil
+		return nil
+	}
 	return n.label
 }
 
@@ -568,6 +572,7 @@ type oneOfElement struct {
 	// a discriminated union: only one field will be set
 	option *optionNode
 	field  *fieldNode
+	group  *groupNode
 	empty  *basicNode
 }
 

diff --git a/desc/protoparse/lexer.go b/desc/protoparse/lexer.go
@@ -489,19 +489,6 @@ func (l *protoLex) readNumber(sofar []rune, allowDot bool, allowExp bool) []rune
 				break
 			}
 			allowDot = false
-			cn, _, err := l.input.readRune()
-			if err != nil {
-				l.input.unreadRune(c)
-				break
-			}
-			if cn < '0' || cn > '9' {
-				l.input.unreadRune(cn)
-				l.input.unreadRune(c)
-				break
-			}
-			l.adjustPos(c)
-			token = append(token, c)
-			c = cn
 		} else if c == 'e' || c == 'E' {
 			if !allowExp {
 				l.input.unreadRune(c)

diff --git a/desc/protoparse/parser.go b/desc/protoparse/parser.go
@@ -999,6 +999,11 @@ func (r *parseResult) addMessageDecls(msgd *dpb.DescriptorProto, decls []*messag
 					fd := r.asFieldDescriptor(oodecl.field)
 					fd.OneofIndex = proto.Int32(int32(oodIndex))
 					msgd.Field = append(msgd.Field, fd)
+				} else if oodecl.group != nil {
+					fd, md := r.asGroupDescriptors(oodecl.group, isProto3)
+					fd.OneofIndex = proto.Int32(int32(oodIndex))
+					msgd.Field = append(msgd.Field, fd)
+					msgd.NestedType = append(msgd.NestedType, md)
 				}
 			}
 		} else if decl.option != nil {

diff --git a/desc/protoparse/proto.y b/desc/protoparse/proto.y
@@ -78,7 +78,7 @@ import (
 %type <agg>       aggFields aggField aggFieldEntry
 %type <fld>       field oneofField
 %type <oo>        oneof
-%type <grp>       group
+%type <grp>       group oneofGroup
 %type <mapFld>    mapField
 %type <mapType>   mapType
 %type <msg>       message
@@ -434,10 +434,18 @@ constantList : constant {
 
 typeIdent : ident
 	| _TYPENAME
+	| '.' ident {
+        $$ = &identNode{val: "." + $2.val}
+        $$.setRange($1, $2)
+	}
 	| typeIdent _TYPENAME {
         $$ = &identNode{val: $1.val + $2.val}
         $$.setRange($1, $2)
 	}
+	| typeIdent '.' ident {
+        $$ = &identNode{val: $1.val + "." + $3.val}
+        $$.setRange($1, $3)
+	}
 
 field : _REQUIRED typeIdent name '=' _INT_LIT ';' {
 		checkTag(protolex, $5.start(), $5.val)
@@ -560,6 +568,9 @@ oneofItem : option {
 	| oneofField {
 		$$ = []*oneOfElement{{field: $1}}
 	}
+	| oneofGroup {
+		$$ = []*oneOfElement{{group: $1}}
+	}
 	| ';' {
 		$$ = []*oneOfElement{{empty: $1}}
 	}
@@ -579,6 +590,15 @@ oneofField : typeIdent name '=' _INT_LIT ';' {
 		$$.setRange($1, $6)
 	}
 
+oneofGroup : _GROUP name '=' _INT_LIT '{' messageBody '}' {
+		checkTag(protolex, $4.start(), $4.val)
+		if !unicode.IsUpper(rune($2.val[0])) {
+			lexError(protolex, $2.start(), fmt.Sprintf("group %s should have a name that starts with a capital letter", $2.val))
+		}
+		$$ = &groupNode{groupKeyword: $1, name: $2, tag: $4, decls: $6}
+		$$.setRange($1, $7)
+	}
+
 mapField : mapType name '=' _INT_LIT ';' {
 		checkTag(protolex, $4.start(), $4.val)
 		$$ = &mapFieldNode{mapType: $1, name: $2, tag: $4}