diff --git a/desc/protoparse/ast.go b/desc/protoparse/ast.go index 9c2f2e0b..79a70fc0 100644 --- a/desc/protoparse/ast.go +++ b/desc/protoparse/ast.go @@ -33,14 +33,14 @@ func unknownPos(filename string) *SourcePos { type node interface { start() *SourcePos end() *SourcePos - leadingComments() []*comment - trailingComments() []*comment + leadingComments() []comment + trailingComments() []comment } type terminalNode interface { node - popLeadingComment() *comment - pushTrailingComment(*comment) + popLeadingComment() comment + pushTrailingComment(comment) } var _ terminalNode = (*basicNode)(nil) @@ -120,38 +120,38 @@ var _ methodDecl = (*methodNode)(nil) var _ methodDecl = (*noSourceNode)(nil) type posRange struct { - start, end *SourcePos + start, end SourcePos } type basicNode struct { posRange - leading []*comment - trailing []*comment + leading []comment + trailing []comment } func (n *basicNode) start() *SourcePos { - return n.posRange.start + return &n.posRange.start } func (n *basicNode) end() *SourcePos { - return n.posRange.end + return &n.posRange.end } -func (n *basicNode) leadingComments() []*comment { +func (n *basicNode) leadingComments() []comment { return n.leading } -func (n *basicNode) trailingComments() []*comment { +func (n *basicNode) trailingComments() []comment { return n.trailing } -func (n *basicNode) popLeadingComment() *comment { +func (n *basicNode) popLeadingComment() comment { c := n.leading[0] n.leading = n.leading[1:] return c } -func (n *basicNode) pushTrailingComment(c *comment) { +func (n *basicNode) pushTrailingComment(c comment) { n.trailing = append(n.trailing, c) } @@ -173,11 +173,11 @@ func (n *basicCompositeNode) end() *SourcePos { return n.last.end() } -func (n *basicCompositeNode) leadingComments() []*comment { +func (n *basicCompositeNode) leadingComments() []comment { return n.first.leadingComments() } -func (n *basicCompositeNode) trailingComments() []*comment { +func (n *basicCompositeNode) trailingComments() []comment { return n.last.trailingComments() } @@ -216,11 +216,11 @@ func (n *fileElement) end() *SourcePos { return n.get().end() } -func (n *fileElement) leadingComments() []*comment { +func (n *fileElement) leadingComments() []comment { return n.get().leadingComments() } -func (n *fileElement) trailingComments() []*comment { +func (n *fileElement) trailingComments() []comment { return n.get().trailingComments() } @@ -282,11 +282,11 @@ func (n *identNode) value() interface{} { return identifier(n.val) } -func (n *identNode) popLeadingComment() *comment { +func (n *identNode) popLeadingComment() comment { return n.first.(terminalNode).popLeadingComment() } -func (n *identNode) pushTrailingComment(c *comment) { +func (n *identNode) pushTrailingComment(c comment) { n.last.(terminalNode).pushTrailingComment(c) } @@ -380,11 +380,11 @@ func (n *stringLiteralNode) value() interface{} { return n.val } -func (n *stringLiteralNode) popLeadingComment() *comment { +func (n *stringLiteralNode) popLeadingComment() comment { return n.first.(terminalNode).popLeadingComment() } -func (n *stringLiteralNode) pushTrailingComment(c *comment) { +func (n *stringLiteralNode) pushTrailingComment(c comment) { n.last.(terminalNode).pushTrailingComment(c) } @@ -415,11 +415,11 @@ func (n *floatLiteralNode) value() interface{} { return n.val } -func (n *floatLiteralNode) popLeadingComment() *comment { +func (n *floatLiteralNode) popLeadingComment() comment { return n.first.(terminalNode).popLeadingComment() } -func (n *floatLiteralNode) pushTrailingComment(c *comment) { +func (n *floatLiteralNode) pushTrailingComment(c comment) { n.last.(terminalNode).pushTrailingComment(c) } @@ -591,11 +591,11 @@ func (n *oneOfElement) end() *SourcePos { return n.get().end() } -func (n *oneOfElement) leadingComments() []*comment { +func (n *oneOfElement) leadingComments() []comment { return n.get().leadingComments() } -func (n *oneOfElement) trailingComments() []*comment { +func (n *oneOfElement) trailingComments() []comment { return n.get().trailingComments() } @@ -664,7 +664,7 @@ func (n *mapFieldNode) valueField() *syntheticMapField { func newSyntheticMapField(ident *identNode, tagNum uint64) *syntheticMapField { tag := &intLiteralNode{ basicNode: basicNode{ - posRange: posRange{start: ident.start(), end: ident.end()}, + posRange: posRange{start: *ident.start(), end: *ident.end()}, }, val: tagNum, } @@ -684,11 +684,11 @@ func (n *syntheticMapField) end() *SourcePos { return n.ident.end() } -func (n *syntheticMapField) leadingComments() []*comment { +func (n *syntheticMapField) leadingComments() []comment { return nil } -func (n *syntheticMapField) trailingComments() []*comment { +func (n *syntheticMapField) trailingComments() []comment { return nil } @@ -764,11 +764,11 @@ func (n *enumElement) end() *SourcePos { return n.get().end() } -func (n *enumElement) leadingComments() []*comment { +func (n *enumElement) leadingComments() []comment { return n.get().leadingComments() } -func (n *enumElement) trailingComments() []*comment { +func (n *enumElement) trailingComments() []comment { return n.get().trailingComments() } @@ -838,11 +838,11 @@ func (n *messageElement) end() *SourcePos { return n.get().end() } -func (n *messageElement) leadingComments() []*comment { +func (n *messageElement) leadingComments() []comment { return n.get().leadingComments() } -func (n *messageElement) trailingComments() []*comment { +func (n *messageElement) trailingComments() []comment { return n.get().trailingComments() } @@ -894,11 +894,11 @@ func (n *extendElement) end() *SourcePos { return n.get().end() } -func (n *extendElement) leadingComments() []*comment { +func (n *extendElement) leadingComments() []comment { return n.get().leadingComments() } -func (n *extendElement) trailingComments() []*comment { +func (n *extendElement) trailingComments() []comment { return n.get().trailingComments() } @@ -934,11 +934,11 @@ func (n *serviceElement) end() *SourcePos { return n.get().end() } -func (n *serviceElement) leadingComments() []*comment { +func (n *serviceElement) leadingComments() []comment { return n.get().leadingComments() } -func (n *serviceElement) trailingComments() []*comment { +func (n *serviceElement) trailingComments() []comment { return n.get().trailingComments() } @@ -987,11 +987,11 @@ func (n noSourceNode) end() *SourcePos { return n.pos } -func (n noSourceNode) leadingComments() []*comment { +func (n noSourceNode) leadingComments() []comment { return nil } -func (n noSourceNode) trailingComments() []*comment { +func (n noSourceNode) trailingComments() []comment { return nil } diff --git a/desc/protoparse/lexer.go b/desc/protoparse/lexer.go index 15e46e46..95162970 100644 --- a/desc/protoparse/lexer.go +++ b/desc/protoparse/lexer.go @@ -55,6 +55,11 @@ type protoLex struct { offset int prevSym terminalNode + + prevLineNo int + prevColNo int + prevOffset int + comments []comment } func newLexer(in io.Reader) *protoLex { @@ -106,8 +111,8 @@ var keywords = map[string]int{ "returns": _RETURNS, } -func (l *protoLex) cur() *SourcePos { - return &SourcePos{ +func (l *protoLex) cur() SourcePos { + return SourcePos{ Filename: l.filename, Offset: l.offset, Line: l.lineNo + 1, @@ -153,127 +158,10 @@ func (l *protoLex) Lex(lval *protoSymType) int { return _ERROR } - prevLineNo := l.lineNo - prevColNo := l.colNo - prevOffset := l.offset - var comments []*comment - - pos := func() posRange { - return posRange{ - start: &SourcePos{ - Filename: l.filename, - Offset: prevOffset, - Line: prevLineNo + 1, - Col: prevColNo + 1, - }, - end: l.cur(), - } - } - basic := func() basicNode { - return basicNode{ - posRange: pos(), - leading: comments, - } - } - setPrev := func(n terminalNode) { - nStart := n.start().Line - if _, ok := n.(*basicNode); ok { - // if the node is a simple rune, don't attribute comments to it - // HACK: adjusting the start line makes leading comments appear - // detached so logic below will naturally associated trailing - // comment to previous symbol - nStart += 2 - } - if l.prevSym != nil && len(n.leadingComments()) > 0 && l.prevSym.end().Line < nStart { - // we may need to re-attribute the first comment to - // instead be previous node's trailing comment - prevEnd := l.prevSym.end().Line - comments := n.leadingComments() - c := comments[0] - commentStart := c.start.Line - if commentStart == prevEnd { - // comment is on same line as previous symbol - n.popLeadingComment() - l.prevSym.pushTrailingComment(c) - } else if commentStart == prevEnd+1 { - // comment is right after previous symbol; see if it is detached - // and if so re-attribute - singleLineStyle := strings.HasPrefix(c.text, "//") - line := c.end.Line - groupEnd := -1 - for i := 1; i < len(comments); i++ { - c := comments[i] - newGroup := false - if !singleLineStyle || c.start.Line > line+1 { - // we've found a gap between comments, which means the - // previous comments were detached - newGroup = true - } else { - line = c.end.Line - singleLineStyle = strings.HasPrefix(comments[i].text, "//") - if !singleLineStyle { - // we've found a switch from // comments to /* - // consider that a new group which means the - // previous comments were detached - newGroup = true - } - } - if newGroup { - groupEnd = i - break - } - } - - if groupEnd == -1 { - // just one group of comments; we'll mark it as a trailing - // comment if it immediately follows previous symbol and is - // detached from current symbol - c1 := comments[0] - c2 := comments[len(comments)-1] - if c1.start.Line <= prevEnd+1 && c2.end.Line < nStart-1 { - groupEnd = len(comments) - } - } - - for i := 0; i < groupEnd; i++ { - l.prevSym.pushTrailingComment(n.popLeadingComment()) - } - } - } - - l.prevSym = n - } - setString := func(val string) { - b := basic() - lval.str = &stringLiteralNode{val: val} - lval.str.setRange(&b, &b) - setPrev(lval.str) - } - setIdent := func(val string, kind identKind) { - b := basic() - lval.id = &identNode{val: val, kind: kind} - lval.id.setRange(&b, &b) - setPrev(lval.id) - } - setInt := func(val uint64) { - lval.ui = &intLiteralNode{basicNode: basic(), val: val} - setPrev(lval.ui) - } - setFloat := func(val float64) { - b := basic() - lval.f = &floatLiteralNode{val: val} - lval.f.setRange(&b, &b) - setPrev(lval.f) - } - setRune := func() { - b := basic() - lval.b = &b - setPrev(lval.b) - } - setError := func(err error) { - lval.err = err - l.err = err - } + l.prevLineNo = l.lineNo + l.prevColNo = l.colNo + l.prevOffset = l.offset + l.comments = nil for { c, n, err := l.input.readRune() @@ -281,16 +169,16 @@ func (l *protoLex) Lex(lval *protoSymType) int { // we're not actually returning a rune, but this will associate // accumulated comments as a trailing comment on last symbol // (if appropriate) - setRune() + l.setRune(lval) return 0 } else if err != nil { - setError(err) + l.setError(lval, err) return _ERROR } - prevLineNo = l.lineNo - prevColNo = l.colNo - prevOffset = l.offset + l.prevLineNo = l.lineNo + l.prevColNo = l.colNo + l.prevOffset = l.offset l.offset += n l.adjustPos(c) @@ -302,14 +190,14 @@ func (l *protoLex) Lex(lval *protoSymType) int { // tokens that start with a dot include type names and decimal literals cn, _, err := l.input.readRune() if err != nil { - setRune() + l.setRune(lval) return int(c) } if cn == '_' || (cn >= 'a' && cn <= 'z') || (cn >= 'A' && cn <= 'Z') { l.adjustPos(cn) token := []rune{c, cn} token = l.readIdentifier(token) - setIdent(string(token), identTypeName) + l.setIdent(lval, string(token), identTypeName) return _TYPENAME } if cn >= '0' && cn <= '9' { @@ -318,14 +206,14 @@ func (l *protoLex) Lex(lval *protoSymType) int { token = l.readNumber(token, false, true) f, err := strconv.ParseFloat(string(token), 64) if err != nil { - setError(err) + l.setError(lval, err) return _ERROR } - setFloat(f) + l.setFloat(lval, f) return _FLOAT_LIT } l.input.unreadRune(cn) - setRune() + l.setRune(lval) return int(c) } @@ -335,14 +223,14 @@ func (l *protoLex) Lex(lval *protoSymType) int { token = l.readIdentifier(token) str := string(token) if strings.Contains(str, ".") { - setIdent(str, identQualified) + l.setIdent(lval, str, identQualified) return _FQNAME } if t, ok := keywords[str]; ok { - setIdent(str, identSimpleName) + l.setIdent(lval, str, identSimpleName) return t } - setIdent(str, identSimpleName) + l.setIdent(lval, str, identSimpleName) return _NAME } @@ -351,14 +239,14 @@ func (l *protoLex) Lex(lval *protoSymType) int { if c == '0' { cn, _, err := l.input.readRune() if err != nil { - setInt(0) + l.setInt(lval, 0) return _INT_LIT } if cn == 'x' || cn == 'X' { cnn, _, err := l.input.readRune() if err != nil { l.input.unreadRune(cn) - setInt(0) + l.setInt(lval, 0) return _INT_LIT } if (cnn >= '0' && cnn <= '9') || (cnn >= 'a' && cnn <= 'f') || (cnn >= 'A' && cnn <= 'F') { @@ -368,15 +256,15 @@ func (l *protoLex) Lex(lval *protoSymType) int { token = l.readHexNumber(token) ui, err := strconv.ParseUint(string(token), 16, 64) if err != nil { - setError(err) + l.setError(lval, err) return _ERROR } - setInt(ui) + l.setInt(lval, ui) return _INT_LIT } l.input.unreadRune(cnn) l.input.unreadRune(cn) - setInt(0) + l.setInt(lval, 0) return _INT_LIT } else { l.input.unreadRune(cn) @@ -389,19 +277,19 @@ func (l *protoLex) Lex(lval *protoSymType) int { // floating point! f, err := strconv.ParseFloat(numstr, 64) if err != nil { - setError(err) + l.setError(lval, err) return _ERROR } - setFloat(f) + l.setFloat(lval, f) return _FLOAT_LIT } // integer! (decimal or octal) ui, err := strconv.ParseUint(numstr, 0, 64) if err != nil { - setError(err) + l.setError(lval, err) return _ERROR } - setInt(ui) + l.setInt(lval, ui) return _INT_LIT } @@ -409,10 +297,10 @@ func (l *protoLex) Lex(lval *protoSymType) int { // string literal str, err := l.readStringLiteral(c) if err != nil { - setError(err) + l.setError(lval, err) return _ERROR } - setString(str) + l.setString(lval, str) return _STRING_LIT } @@ -420,13 +308,13 @@ func (l *protoLex) Lex(lval *protoSymType) int { // comment cn, _, err := l.input.readRune() if err != nil { - setRune() + l.setRune(lval) return int(c) } if cn == '/' { l.adjustPos(cn) hitNewline, txt := l.skipToEndOfLineComment() - commentPos := pos() + commentPos := l.posRange() commentPos.end.Col++ if hitNewline { // we don't do this inside of skipToEndOfLineComment @@ -434,27 +322,152 @@ func (l *protoLex) Lex(lval *protoSymType) int { // line for calculation above l.adjustPos('\n') } - comments = append(comments, &comment{posRange: commentPos, text: txt}) + l.comments = append(l.comments, comment{posRange: commentPos, text: txt}) continue } if cn == '*' { l.adjustPos(cn) if txt, ok := l.skipToEndOfBlockComment(); !ok { - setError(errors.New("block comment never terminates, unexpected EOF")) + l.setError(lval, errors.New("block comment never terminates, unexpected EOF")) return _ERROR } else { - comments = append(comments, &comment{posRange: pos(), text: txt}) + l.comments = append(l.comments, comment{posRange: l.posRange(), text: txt}) } continue } l.input.unreadRune(cn) } - setRune() + l.setRune(lval) return int(c) } } +func (l *protoLex) posRange() posRange { + return posRange{ + start: SourcePos{ + Filename: l.filename, + Offset: l.prevOffset, + Line: l.prevLineNo + 1, + Col: l.prevColNo + 1, + }, + end: l.cur(), + } +} + +func (l *protoLex) newBasicNode() basicNode { + return basicNode{ + posRange: l.posRange(), + leading: l.comments, + } +} + +func (l *protoLex) setPrev(n terminalNode) { + nStart := n.start().Line + if _, ok := n.(*basicNode); ok { + // if the node is a simple rune, don't attribute comments to it + // HACK: adjusting the start line makes leading comments appear + // detached so logic below will naturally associated trailing + // comment to previous symbol + nStart += 2 + } + if l.prevSym != nil && len(n.leadingComments()) > 0 && l.prevSym.end().Line < nStart { + // we may need to re-attribute the first comment to + // instead be previous node's trailing comment + prevEnd := l.prevSym.end().Line + comments := n.leadingComments() + c := comments[0] + commentStart := c.start.Line + if commentStart == prevEnd { + // comment is on same line as previous symbol + n.popLeadingComment() + l.prevSym.pushTrailingComment(c) + } else if commentStart == prevEnd+1 { + // comment is right after previous symbol; see if it is detached + // and if so re-attribute + singleLineStyle := strings.HasPrefix(c.text, "//") + line := c.end.Line + groupEnd := -1 + for i := 1; i < len(comments); i++ { + c := comments[i] + newGroup := false + if !singleLineStyle || c.start.Line > line+1 { + // we've found a gap between comments, which means the + // previous comments were detached + newGroup = true + } else { + line = c.end.Line + singleLineStyle = strings.HasPrefix(comments[i].text, "//") + if !singleLineStyle { + // we've found a switch from // comments to /* + // consider that a new group which means the + // previous comments were detached + newGroup = true + } + } + if newGroup { + groupEnd = i + break + } + } + + if groupEnd == -1 { + // just one group of comments; we'll mark it as a trailing + // comment if it immediately follows previous symbol and is + // detached from current symbol + c1 := comments[0] + c2 := comments[len(comments)-1] + if c1.start.Line <= prevEnd+1 && c2.end.Line < nStart-1 { + groupEnd = len(comments) + } + } + + for i := 0; i < groupEnd; i++ { + l.prevSym.pushTrailingComment(n.popLeadingComment()) + } + } + } + + l.prevSym = n +} + +func (l *protoLex) setString(lval *protoSymType, val string) { + b := l.newBasicNode() + lval.str = &stringLiteralNode{val: val} + lval.str.setRange(&b, &b) + l.setPrev(lval.str) +} + +func (l *protoLex) setIdent(lval *protoSymType, val string, kind identKind) { + b := l.newBasicNode() + lval.id = &identNode{val: val, kind: kind} + lval.id.setRange(&b, &b) + l.setPrev(lval.id) +} + +func (l *protoLex) setInt(lval *protoSymType, val uint64) { + lval.ui = &intLiteralNode{basicNode: l.newBasicNode(), val: val} + l.setPrev(lval.ui) +} + +func (l *protoLex) setFloat(lval *protoSymType, val float64) { + b := l.newBasicNode() + lval.f = &floatLiteralNode{val: val} + lval.f.setRange(&b, &b) + l.setPrev(lval.f) +} + +func (l *protoLex) setRune(lval *protoSymType) { + b := l.newBasicNode() + lval.b = &b + l.setPrev(lval.b) +} + +func (l *protoLex) setError(lval *protoSymType, err error) { + lval.err = err + l.err = err +} + func (l *protoLex) readNumber(sofar []rune, allowDot bool, allowExp bool) []rune { token := sofar for { diff --git a/desc/protoparse/lexer_test.go b/desc/protoparse/lexer_test.go index c409b05c..678751a0 100644 --- a/desc/protoparse/lexer_test.go +++ b/desc/protoparse/lexer_test.go @@ -173,7 +173,7 @@ foo } testutil.Eq(t, len(exp.comments)-exp.trailCount, len(n.leadingComments()), "case %d: wrong number of comments", i) for ci := range exp.comments { - var c *comment + var c comment if ci < exp.trailCount { c = prev.trailingComments()[ci] } else { diff --git a/desc/protoparse/source_code_info.go b/desc/protoparse/source_code_info.go index 00826fa1..92aa2399 100644 --- a/desc/protoparse/source_code_info.go +++ b/desc/protoparse/source_code_info.go @@ -412,7 +412,10 @@ func (sci *sourceCodeInfo) newLoc(n node, path []int32) { trailingComments = nil } detached := groupComments(leadingComments) - trail := combineComments(trailingComments) + var trail *string + if str, ok := combineComments(trailingComments); ok { + trail = proto.String(str) + } var lead *string if len(leadingComments) > 0 && leadingComments[len(leadingComments)-1].end.Line >= n.start().Line-1 { lead = proto.String(detached[len(detached)-1]) @@ -436,19 +439,19 @@ func makeSpan(start, end *SourcePos) []int32 { return []int32{int32(start.Line) - 1, int32(start.Col) - 1, int32(end.Line) - 1, int32(end.Col) - 1} } -func (sci *sourceCodeInfo) commentUsed(c []*comment) bool { +func (sci *sourceCodeInfo) commentUsed(c []comment) bool { if len(c) == 0 { return false } - if _, ok := sci.commentsUsed[c[0]]; ok { + if _, ok := sci.commentsUsed[&c[0]]; ok { return true } - sci.commentsUsed[c[0]] = struct{}{} + sci.commentsUsed[&c[0]] = struct{}{} return false } -func groupComments(comments []*comment) []string { +func groupComments(comments []comment) []string { if len(comments) == 0 { return nil } @@ -463,20 +466,23 @@ func groupComments(comments []*comment) []string { singleLineStyle = strings.HasPrefix(comments[i].text, "//") if !singleLineStyle || prevSingleLine != singleLineStyle || c.start.Line > line+1 { // new group! - groups = append(groups, *combineComments(comments[start:i])) + if str, ok := combineComments(comments[start:i]); ok { + groups = append(groups, str) + } start = i } line = c.end.Line } // don't forget last group - groups = append(groups, *combineComments(comments[start:])) - + if str, ok := combineComments(comments[start:]); ok { + groups = append(groups, str) + } return groups } -func combineComments(comments []*comment) *string { +func combineComments(comments []comment) (string, bool) { if len(comments) == 0 { - return nil + return "", false } var buf bytes.Buffer for _, c := range comments { @@ -512,7 +518,7 @@ func combineComments(comments []*comment) *string { } } } - return proto.String(buf.String()) + return buf.String(), true } func dup(p []int32) []int32 {