From 7c6d019105fc42520a3bb6a1ea87d74cc1342b78 Mon Sep 17 00:00:00 2001 From: Alberto Marchetti Date: Tue, 22 Sep 2020 13:14:31 +0300 Subject: [PATCH 1/2] Improved support for Go templates: * Multiline comments * Negative numbers --- lexers/g/go.go | 18 ++--- lexers/g/go_test.go | 72 +++++++++++++++++++ lexers/lexers_test.go | 2 +- lexers/testdata/go-html-template.actual | 3 + lexers/testdata/go-html-template.expected | 3 + lexers/testdata/go-text-template.actual | 3 + lexers/testdata/go-text-template.expected | 3 + .../testdata/{http-2.actual => http.2.actual} | 0 .../{http-2.expected => http.2.expected} | 0 9 files changed, 94 insertions(+), 10 deletions(-) create mode 100644 lexers/testdata/go-html-template.actual create mode 100644 lexers/testdata/go-html-template.expected create mode 100644 lexers/testdata/go-text-template.actual create mode 100644 lexers/testdata/go-text-template.expected rename lexers/testdata/{http-2.actual => http.2.actual} (100%) rename lexers/testdata/{http-2.expected => http.2.expected} (100%) diff --git a/lexers/g/go.go b/lexers/g/go.go index 33077e0e3..2efbc390d 100644 --- a/lexers/g/go.go +++ b/lexers/g/go.go @@ -60,13 +60,13 @@ var Go = internal.Register(MustNewLexer( var goTemplateRules = Rules{ "root": { + {`{{(- )?/\*(.|\n)*?\*/( -)?}}`, CommentMultiline, nil}, {`{{[-]?`, CommentPreproc, Push("template")}, {`[^{]+`, Other, nil}, {`{`, Other, nil}, }, "template": { {`[-]?}}`, CommentPreproc, Pop(1)}, - {`/\*.*?\*/`, Comment, nil}, {`(?=}})`, CommentPreproc, Pop(1)}, // Terminate the pipeline {`\(`, Operator, Push("subexpression")}, {`"(\\\\|\\"|[^"])*"`, LiteralString, nil}, @@ -84,15 +84,15 @@ var goTemplateRules = Rules{ {`[$]?[^\W\d]\w*`, NameOther, nil}, {`[$]?\.(?:[^\W\d]\w*)?`, NameAttribute, nil}, {`"(\\\\|\\"|[^"])*"`, LiteralString, nil}, - {`\d+i`, LiteralNumber, nil}, - {`\d+\.\d*([Ee][-+]\d+)?i`, LiteralNumber, nil}, + {`-?\d+i`, LiteralNumber, nil}, + {`-?\d+\.\d*([Ee][-+]\d+)?i`, LiteralNumber, nil}, {`\.\d+([Ee][-+]\d+)?i`, LiteralNumber, nil}, - {`\d+[Ee][-+]\d+i`, LiteralNumber, nil}, - {`\d+(\.\d+[eE][+\-]?\d+|\.\d*|[eE][+\-]?\d+)`, LiteralNumberFloat, nil}, - {`\.\d+([eE][+\-]?\d+)?`, LiteralNumberFloat, nil}, - {`0[0-7]+`, LiteralNumberOct, nil}, - {`0[xX][0-9a-fA-F]+`, LiteralNumberHex, nil}, - {`(0|[1-9][0-9]*)`, LiteralNumberInteger, nil}, + {`-?\d+[Ee][-+]\d+i`, LiteralNumber, nil}, + {`-?\d+(\.\d+[eE][+\-]?\d+|\.\d*|[eE][+\-]?\d+)`, LiteralNumberFloat, nil}, + {`-?\.\d+([eE][+\-]?\d+)?`, LiteralNumberFloat, nil}, + {`-?0[0-7]+`, LiteralNumberOct, nil}, + {`-?0[xX][0-9a-fA-F]+`, LiteralNumberHex, nil}, + {`-?(0|[1-9][0-9]*)`, LiteralNumberInteger, nil}, {`'(\\['"\\abfnrtv]|\\x[0-9a-fA-F]{2}|\\[0-7]{1,3}|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|[^\\])'`, LiteralStringChar, nil}, {"`[^`]*`", LiteralString, nil}, }, diff --git a/lexers/g/go_test.go b/lexers/g/go_test.go index 583681e0f..2b949b2b4 100644 --- a/lexers/g/go_test.go +++ b/lexers/g/go_test.go @@ -23,6 +23,9 @@ func TestGoHTMLTemplateIssue126(t *testing.T) { {{ with .OutputFormats.Get "RSS" }} {{ printf "" .Permalink .MediaType | safeHTML }} {{ end }} + {{/* + Print all pages + */}} {{ range .Data.Pages }} {{ .Title }} @@ -48,3 +51,72 @@ func TestGoHTMLTemplateIssue126(t *testing.T) { assert.Equal(t, source, chroma.Stringify(tokens...)) } } + +func TestGoHTMLTemplateMultilineComments(t *testing.T) { + for _, source := range []string{ + ` +{{/* + This is a multiline comment +*/}} +`, + ` +{{- /* + This is a multiline comment +*/}} +`, + ` +{{/* + This is a multiline comment +*/ -}} +`, + ` +{{- /* + This is a multiline comment +*/ -}} +`, + } { + tokens, err := chroma.Tokenise(GoHTMLTemplate, nil, source) + assert.NoError(t, err) + assert.Equal(t, source, chroma.Stringify(tokens...)) + + // Make sure that there are no errors + for _, token := range tokens { + assert.NotEqual(t, chroma.Error, token.Type) + } + + // Make sure that multiline comments are printed + found := false + for _, token := range tokens { + if token.Type == chroma.CommentMultiline { + found = true + } + } + assert.True(t, found) + } +} + +func TestGoHTMLTemplateNegativeNumber(t *testing.T) { + for _, source := range []string{ + ` +{{ fn -3 }} +`, + } { + tokens, err := chroma.Tokenise(GoHTMLTemplate, nil, source) + assert.NoError(t, err) + assert.Equal(t, source, chroma.Stringify(tokens...)) + + // Make sure that there are no errors + for _, token := range tokens { + assert.NotEqual(t, chroma.Error, token.Type) + } + + // Make sure that negative number is found + found := false + for _, token := range tokens { + if token.Type == chroma.LiteralNumberInteger { + found = true + } + } + assert.True(t, found) + } +} diff --git a/lexers/lexers_test.go b/lexers/lexers_test.go index 93d518928..8468a662d 100644 --- a/lexers/lexers_test.go +++ b/lexers/lexers_test.go @@ -55,7 +55,7 @@ func TestLexers(t *testing.T) { continue } - base := strings.Split(strings.TrimSuffix(file.Name(), filepath.Ext(file.Name())), "-")[0] + base := strings.Split(strings.TrimSuffix(file.Name(), filepath.Ext(file.Name())), ".")[0] lexer := lexers.Get(base) assert.NotNil(t, lexer) diff --git a/lexers/testdata/go-html-template.actual b/lexers/testdata/go-html-template.actual new file mode 100644 index 000000000..725878760 --- /dev/null +++ b/lexers/testdata/go-html-template.actual @@ -0,0 +1,3 @@ +{{/* + This is a multiline comment +*/}} \ No newline at end of file diff --git a/lexers/testdata/go-html-template.expected b/lexers/testdata/go-html-template.expected new file mode 100644 index 000000000..6a32386ed --- /dev/null +++ b/lexers/testdata/go-html-template.expected @@ -0,0 +1,3 @@ +[ + {"type":"CommentMultiline","value":"{{/*\n This is a multiline comment\n*/}}"} +] diff --git a/lexers/testdata/go-text-template.actual b/lexers/testdata/go-text-template.actual new file mode 100644 index 000000000..725878760 --- /dev/null +++ b/lexers/testdata/go-text-template.actual @@ -0,0 +1,3 @@ +{{/* + This is a multiline comment +*/}} \ No newline at end of file diff --git a/lexers/testdata/go-text-template.expected b/lexers/testdata/go-text-template.expected new file mode 100644 index 000000000..6a32386ed --- /dev/null +++ b/lexers/testdata/go-text-template.expected @@ -0,0 +1,3 @@ +[ + {"type":"CommentMultiline","value":"{{/*\n This is a multiline comment\n*/}}"} +] diff --git a/lexers/testdata/http-2.actual b/lexers/testdata/http.2.actual similarity index 100% rename from lexers/testdata/http-2.actual rename to lexers/testdata/http.2.actual diff --git a/lexers/testdata/http-2.expected b/lexers/testdata/http.2.expected similarity index 100% rename from lexers/testdata/http-2.expected rename to lexers/testdata/http.2.expected From 9cbceac4e9e79c3b463351e2983fceace5392202 Mon Sep 17 00:00:00 2001 From: Alberto Marchetti Date: Tue, 22 Sep 2020 15:29:10 +0300 Subject: [PATCH 2/2] Support also single assignment to variable --- lexers/g/go.go | 2 +- lexers/testdata/go-text-template.actual | 5 ++++- lexers/testdata/go-text-template.expected | 22 +++++++++++++++++++++- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/lexers/g/go.go b/lexers/g/go.go index 2efbc390d..ef400b77e 100644 --- a/lexers/g/go.go +++ b/lexers/g/go.go @@ -80,7 +80,7 @@ var goTemplateRules = Rules{ {`\s+`, Whitespace, nil}, {`\(`, Operator, Push("subexpression")}, {`(range|if|else|while|with|template|end|true|false|nil|and|call|html|index|js|len|not|or|print|printf|println|urlquery|eq|ne|lt|le|gt|ge)\b`, Keyword, nil}, - {`\||:=`, Operator, nil}, + {`\||:?=`, Operator, nil}, {`[$]?[^\W\d]\w*`, NameOther, nil}, {`[$]?\.(?:[^\W\d]\w*)?`, NameAttribute, nil}, {`"(\\\\|\\"|[^"])*"`, LiteralString, nil}, diff --git a/lexers/testdata/go-text-template.actual b/lexers/testdata/go-text-template.actual index 725878760..7761c3d34 100644 --- a/lexers/testdata/go-text-template.actual +++ b/lexers/testdata/go-text-template.actual @@ -1,3 +1,6 @@ {{/* This is a multiline comment -*/}} \ No newline at end of file +*/}} + +{{ $myVar := 2 }} +{{ $myVar = 4 }} \ No newline at end of file diff --git a/lexers/testdata/go-text-template.expected b/lexers/testdata/go-text-template.expected index 6a32386ed..f774669c3 100644 --- a/lexers/testdata/go-text-template.expected +++ b/lexers/testdata/go-text-template.expected @@ -1,3 +1,23 @@ [ - {"type":"CommentMultiline","value":"{{/*\n This is a multiline comment\n*/}}"} + {"type":"CommentMultiline","value":"{{/*\n This is a multiline comment\n*/}}"}, + {"type":"Other","value":"\n\n"}, + {"type":"CommentPreproc","value":"{{"}, + {"type":"TextWhitespace","value":" "}, + {"type":"NameOther","value":"$myVar"}, + {"type":"TextWhitespace","value":" "}, + {"type":"Operator","value":":="}, + {"type":"TextWhitespace","value":" "}, + {"type":"NameOther","value":"2"}, + {"type":"TextWhitespace","value":" "}, + {"type":"CommentPreproc","value":"}}"}, + {"type":"Other","value":"\n"}, + {"type":"CommentPreproc","value":"{{"}, + {"type":"TextWhitespace","value":" "}, + {"type":"NameOther","value":"$myVar"}, + {"type":"TextWhitespace","value":" "}, + {"type":"Operator","value":"="}, + {"type":"TextWhitespace","value":" "}, + {"type":"NameOther","value":"4"}, + {"type":"TextWhitespace","value":" "}, + {"type":"CommentPreproc","value":"}}"} ]