From f49267bbc41a5d67309862182f7eca296be330d1 Mon Sep 17 00:00:00 2001 From: Pedro Loureiro <1092466+pedroql@users.noreply.github.com> Date: Fri, 21 Aug 2020 13:18:30 +0100 Subject: [PATCH 1/8] Add interpolated strings to kotlin --- lexers/k/kotlin.go | 26 ++++++++++- lexers/testdata/kotlin.actual | 17 +++++++ lexers/testdata/kotlin.expected | 81 +++++++++++++++++++++++++++++++-- 3 files changed, 118 insertions(+), 6 deletions(-) diff --git a/lexers/k/kotlin.go b/lexers/k/kotlin.go index 3e90af7d3..843646a7f 100644 --- a/lexers/k/kotlin.go +++ b/lexers/k/kotlin.go @@ -28,8 +28,9 @@ var Kotlin = internal.Register(MustNewLexer( {`%=|&&|\*=|\+\+|\+=|--|-=|->|\.\.|\/=|::|<=|==|>=|!!|!=|\|\||\?[:.]`, Operator, nil}, {`[~!%^&*()+=|\[\]:;,.<>\/?-]`, Punctuation, nil}, {`[{}]`, Punctuation, nil}, - {`"""[^"]*"""`, LiteralString, nil}, - {`"(\\\\|\\"|[^"\n])*["\n]`, LiteralString, nil}, + {`"""`, LiteralString, Push("rawstring")}, + {`"`, LiteralStringDouble, Push("string")}, + {`(')(\\u[0-9a-fA-F]{4})(')`, ByGroups(LiteralStringChar,LiteralStringEscape,LiteralStringChar), nil}, {`'\\.'|'[^\\]'`, LiteralStringChar, nil}, {`0[xX][0-9a-fA-F]+[Uu]?[Ll]?|[0-9]+(\.[0-9]*)?([eE][+-][0-9]+)?[fF]?[Uu]?[Ll]?`, LiteralNumber, nil}, {`(companion)(\s+)(object)`, ByGroups(Keyword, Text, Keyword), nil}, @@ -52,5 +53,26 @@ var Kotlin = internal.Register(MustNewLexer( "function": { {"(@?[" + kotlinIdentifier + " ]*`)", NameFunction, Pop(1)}, }, + "rawstring": { + // raw strings don't allow character escaping + {`"""`, LiteralString, Pop(1)}, + {`(?:[^$"]+|\"{1,2}[^"])+`, LiteralString, nil}, + Include("string-interpol"), + // remaining dollar signs are just a string + {`\$`, LiteralString, nil}, + }, + "string": { + {`\\[tbnr'"\\\$]`, LiteralStringEscape, nil}, + {`\\u[0-9a-fA-F]{4}`, LiteralStringEscape, nil}, + {`"`, LiteralStringDouble, Pop(1)}, + Include("string-interpol"), + {`[^\n\\"$]+`, LiteralStringDouble, nil}, + // remaining dollar signs are just a string + {`\$`, LiteralStringDouble, nil}, + }, + "string-interpol": { + {`\$[` + kotlinIdentifier + `]+`, LiteralStringInterpol, nil}, + {`\${[^}\n]*}`, LiteralStringInterpol, nil}, + }, }, )) diff --git a/lexers/testdata/kotlin.actual b/lexers/testdata/kotlin.actual index e28a03ab6..2a6db54ad 100644 --- a/lexers/testdata/kotlin.actual +++ b/lexers/testdata/kotlin.actual @@ -21,6 +21,23 @@ fun nullable2(nullable: String?): Int = nullable?.length ?: run { 1 + 2 } +val rawStringWithQuotes = """ +Hello "example" ${1 + 2} +And now { Just the braces } +Escapes here don't work so this is just text \t \n \u1234 $ \$ +""" + +fun returnsSomething(): Int { + return "".length +} + +fun returnsSomethingElse(): Int = "\\\n".length + +val character = '"' +val escapedCharacter = '\"' +// escaping a double quote character inside a character is optional +val stringWithSingleQuote = "'" + typealias MySecretAlias = (A, B) -> Unit val impl : MySecretAlias = { a, _ -> Unit } diff --git a/lexers/testdata/kotlin.expected b/lexers/testdata/kotlin.expected index f562eee9e..42fbd0e9e 100644 --- a/lexers/testdata/kotlin.expected +++ b/lexers/testdata/kotlin.expected @@ -31,10 +31,10 @@ {"type":"NameProperty","value":"with"}, {"type":"Text","value":" "}, {"type":"Name","value":"spaces"}, - {"type":"LiteralString","value":"\" + ` = \""}, + {"type":"LiteralStringDouble","value":"\" + ` = \""}, {"type":"Name","value":"hello"}, - {"type":"LiteralString","value":"\"\n"}, - {"type":"Text","value":" "}, + {"type":"LiteralStringDouble","value":"\""}, + {"type":"Text","value":"\n "}, {"type":"Keyword","value":"val"}, {"type":"Text","value":" "}, {"type":"NameProperty","value":"multiline"}, @@ -130,6 +130,73 @@ {"type":"Text","value":"\n"}, {"type":"Punctuation","value":"}"}, {"type":"Text","value":"\n\n"}, + {"type":"Keyword","value":"val"}, + {"type":"Text","value":" "}, + {"type":"NameProperty","value":"rawStringWithQuotes"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":" "}, + {"type":"LiteralString","value":"\"\"\"\nHello \"example\" "}, + {"type":"LiteralStringInterpol","value":"${1 + 2}"}, + {"type":"LiteralString","value":"\nAnd now { Just the braces }\nEscapes here don't work so this is just text \\t \\n \\u1234 $ \\$\n\"\"\""}, + {"type":"Text","value":"\n\n"}, + {"type":"Keyword","value":"fun"}, + {"type":"Text","value":" "}, + {"type":"NameFunction","value":"returnsSomething"}, + {"type":"Punctuation","value":"():"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Int"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"{"}, + {"type":"Text","value":"\n "}, + {"type":"Keyword","value":"return"}, + {"type":"Text","value":" "}, + {"type":"LiteralStringDouble","value":"\"\""}, + {"type":"Punctuation","value":"."}, + {"type":"Name","value":"length"}, + {"type":"Text","value":"\n"}, + {"type":"Punctuation","value":"}"}, + {"type":"Text","value":"\n\n"}, + {"type":"Keyword","value":"fun"}, + {"type":"Text","value":" "}, + {"type":"NameFunction","value":"returnsSomethingElse"}, + {"type":"Punctuation","value":"():"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Int"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":" "}, + {"type":"LiteralStringDouble","value":"\""}, + {"type":"LiteralStringEscape","value":"\\\\\\n"}, + {"type":"LiteralStringDouble","value":"\""}, + {"type":"Punctuation","value":"."}, + {"type":"Name","value":"length"}, + {"type":"Text","value":"\n\n"}, + {"type":"Keyword","value":"val"}, + {"type":"Text","value":" "}, + {"type":"NameProperty","value":"character"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":" "}, + {"type":"LiteralStringChar","value":"'\"'"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"val"}, + {"type":"Text","value":" "}, + {"type":"NameProperty","value":"escapedCharacter"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":" "}, + {"type":"LiteralStringChar","value":"'\\\"'"}, + {"type":"Text","value":"\n"}, + {"type":"CommentSingle","value":"// escaping a double quote character inside a character is optional\n"}, + {"type":"Keyword","value":"val"}, + {"type":"Text","value":" "}, + {"type":"NameProperty","value":"stringWithSingleQuote"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":" "}, + {"type":"LiteralStringDouble","value":"\"'\""}, + {"type":"Text","value":"\n\n"}, {"type":"Keyword","value":"typealias"}, {"type":"Text","value":" "}, {"type":"Name","value":"MySecretAlias"}, @@ -214,7 +281,13 @@ {"type":"Text","value":"\n "}, {"type":"Name","value":"println"}, {"type":"Punctuation","value":"("}, - {"type":"LiteralString","value":"\"This is an example a = $a and the sum is ${a + b} ${ A.foo() }\""}, + {"type":"LiteralStringDouble","value":"\"This is an example a = "}, + {"type":"LiteralStringInterpol","value":"$a"}, + {"type":"LiteralStringDouble","value":" and the sum is "}, + {"type":"LiteralStringInterpol","value":"${a + b}"}, + {"type":"LiteralStringDouble","value":" "}, + {"type":"LiteralStringInterpol","value":"${ A.foo() }"}, + {"type":"LiteralStringDouble","value":"\""}, {"type":"Punctuation","value":")"}, {"type":"Text","value":"\n "}, {"type":"Name","value":"println"}, From 3180765401ca5b6075da2c822fcff4049e561bcf Mon Sep 17 00:00:00 2001 From: Pedro Loureiro <1092466+pedroql@users.noreply.github.com> Date: Fri, 21 Aug 2020 13:55:45 +0100 Subject: [PATCH 2/8] Fix parsing of property/method/classes named with back ticks E.g. fun `Hello there!`() { ... } --- lexers/k/kotlin.go | 5 ++++- lexers/testdata/kotlin.expected | 10 ++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lexers/k/kotlin.go b/lexers/k/kotlin.go index 843646a7f..d65294631 100644 --- a/lexers/k/kotlin.go +++ b/lexers/k/kotlin.go @@ -45,12 +45,15 @@ var Kotlin = internal.Register(MustNewLexer( {`\S+`, NameNamespace, Pop(1)}, }, "class": { - {"(@?[" + kotlinIdentifier + "]*`)", NameClass, Pop(1)}, + {"\x60[^\x60]+?\x60", NameClass, Pop(1)}, + {"(@?[" + kotlinIdentifier + "]*)", NameClass, Pop(1)}, }, "property": { + {"\x60[^\x60]+?\x60", NameProperty, Pop(1)}, {"(@?[" + kotlinIdentifier + " ]*`)", NameProperty, Pop(1)}, }, "function": { + {"\x60[^\x60]+?\x60", NameFunction, Pop(1)}, {"(@?[" + kotlinIdentifier + " ]*`)", NameFunction, Pop(1)}, }, "rawstring": { diff --git a/lexers/testdata/kotlin.expected b/lexers/testdata/kotlin.expected index 42fbd0e9e..837dfa6c1 100644 --- a/lexers/testdata/kotlin.expected +++ b/lexers/testdata/kotlin.expected @@ -27,13 +27,11 @@ {"type":"Text","value":"\n "}, {"type":"Keyword","value":"val"}, {"type":"Text","value":" "}, - {"type":"Error","value":"` + \""}, - {"type":"NameProperty","value":"with"}, + {"type":"NameProperty","value":"` + \"with spaces\" + `"}, {"type":"Text","value":" "}, - {"type":"Name","value":"spaces"}, - {"type":"LiteralStringDouble","value":"\" + ` = \""}, - {"type":"Name","value":"hello"}, - {"type":"LiteralStringDouble","value":"\""}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":" "}, + {"type":"LiteralStringDouble","value":"\"hello\""}, {"type":"Text","value":"\n "}, {"type":"Keyword","value":"val"}, {"type":"Text","value":" "}, From 9b3a60412456aaf0bd3b830b7db73bb44db22132 Mon Sep 17 00:00:00 2001 From: Pedro Loureiro <1092466+pedroql@users.noreply.github.com> Date: Fri, 21 Aug 2020 14:20:52 +0100 Subject: [PATCH 3/8] Add annotation examples to kotlin.actual --- lexers/testdata/kotlin.actual | 15 +++++++ lexers/testdata/kotlin.expected | 75 +++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/lexers/testdata/kotlin.actual b/lexers/testdata/kotlin.actual index 2a6db54ad..4c81ed976 100644 --- a/lexers/testdata/kotlin.actual +++ b/lexers/testdata/kotlin.actual @@ -83,6 +83,21 @@ fun moreOperators(arg: Any?) { println(arg === Unit) } +@Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION, + AnnotationTarget.VALUE_PARAMETER, AnnotationTarget.EXPRESSION) +@Retention(AnnotationRetention.SOURCE) +annotation class Annotated + +@Annotated class A { + @Annotated fun a( + @Annotated param: Int + ) { + + @Annotated val y = 0 + + } +} + class X { companion object { } diff --git a/lexers/testdata/kotlin.expected b/lexers/testdata/kotlin.expected index 837dfa6c1..25ca67e6b 100644 --- a/lexers/testdata/kotlin.expected +++ b/lexers/testdata/kotlin.expected @@ -486,6 +486,81 @@ {"type":"Text","value":"\n"}, {"type":"Punctuation","value":"}"}, {"type":"Text","value":"\n\n"}, + {"type":"Name","value":"@Target"}, + {"type":"Punctuation","value":"("}, + {"type":"Name","value":"AnnotationTarget"}, + {"type":"Punctuation","value":"."}, + {"type":"Name","value":"CLASS"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"Name","value":"AnnotationTarget"}, + {"type":"Punctuation","value":"."}, + {"type":"Name","value":"FUNCTION"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":"\n "}, + {"type":"Name","value":"AnnotationTarget"}, + {"type":"Punctuation","value":"."}, + {"type":"Name","value":"VALUE_PARAMETER"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"Name","value":"AnnotationTarget"}, + {"type":"Punctuation","value":"."}, + {"type":"Name","value":"EXPRESSION"}, + {"type":"Punctuation","value":")"}, + {"type":"Text","value":"\n"}, + {"type":"Name","value":"@Retention"}, + {"type":"Punctuation","value":"("}, + {"type":"Name","value":"AnnotationRetention"}, + {"type":"Punctuation","value":"."}, + {"type":"Name","value":"SOURCE"}, + {"type":"Punctuation","value":")"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"annotation"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"class"}, + {"type":"Text","value":" "}, + {"type":"NameClass","value":"Annotated"}, + {"type":"Text","value":"\n\n"}, + {"type":"Name","value":"@Annotated"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"class"}, + {"type":"Text","value":" "}, + {"type":"NameClass","value":"A"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"{"}, + {"type":"Text","value":"\n "}, + {"type":"Name","value":"@Annotated"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"fun"}, + {"type":"Text","value":" "}, + {"type":"NameFunction","value":"a"}, + {"type":"Punctuation","value":"("}, + {"type":"Text","value":"\n "}, + {"type":"Name","value":"@Annotated"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"param"}, + {"type":"Punctuation","value":":"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Int"}, + {"type":"Text","value":"\n "}, + {"type":"Punctuation","value":")"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"{"}, + {"type":"Text","value":"\n\n "}, + {"type":"Name","value":"@Annotated"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"val"}, + {"type":"Text","value":" "}, + {"type":"NameProperty","value":"y"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":" "}, + {"type":"LiteralNumber","value":"0"}, + {"type":"Text","value":"\n\n "}, + {"type":"Punctuation","value":"}"}, + {"type":"Text","value":"\n"}, + {"type":"Punctuation","value":"}"}, + {"type":"Text","value":"\n\n"}, {"type":"Keyword","value":"class"}, {"type":"Text","value":" "}, {"type":"NameClass","value":"X"}, From 29d735d8dadfc7e5a6794e3e8557ecddaf84749c Mon Sep 17 00:00:00 2001 From: Pedro Loureiro <1092466+pedroql@users.noreply.github.com> Date: Fri, 21 Aug 2020 14:21:27 +0100 Subject: [PATCH 4/8] Fix kotlin expressions for class, method, and properties that do not look correct --- lexers/k/kotlin.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lexers/k/kotlin.go b/lexers/k/kotlin.go index d65294631..78cea5493 100644 --- a/lexers/k/kotlin.go +++ b/lexers/k/kotlin.go @@ -45,16 +45,17 @@ var Kotlin = internal.Register(MustNewLexer( {`\S+`, NameNamespace, Pop(1)}, }, "class": { + // \x60 is the back tick character (`) {"\x60[^\x60]+?\x60", NameClass, Pop(1)}, - {"(@?[" + kotlinIdentifier + "]*)", NameClass, Pop(1)}, + {"[" + kotlinIdentifier + "]+", NameClass, Pop(1)}, }, "property": { {"\x60[^\x60]+?\x60", NameProperty, Pop(1)}, - {"(@?[" + kotlinIdentifier + " ]*`)", NameProperty, Pop(1)}, + {"[" + kotlinIdentifier + "]+", NameProperty, Pop(1)}, }, "function": { {"\x60[^\x60]+?\x60", NameFunction, Pop(1)}, - {"(@?[" + kotlinIdentifier + " ]*`)", NameFunction, Pop(1)}, + {"[" + kotlinIdentifier + "]+", NameFunction, Pop(1)}, }, "rawstring": { // raw strings don't allow character escaping From a54e4c2a858e60131477ae525ab65fe4834130a0 Mon Sep 17 00:00:00 2001 From: Pedro Loureiro <1092466+pedroql@users.noreply.github.com> Date: Fri, 21 Aug 2020 14:30:19 +0100 Subject: [PATCH 5/8] Add annotation support for kotlin --- lexers/k/kotlin.go | 3 ++- lexers/testdata/kotlin.expected | 12 ++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/lexers/k/kotlin.go b/lexers/k/kotlin.go index 78cea5493..3d16ab7a1 100644 --- a/lexers/k/kotlin.go +++ b/lexers/k/kotlin.go @@ -39,7 +39,8 @@ var Kotlin = internal.Register(MustNewLexer( {`(val|var)(\s+)`, ByGroups(Keyword, Text), Push("property")}, {`(fun)(\s+)(<[^>]*>\s+)?`, ByGroups(Keyword, Text, Text), Push("function")}, {`(abstract|actual|annotation|as|as\?|break|by|catch|class|companion|const|constructor|continue|crossinline|data|delegate|do|dynamic|else|enum|expect|external|false|field|file|final|finally|for|fun|get|if|import|in|infix|init|inline|inner|interface|internal|is|it|lateinit|noinline|null|object|open|operator|out|override|package|param|private|property|protected|public|receiver|reified|return|sealed|set|setparam|super|suspend|tailrec|this|throw|true|try|typealias|typeof|val|var|vararg|when|where|while)\b`, Keyword, nil}, - {"(@?[" + kotlinIdentifier + "]*`)", Name, nil}, + {"@[" + kotlinIdentifier + "]+", NameDecorator, nil}, + {"[" + kotlinIdentifier + "]+", Name, nil}, }, "package": { {`\S+`, NameNamespace, Pop(1)}, diff --git a/lexers/testdata/kotlin.expected b/lexers/testdata/kotlin.expected index 25ca67e6b..0f7021107 100644 --- a/lexers/testdata/kotlin.expected +++ b/lexers/testdata/kotlin.expected @@ -486,7 +486,7 @@ {"type":"Text","value":"\n"}, {"type":"Punctuation","value":"}"}, {"type":"Text","value":"\n\n"}, - {"type":"Name","value":"@Target"}, + {"type":"NameDecorator","value":"@Target"}, {"type":"Punctuation","value":"("}, {"type":"Name","value":"AnnotationTarget"}, {"type":"Punctuation","value":"."}, @@ -508,7 +508,7 @@ {"type":"Name","value":"EXPRESSION"}, {"type":"Punctuation","value":")"}, {"type":"Text","value":"\n"}, - {"type":"Name","value":"@Retention"}, + {"type":"NameDecorator","value":"@Retention"}, {"type":"Punctuation","value":"("}, {"type":"Name","value":"AnnotationRetention"}, {"type":"Punctuation","value":"."}, @@ -521,7 +521,7 @@ {"type":"Text","value":" "}, {"type":"NameClass","value":"Annotated"}, {"type":"Text","value":"\n\n"}, - {"type":"Name","value":"@Annotated"}, + {"type":"NameDecorator","value":"@Annotated"}, {"type":"Text","value":" "}, {"type":"Keyword","value":"class"}, {"type":"Text","value":" "}, @@ -529,14 +529,14 @@ {"type":"Text","value":" "}, {"type":"Punctuation","value":"{"}, {"type":"Text","value":"\n "}, - {"type":"Name","value":"@Annotated"}, + {"type":"NameDecorator","value":"@Annotated"}, {"type":"Text","value":" "}, {"type":"Keyword","value":"fun"}, {"type":"Text","value":" "}, {"type":"NameFunction","value":"a"}, {"type":"Punctuation","value":"("}, {"type":"Text","value":"\n "}, - {"type":"Name","value":"@Annotated"}, + {"type":"NameDecorator","value":"@Annotated"}, {"type":"Text","value":" "}, {"type":"Keyword","value":"param"}, {"type":"Punctuation","value":":"}, @@ -547,7 +547,7 @@ {"type":"Text","value":" "}, {"type":"Punctuation","value":"{"}, {"type":"Text","value":"\n\n "}, - {"type":"Name","value":"@Annotated"}, + {"type":"NameDecorator","value":"@Annotated"}, {"type":"Text","value":" "}, {"type":"Keyword","value":"val"}, {"type":"Text","value":" "}, From e5674eecfc5e7a6f24228cecec3f22a51006e3b4 Mon Sep 17 00:00:00 2001 From: Pedro Loureiro <1092466+pedroql@users.noreply.github.com> Date: Fri, 21 Aug 2020 15:43:40 +0100 Subject: [PATCH 6/8] Improve kotlin support for generic methods --- lexers/k/kotlin.go | 28 ++++++++++++++++--------- lexers/testdata/kotlin.actual | 3 +++ lexers/testdata/kotlin.expected | 36 ++++++++++++++++++++++++++++++++- 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/lexers/k/kotlin.go b/lexers/k/kotlin.go index 3d16ab7a1..d5046b0c8 100644 --- a/lexers/k/kotlin.go +++ b/lexers/k/kotlin.go @@ -37,26 +37,36 @@ var Kotlin = internal.Register(MustNewLexer( {`(class|interface|object)(\s+)`, ByGroups(Keyword, Text), Push("class")}, {`(package|import)(\s+)`, ByGroups(Keyword, Text), Push("package")}, {`(val|var)(\s+)`, ByGroups(Keyword, Text), Push("property")}, - {`(fun)(\s+)(<[^>]*>\s+)?`, ByGroups(Keyword, Text, Text), Push("function")}, + {`(fun)(\s+)`, ByGroups(Keyword, Text), Push("function")}, {`(abstract|actual|annotation|as|as\?|break|by|catch|class|companion|const|constructor|continue|crossinline|data|delegate|do|dynamic|else|enum|expect|external|false|field|file|final|finally|for|fun|get|if|import|in|infix|init|inline|inner|interface|internal|is|it|lateinit|noinline|null|object|open|operator|out|override|package|param|private|property|protected|public|receiver|reified|return|sealed|set|setparam|super|suspend|tailrec|this|throw|true|try|typealias|typeof|val|var|vararg|when|where|while)\b`, Keyword, nil}, - {"@[" + kotlinIdentifier + "]+", NameDecorator, nil}, - {"[" + kotlinIdentifier + "]+", Name, nil}, + {`@[` + kotlinIdentifier + `]+`, NameDecorator, nil}, + {`[` + kotlinIdentifier + `]+`, Name, nil}, }, "package": { {`\S+`, NameNamespace, Pop(1)}, }, "class": { // \x60 is the back tick character (`) - {"\x60[^\x60]+?\x60", NameClass, Pop(1)}, - {"[" + kotlinIdentifier + "]+", NameClass, Pop(1)}, + {`\x60[^\x60]+?\x60`, NameClass, Pop(1)}, + {`[` + kotlinIdentifier + `]+`, NameClass, Pop(1)}, }, "property": { - {"\x60[^\x60]+?\x60", NameProperty, Pop(1)}, - {"[" + kotlinIdentifier + "]+", NameProperty, Pop(1)}, + {`\x60[^\x60]+?\x60`, NameProperty, Pop(1)}, + {`[` + kotlinIdentifier + `]+`, NameProperty, Pop(1)}, + }, + "generics-specification": { + {`>`, Punctuation, Pop(1)}, + {`[,:*?]`, Punctuation, nil}, + {`(in|out|reified)`, Keyword, nil}, + {`\x60[^\x60]+?\x60`, NameClass, nil}, + {`[` + kotlinIdentifier + `]+`, NameClass, nil}, + {`\s+`, Text, nil}, }, "function": { - {"\x60[^\x60]+?\x60", NameFunction, Pop(1)}, - {"[" + kotlinIdentifier + "]+", NameFunction, Pop(1)}, + {`<`, Punctuation, Push("generics-specification")}, + {`\x60[^\x60]+?\x60`, NameFunction, Pop(1)}, + {`[` + kotlinIdentifier + `]+`, NameFunction, Pop(1)}, + {`\s+`, Text, nil}, }, "rawstring": { // raw strings don't allow character escaping diff --git a/lexers/testdata/kotlin.actual b/lexers/testdata/kotlin.actual index 4c81ed976..1bbdbe8c9 100644 --- a/lexers/testdata/kotlin.actual +++ b/lexers/testdata/kotlin.actual @@ -98,6 +98,9 @@ annotation class Annotated } } +open class TUpper +fun generic() = 123 + class X { companion object { } diff --git a/lexers/testdata/kotlin.expected b/lexers/testdata/kotlin.expected index 0f7021107..c5c508470 100644 --- a/lexers/testdata/kotlin.expected +++ b/lexers/testdata/kotlin.expected @@ -561,6 +561,34 @@ {"type":"Text","value":"\n"}, {"type":"Punctuation","value":"}"}, {"type":"Text","value":"\n\n"}, + {"type":"Keyword","value":"open"}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"class"}, + {"type":"Text","value":" "}, + {"type":"NameClass","value":"TUpper"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"fun"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"NameClass","value":"T"}, + {"type":"Punctuation","value":":"}, + {"type":"Text","value":" "}, + {"type":"NameClass","value":"TUpper"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"NameClass","value":"R"}, + {"type":"Punctuation","value":":"}, + {"type":"Text","value":" "}, + {"type":"NameClass","value":"Any"}, + {"type":"Punctuation","value":"?\u003e"}, + {"type":"Text","value":" "}, + {"type":"NameFunction","value":"generic"}, + {"type":"Punctuation","value":"()"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":" "}, + {"type":"LiteralNumber","value":"123"}, + {"type":"Text","value":"\n\n"}, {"type":"Keyword","value":"class"}, {"type":"Text","value":" "}, {"type":"NameClass","value":"X"}, @@ -580,7 +608,13 @@ {"type":"Keyword","value":"inline"}, {"type":"Text","value":" "}, {"type":"Keyword","value":"fun"}, - {"type":"Text","value":" \u003creified T\u003e "}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"Keyword","value":"reified"}, + {"type":"Text","value":" "}, + {"type":"NameClass","value":"T"}, + {"type":"Punctuation","value":"\u003e"}, + {"type":"Text","value":" "}, {"type":"NameFunction","value":"generic"}, {"type":"Punctuation","value":"("}, {"type":"Name","value":"t"}, From bb6d0a8ab700e52c123e4fa35d1790b3f8967115 Mon Sep 17 00:00:00 2001 From: Pedro Loureiro <1092466+pedroql@users.noreply.github.com> Date: Fri, 21 Aug 2020 16:19:05 +0100 Subject: [PATCH 7/8] Add kotlin generics test cases and fix nested generics --- lexers/k/kotlin.go | 1 + lexers/testdata/kotlin.actual | 18 ++++ lexers/testdata/kotlin.expected | 151 ++++++++++++++++++++++++++++++++ 3 files changed, 170 insertions(+) diff --git a/lexers/k/kotlin.go b/lexers/k/kotlin.go index d5046b0c8..cd66d2866 100644 --- a/lexers/k/kotlin.go +++ b/lexers/k/kotlin.go @@ -55,6 +55,7 @@ var Kotlin = internal.Register(MustNewLexer( {`[` + kotlinIdentifier + `]+`, NameProperty, Pop(1)}, }, "generics-specification": { + {`<`, Punctuation, Push("generics-specification")}, // required for generics inside generics e.g. > {`>`, Punctuation, Pop(1)}, {`[,:*?]`, Punctuation, nil}, {`(in|out|reified)`, Keyword, nil}, diff --git a/lexers/testdata/kotlin.actual b/lexers/testdata/kotlin.actual index 1bbdbe8c9..68e05af4c 100644 --- a/lexers/testdata/kotlin.actual +++ b/lexers/testdata/kotlin.actual @@ -101,6 +101,24 @@ annotation class Annotated open class TUpper fun generic() = 123 +class Holder + +class Holder2 + +var holder1: Holder? = null + +class Some( + val holder2: Holder? = null +) { + var holder3: Holder? = null + fun example() { + var holder4: Holder? = null + } +} +fun > sort(list: List) { + +} + class X { companion object { } diff --git a/lexers/testdata/kotlin.expected b/lexers/testdata/kotlin.expected index c5c508470..bf1caaf9a 100644 --- a/lexers/testdata/kotlin.expected +++ b/lexers/testdata/kotlin.expected @@ -591,6 +591,157 @@ {"type":"Text","value":"\n\n"}, {"type":"Keyword","value":"class"}, {"type":"Text","value":" "}, + {"type":"NameClass","value":"Holder"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"Keyword","value":"in"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"A"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"out"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"B"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"Name","value":"C"}, + {"type":"Punctuation","value":":"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"TUpper"}, + {"type":"Punctuation","value":"\u003e"}, + {"type":"Text","value":"\n\n"}, + {"type":"Keyword","value":"class"}, + {"type":"Text","value":" "}, + {"type":"NameClass","value":"Holder2"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"Name","value":"T"}, + {"type":"Punctuation","value":"\u003e"}, + {"type":"Text","value":"\n\n"}, + {"type":"Keyword","value":"var"}, + {"type":"Text","value":" "}, + {"type":"NameProperty","value":"holder1"}, + {"type":"Punctuation","value":":"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Holder"}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"Name","value":"Int"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"Name","value":"String"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"*\u003e?"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"null"}, + {"type":"Text","value":"\n\n"}, + {"type":"Keyword","value":"class"}, + {"type":"Text","value":" "}, + {"type":"NameClass","value":"Some"}, + {"type":"Punctuation","value":"("}, + {"type":"Text","value":"\n "}, + {"type":"Keyword","value":"val"}, + {"type":"Text","value":" "}, + {"type":"NameProperty","value":"holder2"}, + {"type":"Punctuation","value":":"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Holder"}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"Name","value":"Int"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"Name","value":"String"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"*\u003e?"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"null"}, + {"type":"Text","value":"\n"}, + {"type":"Punctuation","value":")"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"{"}, + {"type":"Text","value":"\n "}, + {"type":"Keyword","value":"var"}, + {"type":"Text","value":" "}, + {"type":"NameProperty","value":"holder3"}, + {"type":"Punctuation","value":":"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Holder"}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"Name","value":"Int"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"Name","value":"String"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"*\u003e?"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"null"}, + {"type":"Text","value":"\n "}, + {"type":"Keyword","value":"fun"}, + {"type":"Text","value":" "}, + {"type":"NameFunction","value":"example"}, + {"type":"Punctuation","value":"()"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"{"}, + {"type":"Text","value":"\n "}, + {"type":"Keyword","value":"var"}, + {"type":"Text","value":" "}, + {"type":"NameProperty","value":"holder4"}, + {"type":"Punctuation","value":":"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"Holder"}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"Name","value":"Int"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"Name","value":"String"}, + {"type":"Punctuation","value":","}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"*\u003e?"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"="}, + {"type":"Text","value":" "}, + {"type":"Keyword","value":"null"}, + {"type":"Text","value":"\n "}, + {"type":"Punctuation","value":"}"}, + {"type":"Text","value":"\n"}, + {"type":"Punctuation","value":"}"}, + {"type":"Text","value":"\n"}, + {"type":"Keyword","value":"fun"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"NameClass","value":"T"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":":"}, + {"type":"Text","value":" "}, + {"type":"NameClass","value":"Comparable"}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"NameClass","value":"T"}, + {"type":"Punctuation","value":"\u003e\u003e"}, + {"type":"Text","value":" "}, + {"type":"NameFunction","value":"sort"}, + {"type":"Punctuation","value":"("}, + {"type":"Name","value":"list"}, + {"type":"Punctuation","value":":"}, + {"type":"Text","value":" "}, + {"type":"Name","value":"List"}, + {"type":"Punctuation","value":"\u003c"}, + {"type":"Name","value":"T"}, + {"type":"Punctuation","value":"\u003e)"}, + {"type":"Text","value":" "}, + {"type":"Punctuation","value":"{"}, + {"type":"Text","value":"\n \n"}, + {"type":"Punctuation","value":"}"}, + {"type":"Text","value":"\n\n"}, + {"type":"Keyword","value":"class"}, + {"type":"Text","value":" "}, {"type":"NameClass","value":"X"}, {"type":"Text","value":" "}, {"type":"Punctuation","value":"{"}, From e468839fecb4bf9c3fd2a910008d98caaa5d1de7 Mon Sep 17 00:00:00 2001 From: Pedro Loureiro <1092466+pedroql@users.noreply.github.com> Date: Fri, 21 Aug 2020 22:47:18 +0100 Subject: [PATCH 8/8] run gofmt --- lexers/k/kotlin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lexers/k/kotlin.go b/lexers/k/kotlin.go index cd66d2866..17cdfa629 100644 --- a/lexers/k/kotlin.go +++ b/lexers/k/kotlin.go @@ -30,7 +30,7 @@ var Kotlin = internal.Register(MustNewLexer( {`[{}]`, Punctuation, nil}, {`"""`, LiteralString, Push("rawstring")}, {`"`, LiteralStringDouble, Push("string")}, - {`(')(\\u[0-9a-fA-F]{4})(')`, ByGroups(LiteralStringChar,LiteralStringEscape,LiteralStringChar), nil}, + {`(')(\\u[0-9a-fA-F]{4})(')`, ByGroups(LiteralStringChar, LiteralStringEscape, LiteralStringChar), nil}, {`'\\.'|'[^\\]'`, LiteralStringChar, nil}, {`0[xX][0-9a-fA-F]+[Uu]?[Ll]?|[0-9]+(\.[0-9]*)?([eE][+-][0-9]+)?[fF]?[Uu]?[Ll]?`, LiteralNumber, nil}, {`(companion)(\s+)(object)`, ByGroups(Keyword, Text, Keyword), nil},