Kotlin improvements around string interpolation, escaping, and generics

lexers/k/kotlin.go

@@ -28,29 +28,67 @@ var Kotlin = internal.Register(MustNewLexer(
 			{`%=|&&|\*=|\+\+|\+=|--|-=|->|\.\.|\/=|::|<=|==|>=|!!|!=|\|\||\?[:.]`, Operator, nil},
 			{`[~!%^&*()+=|\[\]:;,.<>\/?-]`, Punctuation, nil},
 			{`[{}]`, Punctuation, nil},
-			{`"""[^"]*"""`, LiteralString, nil},
-			{`"(\\\\|\\"|[^"\n])*["\n]`, LiteralString, nil},
+			{`"""`, LiteralString, Push("rawstring")},
+			{`"`, LiteralStringDouble, Push("string")},
+			{`(')(\\u[0-9a-fA-F]{4})(')`, ByGroups(LiteralStringChar, LiteralStringEscape, LiteralStringChar), nil},
 			{`'\\.'|'[^\\]'`, LiteralStringChar, nil},
 			{`0[xX][0-9a-fA-F]+[Uu]?[Ll]?|[0-9]+(\.[0-9]*)?([eE][+-][0-9]+)?[fF]?[Uu]?[Ll]?`, LiteralNumber, nil},
 			{`(companion)(\s+)(object)`, ByGroups(Keyword, Text, Keyword), nil},
 			{`(class|interface|object)(\s+)`, ByGroups(Keyword, Text), Push("class")},
 			{`(package|import)(\s+)`, ByGroups(Keyword, Text), Push("package")},
 			{`(val|var)(\s+)`, ByGroups(Keyword, Text), Push("property")},
-			{`(fun)(\s+)(<[^>]*>\s+)?`, ByGroups(Keyword, Text, Text), Push("function")},
+			{`(fun)(\s+)`, ByGroups(Keyword, Text), Push("function")},
 			{`(abstract|actual|annotation|as|as\?|break|by|catch|class|companion|const|constructor|continue|crossinline|data|delegate|do|dynamic|else|enum|expect|external|false|field|file|final|finally|for|fun|get|if|import|in|infix|init|inline|inner|interface|internal|is|it|lateinit|noinline|null|object|open|operator|out|override|package|param|private|property|protected|public|receiver|reified|return|sealed|set|setparam|super|suspend|tailrec|this|throw|true|try|typealias|typeof|val|var|vararg|when|where|while)\b`, Keyword, nil},
-			{"(@?[" + kotlinIdentifier + "]*`)", Name, nil},
+			{`@[` + kotlinIdentifier + `]+`, NameDecorator, nil},
+			{`[` + kotlinIdentifier + `]+`, Name, nil},
 		},
 		"package": {
 			{`\S+`, NameNamespace, Pop(1)},
 		},
 		"class": {
-			{"(@?[" + kotlinIdentifier + "]*`)", NameClass, Pop(1)},
+			// \x60 is the back tick character (`)
+			{`\x60[^\x60]+?\x60`, NameClass, Pop(1)},
+			{`[` + kotlinIdentifier + `]+`, NameClass, Pop(1)},
 		},
 		"property": {
-			{"(@?[" + kotlinIdentifier + " ]*`)", NameProperty, Pop(1)},
+			{`\x60[^\x60]+?\x60`, NameProperty, Pop(1)},
+			{`[` + kotlinIdentifier + `]+`, NameProperty, Pop(1)},
+		},
+		"generics-specification": {
+			{`<`, Punctuation, Push("generics-specification")}, // required for generics inside generics e.g. <T : List<Int> >
+			{`>`, Punctuation, Pop(1)},
+			{`[,:*?]`, Punctuation, nil},
+			{`(in|out|reified)`, Keyword, nil},
+			{`\x60[^\x60]+?\x60`, NameClass, nil},
+			{`[` + kotlinIdentifier + `]+`, NameClass, nil},
+			{`\s+`, Text, nil},
 		},
 		"function": {
-			{"(@?[" + kotlinIdentifier + " ]*`)", NameFunction, Pop(1)},
+			{`<`, Punctuation, Push("generics-specification")},
+			{`\x60[^\x60]+?\x60`, NameFunction, Pop(1)},
+			{`[` + kotlinIdentifier + `]+`, NameFunction, Pop(1)},
+			{`\s+`, Text, nil},
+		},
+		"rawstring": {
+			// raw strings don't allow character escaping
+			{`"""`, LiteralString, Pop(1)},
+			{`(?:[^$"]+|\"{1,2}[^"])+`, LiteralString, nil},
+			Include("string-interpol"),
+			// remaining dollar signs are just a string
+			{`\$`, LiteralString, nil},
+		},
+		"string": {
+			{`\\[tbnr'"\\\$]`, LiteralStringEscape, nil},
+			{`\\u[0-9a-fA-F]{4}`, LiteralStringEscape, nil},
+			{`"`, LiteralStringDouble, Pop(1)},
+			Include("string-interpol"),
+			{`[^\n\\"$]+`, LiteralStringDouble, nil},
+			// remaining dollar signs are just a string
+			{`\$`, LiteralStringDouble, nil},
+		},
+		"string-interpol": {
+			{`\$[` + kotlinIdentifier + `]+`, LiteralStringInterpol, nil},
+			{`\${[^}\n]*}`, LiteralStringInterpol, nil},
 		},
 	},
 ))

lexers/testdata/kotlin.actual

@@ -21,6 +21,23 @@ fun nullable2(nullable: String?): Int = nullable?.length ?: run {
     1 + 2
 }
 
+val rawStringWithQuotes = """
+Hello "example" ${1 + 2}
+And now { Just the braces }
+Escapes here don't work so this is just text \t \n \u1234 $ \$
+"""
+
+fun returnsSomething(): Int {
+    return "".length
+}
+
+fun returnsSomethingElse(): Int = "\\\n".length
+
+val character = '"'
+val escapedCharacter = '\"'
+// escaping a double quote character inside a character is optional
+val stringWithSingleQuote = "'"
+
 typealias MySecretAlias<A, B> = (A, B) -> Unit
 
 val impl : MySecretAlias<Int, Int> = { a, _ -> Unit }
@@ -66,6 +83,42 @@ fun moreOperators(arg: Any?) {
     	println(arg === Unit)
 }
 
+@Target(AnnotationTarget.CLASS, AnnotationTarget.FUNCTION,
+        AnnotationTarget.VALUE_PARAMETER, AnnotationTarget.EXPRESSION)
+@Retention(AnnotationRetention.SOURCE)
+annotation class Annotated
+
+@Annotated class A {
+    @Annotated fun a(
+        @Annotated param: Int
+    ) {
+
+        @Annotated val y = 0
+
+    }
+}
+
+open class TUpper
+fun <T: TUpper, R: Any?> generic() = 123
+
+class Holder <in A, out B, C: TUpper>
+
+class Holder2  <T>
+
+var holder1: Holder<Int, String, *>? = null
+
+class Some(
+    val holder2: Holder<Int, String, *>? = null
+) {
+    var holder3: Holder<Int, String, *>? = null
+    fun example() {
+        var holder4: Holder<Int, String, *>? = null
+    }
+}
+fun <T : Comparable<T>> sort(list: List<T>) {
+
+}
+
 class X {
     companion object {
     }