Crystal: Improved tokenization

components/prism-crystal.js

@@ -1,51 +1,57 @@
 (function (Prism) {
 	Prism.languages.crystal = Prism.languages.extend('ruby', {
-		keyword: [
-			/\b(?:__DIR__|__END_LINE__|__FILE__|__LINE__|abstract|alias|as|asm|begin|break|case|class|def|do|else|elsif|end|ensure|enum|extend|for|fun|if|include|instance_sizeof|lib|macro|module|next|of|out|pointerof|private|protected|require|rescue|return|select|self|sizeof|struct|super|then|type|typeof|uninitialized|union|unless|until|when|while|with|yield)\b/,
+		'keyword': [
+			/\b(?:__DIR__|__END_LINE__|__FILE__|__LINE__|abstract|alias|annotation|as|asm|begin|break|case|class|def|do|else|elsif|end|ensure|enum|extend|for|fun|if|ifdef|include|instance_sizeof|lib|macro|module|next|of|out|pointerof|private|protected|ptr|require|rescue|return|select|self|sizeof|struct|super|then|type|typeof|undef|uninitialized|union|unless|until|when|while|with|yield)\b/,
 			{
 				pattern: /(\.\s*)(?:is_a|responds_to)\?/,
 				lookbehind: true
 			}
 		],
-
-		number: /\b(?:0b[01_]*[01]|0o[0-7_]*[0-7]|0x[\da-fA-F_]*[\da-fA-F]|(?:\d(?:[\d_]*\d)?)(?:\.[\d_]*\d)?(?:[eE][+-]?[\d_]*\d)?)(?:_(?:[uif](?:8|16|32|64))?)?\b/
+		'number': /\b(?:0b[01_]*[01]|0o[0-7_]*[0-7]|0x[\da-fA-F_]*[\da-fA-F]|(?:\d(?:[\d_]*\d)?)(?:\.[\d_]*\d)?(?:[eE][+-]?[\d_]*\d)?)(?:_(?:[uif](?:8|16|32|64))?)?\b/,
+		'operator': [
+			/->/,
+			Prism.languages.ruby.operator,
+		],
+		'punctuation': /[(){}[\].,;\\]/,
 	});
 
 	Prism.languages.insertBefore('crystal', 'string-literal', {
-		attribute: {
-			pattern: /@\[.+?\]/,
-			alias: 'attr-name',
+		'attribute': {
+			pattern: /@\[.*?\]/,
 			inside: {
-				delimiter: {
+				'delimiter': {
 					pattern: /^@\[|\]$/,
-					alias: 'tag'
+					alias: 'punctuation'
+				},
+				'attribute': {
+					pattern: /^(\s*)\w+/,
+					lookbehind: true,
+					alias: 'class-name'
+				},
+				'args': {
+					pattern: /\S(?:[\s\S]*\S)?/,
+					inside: Prism.languages.crystal
 				},
-				rest: Prism.languages.crystal
 			}
 		},
-
-		expansion: [
-			{
-				pattern: /\{\{.+?\}\}/,
-				inside: {
-					delimiter: {
-						pattern: /^\{\{|\}\}$/,
-						alias: 'tag'
-					},
-					rest: Prism.languages.crystal
-				}
-			},
-			{
-				pattern: /\{%.+?%\}/,
-				inside: {
-					delimiter: {
-						pattern: /^\{%|%\}$/,
-						alias: 'tag'
-					},
-					rest: Prism.languages.crystal
+		'expansion': {
+			pattern: /\{(?:\{.*?\}|%.*?%)\}/,
+			inside: {
+				'content': {
+					pattern: /^(\{.)[\s\S]+(?=.\}$)/,
+					lookbehind: true,
+					inside: Prism.languages.crystal
+				},
+				'delimiter': {
+					pattern: /^\{[\{%]|[\}%]\}$/,
+					alias: 'operator'
 				}
 			}
-		]
+		},
+		'char': {
+			pattern: /'(?:[^\\\r\n]{1,2}|\\(?:.|u(?:[A-Fa-f0-9]{1,4}|\{[A-Fa-f0-9]{1,6}\})))'/,
+			greedy: true
+		}
 	});
 
 }(Prism));

tests/languages/crystal/attribute_feature.test

@@ -8,47 +8,53 @@
 [
 	["attribute", [
 		["delimiter", "@["],
-		"AlwaysInline",
+		["attribute", "AlwaysInline"],
 		["delimiter", "]"]
 	]],
 	["attribute", [
 		["delimiter", "@["],
-		"CallConvention",
-		["punctuation", "("],
-		["string-literal", [
-			["string", "\"X86_StdCall\""]
+		["attribute", "CallConvention"],
+		["args", [
+			["punctuation", "("],
+			["string-literal", [
+				["string", "\"X86_StdCall\""]
+			]],
+			["punctuation", ")"]
 		]],
-		["punctuation", ")"],
 		["delimiter", "]"]
 	]],
 	["attribute", [
 		["delimiter", "@["],
-		"MyAnnotation",
-		["punctuation", "("],
-		["symbol", "key"],
-		["operator", ":"],
-		["string-literal", [
-			["string", "\"value\""]
+		["attribute", "MyAnnotation"],
+		["args", [
+			["punctuation", "("],
+			["symbol", "key"],
+			["operator", ":"],
+			["string-literal", [
+				["string", "\"value\""]
+			]],
+			["punctuation", ","],
+			["symbol", "value"],
+			["operator", ":"],
+			["number", "123"],
+			["punctuation", ")"]
 		]],
-		["punctuation", ","],
-		["symbol", "value"],
-		["operator", ":"],
-		["number", "123"],
-		["punctuation", ")"],
 		["delimiter", "]"]
 	]],
 	["attribute", [
 		["delimiter", "@["],
-		"MyAnnotation",
-		["punctuation", "("],
-		["string-literal", [
-			["string", "\"foo\""]
+		["attribute", "MyAnnotation"],
+		["args", [
+			["punctuation", "("],
+			["string-literal", [
+				["string", "\"foo\""]
+			]],
+			["punctuation", ","],
+			["number", "123"],
+			["punctuation", ","],
+			["boolean", "false"],
+			["punctuation", ")"]
 		]],
-		["punctuation", ","],
-		["number", "123"],
-		["punctuation", ","],
-		["boolean", "false"],
-		["punctuation", ")"],
 		["delimiter", "]"]
 	]]
 ]

tests/languages/crystal/char_feature.test

@@ -0,0 +1,41 @@
+'a'
+'z'
+'0'
+'_'
+'あ'
+'\''         # single quote
+'\\'         # backslash
+'\a'         # alert
+'\b'         # backspace
+'\e'         # escape
+'\f'         # form feed
+'\n'         # newline
+'\r'         # carriage return
+'\t'         # tab
+'\v'         # vertical tab
+'\0'         # null character
+'\uFFFF'     # hexadecimal unicode character
+'\u{10FFFF}' # hexadecimal unicode character
+
+----------------------------------------------------
+
+[
+	["char", "'a'"],
+	["char", "'z'"],
+	["char", "'0'"],
+	["char", "'_'"],
+	["char", "'あ'"],
+	["char", "'\\''"], ["comment", "# single quote"],
+	["char", "'\\\\'"], ["comment", "# backslash"],
+	["char", "'\\a'"], ["comment", "# alert"],
+	["char", "'\\b'"], ["comment", "# backspace"],
+	["char", "'\\e'"], ["comment", "# escape"],
+	["char", "'\\f'"], ["comment", "# form feed"],
+	["char", "'\\n'"], ["comment", "# newline"],
+	["char", "'\\r'"], ["comment", "# carriage return"],
+	["char", "'\\t'"], ["comment", "# tab"],
+	["char", "'\\v'"], ["comment", "# vertical tab"],
+	["char", "'\\0'"], ["comment", "# null character"],
+	["char", "'\\uFFFF'"], ["comment", "# hexadecimal unicode character"],
+	["char", "'\\u{10FFFF}'"], ["comment", "# hexadecimal unicode character"]
+]

tests/languages/crystal/expansion_feature.test

@@ -8,26 +8,34 @@
 [
 	["expansion", [
 		["delimiter", "{{"],
-		["number", "1_u32"],
+		["content", [
+			["number", "1_u32"]
+		]],
 		["delimiter", "}}"]
 	]],
 	["expansion", [
 		["delimiter", "{%"],
-		["number", "2_u32"],
+		["content", [
+			["number", "2_u32"]
+		]],
 		["delimiter", "%}"]
 	]],
 	["expansion", [
 		["delimiter", "{{"],
-		["punctuation", "{"],
-		["number", "3_u32"],
-		["punctuation", "}"],
+		["content", [
+			["punctuation", "{"],
+			["number", "3_u32"],
+			["punctuation", "}"]
+		]],
 		["delimiter", "}}"]
 	]],
 	["expansion", [
 		["delimiter", "{%"],
-		["operator", "%"],
-		["number", "4_u32"],
-		["operator", "%"],
+		["content", [
+			["operator", "%"],
+			["number", "4_u32"],
+			["operator", "%"]
+		]],
 		["delimiter", "%}"]
 	]]
 ]

tests/languages/crystal/keyword_feature.test

@@ -2,6 +2,7 @@
 .responds_to?
 abstract
 alias
+annotation;
 as
 asm
 begin
@@ -19,6 +20,7 @@ extend
 for
 fun
 if
+ifdef
 include
 instance_sizeof
 lib
@@ -30,6 +32,7 @@ out
 pointerof
 private
 protected
+ptr
 require
 rescue
 return
@@ -41,6 +44,7 @@ super
 then
 type
 typeof
+undef
 uninitialized
 union
 unless
@@ -62,6 +66,7 @@ __LINE__
 	["punctuation", "."], ["keyword", "responds_to?"],
 	["keyword", "abstract"],
 	["keyword", "alias"],
+	["keyword", "annotation"], ["punctuation", ";"],
 	["keyword", "as"],
 	["keyword", "asm"],
 	["keyword", "begin"],
@@ -79,6 +84,7 @@ __LINE__
 	["keyword", "for"],
 	["keyword", "fun"],
 	["keyword", "if"],
+	["keyword", "ifdef"],
 	["keyword", "include"],
 	["keyword", "instance_sizeof"],
 	["keyword", "lib"],
@@ -90,6 +96,7 @@ __LINE__
 	["keyword", "pointerof"],
 	["keyword", "private"],
 	["keyword", "protected"],
+	["keyword", "ptr"],
 	["keyword", "require"],
 	["keyword", "rescue"],
 	["keyword", "return"],
@@ -101,6 +108,7 @@ __LINE__
 	["keyword", "then"],
 	["keyword", "type"],
 	["keyword", "typeof"],
+	["keyword", "undef"],
 	["keyword", "uninitialized"],
 	["keyword", "union"],
 	["keyword", "unless"],