From f580a41b6d3d32f90e801c0154c401e9ea75528a Mon Sep 17 00:00:00 2001 From: PureFox48 <64583745+PureFox48@users.noreply.github.com> Date: Fri, 4 Nov 2022 13:45:15 +0000 Subject: [PATCH 01/16] Create wren.py --- pygments/lexers/wren.py | 97 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 pygments/lexers/wren.py diff --git a/pygments/lexers/wren.py b/pygments/lexers/wren.py new file mode 100644 index 0000000000..fee867e3de --- /dev/null +++ b/pygments/lexers/wren.py @@ -0,0 +1,97 @@ +""" + pygments.lexers.wren + ~~~~~~~~~~~~~~~~~~~~ + Lexer for Wren. + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer +from pygments.token import Whitespace, Punctuation, Keyword, Name, Comment, \ + Operator, Number, String + +__all__ = ['WrenLexer'] + +class WrenLexer(RegexLexer): + name = 'Wren' + aliases = ['wren'] + filenames = ['*.wren'] + + flags = re.MULTILINE | re.DOTALL + + tokens = { + 'root': [ + # Whitespace. + (r'\s+', Whitespace), + (r'[,\\\[\]{}]', Punctuation), + + # Push a parenthesized state so that we know the corresponding ')' + # is for a parenthesized expression and not interpolation. + (r'\(', Punctuation, ('parenthesized', 'root')), + + # In this state, we don't know whether a closing ')' is for a + # parenthesized expression or the end of an interpolation. So, do + # a non-consuming match and let the parent state (either + # 'parenthesized' or 'interpolation') decide. + (r'(?=\))', Whitespace, '#pop'), + + # Keywords. + (r'(as|break|class|construct|continue|else|for|foreign|if|import|' + r'in|is|return|static|super|var|while)\b', Keyword), + + (r'(true|false|null)\b', Keyword.Constant), + + (r'this\b', Name.Builtin), + + # Comments. + (r'/\*', Comment.Multiline, 'comment'), # Multiline, can nest. + (r'//.*?$', Comment.Single), # Single line. + (r'#.*?(\(.*?\))?$', Comment.Special), # Attribute or shebang. + + # Names and operators. + (r'[!%&*+\-./:<=>?\\^|~]+', Operator), + (r'[a-z][a-zA-Z_0-9]*', Name), + (r'[A-Z][a-zA-Z_0-9]*', Name.Class), + (r'__[a-zA-Z_0-9]*', Name.Variable.Class), + (r'_[a-zA-Z_0-9]*', Name.Variable.Instance), + + # Numbers. + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'\d+(\.\d+)?([eE][-+]?\d+)?', Number.Float), + + # Strings. + (r'"""', String, 'raw_string'), + (r'"', String, 'string'), + ], + 'comment': [ + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'.', Comment.Multiline), # All other characters. + ], + 'raw_string': [ + (r'"""', String, '#pop'), + (r'"', String), + (r'"', String, '#pop'), + (r'.', String), # All characters. + ], + 'string': [ + (r'"', String, '#pop'), + (r'\\[\\%"0abefnrtv]', String.Escape), # Escape. + (r'\\x[a-fA-F0-9]{2}', String.Escape), # Byte escape. + (r'\\u[a-fA-F0-9]{4}', String.Escape), # Unicode escape. + (r'\\U[a-fA-F0-9]{8}', String.Escape), # Long Unicode escape. + + (r'%\(', String.Interpol, ('interpolation', 'root')), + (r'.', String), # All other characters. + ], + 'parenthesized': [ + # We only get to this state when we're at a ')'. + (r'\)', Punctuation, '#pop'), + ], + 'interpolation': [ + # We only get to this state when we're at a ')'. + (r'\)', String.Interpol, '#pop'), + ], + } From 0c5affcbfa0933db74c48464a90b9b6ce9d3ae00 Mon Sep 17 00:00:00 2001 From: PureFox48 <64583745+PureFox48@users.noreply.github.com> Date: Fri, 4 Nov 2022 13:49:26 +0000 Subject: [PATCH 02/16] Create example.wren --- tests/examplefiles/wren/example.wren | 121 +++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 tests/examplefiles/wren/example.wren diff --git a/tests/examplefiles/wren/example.wren b/tests/examplefiles/wren/example.wren new file mode 100644 index 0000000000..a0a79be979 --- /dev/null +++ b/tests/examplefiles/wren/example.wren @@ -0,0 +1,121 @@ +#!/bin/wren + +/* IMPORTS */ +import "random" for Random as Rand + +/* COMMENTS */ + +// single line comment + +/* + multiline comment + /* + nested multiline comment + */ +*/ + +/* CLASSES & ATTRIBUTES */ + +#!type = "parent" +class Parent { + #method + static setField (field) { + __field = field + } + + construct new(parent) { + _parent = parent + return + } + + parent { _parent } + + foreign method() +} + +#!type = "child" +#group( + multiple, + lines = true +) +class Child is Parent { + construct new(parent, child) { + super(parent) + _child = child + } + + child { _child } + + toString { this.parent } +} + +/* VARIABLES & STRINGS */ + +var rand = Rand.new() +var name = """David""" +var fullName = "%(name) Smith" +var firstChild = Child.new("Philip Smith", fullName) +var age = 21 +var weight = 70.25 +var male = true +var sex = male ? "M" : "F" +var address = """ + "House name" 12 Any Street + Some Town + \t %("Some Country") " +""" + +/* LOOPS & CONDITIONALS */ +for (i in 1..5) { + if (i == 2) { + continue + } else if (i == 4) { + break + } + System.print(i) +} +var j = 6 +while (j <= 1e+1) { + if (j == 8) break + System.print(j) + j = j + 1 +} + +/* ARITHMETIC OPERATORS */ +var a = 1 +var b = 2 +var c = [-a, a + b, a - b, a * b, a / b, a % b] +var add = c[1] + +/* BITWISE OPERATORS */ +var d = 3 +var e = 4 +var f = [~d, d & e, d | e, d ^ e, d << 2, e >> 1] + +/* COMPARISON OPERATORS */ +var g = 5 +var h = 6 +var i = [a == b, a != b, a < b, a <= b, a > b, a >= b] +var k = firstChild is Parent + +/* FUNCTIONS */ +var func = Fn.new { |param| + var z = "this" + System.print(z + " " + param) +} +func.call("function") + +/* MISCELLANEOUS */ + +var hex = 0x12ac +var nul = null +var l = false +var m = true +var n = l && m +var o = l || m +var esc = "\\ \% \" \0 \a \b \t \f \n \r \v \e \x01 \uabcd \Uabcdef01" +var uni = "£ é 😀 ‎🎷" +var map = {"a": 1, "b": 2} +var iex = "%(map["a"] + map["b"])" +var odd = (1...h).where { |i| i % 2 == 1 } + .toList From 39b4f653b1d2e5dbc2f0acf027518cbf7d4778ef Mon Sep 17 00:00:00 2001 From: PureFox48 <64583745+PureFox48@users.noreply.github.com> Date: Fri, 4 Nov 2022 13:50:30 +0000 Subject: [PATCH 03/16] Create example.wren.output --- tests/examplefiles/wren/example.wren.output | 1285 +++++++++++++++++++ 1 file changed, 1285 insertions(+) create mode 100644 tests/examplefiles/wren/example.wren.output diff --git a/tests/examplefiles/wren/example.wren.output b/tests/examplefiles/wren/example.wren.output new file mode 100644 index 0000000000..d114c61ada --- /dev/null +++ b/tests/examplefiles/wren/example.wren.output @@ -0,0 +1,1285 @@ +'#!/bin/wren' Comment.Special +'\n\n' Text.Whitespace + +'/*' Comment.Multiline +' ' Comment.Multiline +'I' Comment.Multiline +'M' Comment.Multiline +'P' Comment.Multiline +'O' Comment.Multiline +'R' Comment.Multiline +'T' Comment.Multiline +'S' Comment.Multiline +' ' Comment.Multiline +'*/' Comment.Multiline +'\n' Text.Whitespace + +'import' Keyword +' ' Text.Whitespace +'"' Literal.String +'r' Literal.String +'a' Literal.String +'n' Literal.String +'d' Literal.String +'o' Literal.String +'m' Literal.String +'"' Literal.String +' ' Text.Whitespace +'for' Keyword +' ' Text.Whitespace +'Random' Name.Class +' ' Text.Whitespace +'as' Keyword +' ' Text.Whitespace +'Rand' Name.Class +'\n\n' Text.Whitespace + +'/*' Comment.Multiline +' ' Comment.Multiline +'C' Comment.Multiline +'O' Comment.Multiline +'M' Comment.Multiline +'M' Comment.Multiline +'E' Comment.Multiline +'N' Comment.Multiline +'T' Comment.Multiline +'S' Comment.Multiline +' ' Comment.Multiline +'*/' Comment.Multiline +'\n\n' Text.Whitespace + +'// single line comment' Comment.Single +'\n\n' Text.Whitespace + +'/*' Comment.Multiline +'\n' Comment.Multiline + +' ' Comment.Multiline +' ' Comment.Multiline +' ' Comment.Multiline +' ' Comment.Multiline +'m' Comment.Multiline +'u' Comment.Multiline +'l' Comment.Multiline +'t' Comment.Multiline +'i' Comment.Multiline +'l' Comment.Multiline +'i' Comment.Multiline +'n' Comment.Multiline +'e' Comment.Multiline +' ' Comment.Multiline +'c' Comment.Multiline +'o' Comment.Multiline +'m' Comment.Multiline +'m' Comment.Multiline +'e' Comment.Multiline +'n' Comment.Multiline +'t' Comment.Multiline +'\n' Comment.Multiline + +' ' Comment.Multiline +' ' Comment.Multiline +' ' Comment.Multiline +' ' Comment.Multiline +'/*' Comment.Multiline +'\n' Comment.Multiline + +' ' Comment.Multiline +' ' Comment.Multiline +' ' Comment.Multiline +' ' Comment.Multiline +' ' Comment.Multiline +' ' Comment.Multiline +' ' Comment.Multiline +' ' Comment.Multiline +'n' Comment.Multiline +'e' Comment.Multiline +'s' Comment.Multiline +'t' Comment.Multiline +'e' Comment.Multiline +'d' Comment.Multiline +' ' Comment.Multiline +'m' Comment.Multiline +'u' Comment.Multiline +'l' Comment.Multiline +'t' Comment.Multiline +'i' Comment.Multiline +'l' Comment.Multiline +'i' Comment.Multiline +'n' Comment.Multiline +'e' Comment.Multiline +' ' Comment.Multiline +'c' Comment.Multiline +'o' Comment.Multiline +'m' Comment.Multiline +'m' Comment.Multiline +'e' Comment.Multiline +'n' Comment.Multiline +'t' Comment.Multiline +'\n' Comment.Multiline + +' ' Comment.Multiline +' ' Comment.Multiline +' ' Comment.Multiline +' ' Comment.Multiline +'*/' Comment.Multiline +'\n' Comment.Multiline + +'*/' Comment.Multiline +'\n\n' Text.Whitespace + +'/*' Comment.Multiline +' ' Comment.Multiline +'C' Comment.Multiline +'L' Comment.Multiline +'A' Comment.Multiline +'S' Comment.Multiline +'S' Comment.Multiline +'E' Comment.Multiline +'S' Comment.Multiline +' ' Comment.Multiline +'&' Comment.Multiline +' ' Comment.Multiline +'A' Comment.Multiline +'T' Comment.Multiline +'T' Comment.Multiline +'R' Comment.Multiline +'I' Comment.Multiline +'B' Comment.Multiline +'U' Comment.Multiline +'T' Comment.Multiline +'E' Comment.Multiline +'S' Comment.Multiline +' ' Comment.Multiline +'*/' Comment.Multiline +'\n\n' Text.Whitespace + +'#!type = "parent"' Comment.Special +'\n' Text.Whitespace + +'class' Keyword +' ' Text.Whitespace +'Parent' Name.Class +' ' Text.Whitespace +'{' Punctuation +'\n ' Text.Whitespace +'#method' Comment.Special +'\n ' Text.Whitespace +'static' Keyword +' ' Text.Whitespace +'setField' Name +' ' Text.Whitespace +'(' Punctuation +'field' Name +'' Text.Whitespace +')' Punctuation +' ' Text.Whitespace +'{' Punctuation +'\n ' Text.Whitespace +'__field' Name.Variable.Class +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'field' Name +'\n ' Text.Whitespace +'}' Punctuation +'\n\n ' Text.Whitespace +'construct' Keyword +' ' Text.Whitespace +'new' Name +'(' Punctuation +'parent' Name +'' Text.Whitespace +')' Punctuation +' ' Text.Whitespace +'{' Punctuation +'\n ' Text.Whitespace +'_parent' Name.Variable.Instance +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'parent' Name +'\n ' Text.Whitespace +'return' Keyword +'\n ' Text.Whitespace +'}' Punctuation +'\n\n ' Text.Whitespace +'parent' Name +' ' Text.Whitespace +'{' Punctuation +' ' Text.Whitespace +'_parent' Name.Variable.Instance +' ' Text.Whitespace +'}' Punctuation +'\n\n ' Text.Whitespace +'foreign' Keyword +' ' Text.Whitespace +'method' Name +'(' Punctuation +'' Text.Whitespace +')' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +'\n\n' Text.Whitespace + +'#!type = "child"' Comment.Special +'\n' Text.Whitespace + +'#group(\n multiple,\n lines = true\n)' Comment.Special +'\n' Text.Whitespace + +'class' Keyword +' ' Text.Whitespace +'Child' Name.Class +' ' Text.Whitespace +'is' Keyword +' ' Text.Whitespace +'Parent' Name.Class +' ' Text.Whitespace +'{' Punctuation +'\n ' Text.Whitespace +'construct' Keyword +' ' Text.Whitespace +'new' Name +'(' Punctuation +'parent' Name +',' Punctuation +' ' Text.Whitespace +'child' Name +'' Text.Whitespace +')' Punctuation +' ' Text.Whitespace +'{' Punctuation +'\n ' Text.Whitespace +'super' Keyword +'(' Punctuation +'parent' Name +'' Text.Whitespace +')' Punctuation +'\n ' Text.Whitespace +'_child' Name.Variable.Instance +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'child' Name +'\n ' Text.Whitespace +'}' Punctuation +'\n\n ' Text.Whitespace +'child' Name +' ' Text.Whitespace +'{' Punctuation +' ' Text.Whitespace +'_child' Name.Variable.Instance +' ' Text.Whitespace +'}' Punctuation +'\n\n ' Text.Whitespace +'toString' Name +' ' Text.Whitespace +'{' Punctuation +' ' Text.Whitespace +'this' Name.Builtin +'.' Operator +'parent' Name +' ' Text.Whitespace +'}' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +'\n\n' Text.Whitespace + +'/*' Comment.Multiline +' ' Comment.Multiline +'V' Comment.Multiline +'A' Comment.Multiline +'R' Comment.Multiline +'I' Comment.Multiline +'A' Comment.Multiline +'B' Comment.Multiline +'L' Comment.Multiline +'E' Comment.Multiline +'S' Comment.Multiline +' ' Comment.Multiline +'&' Comment.Multiline +' ' Comment.Multiline +'S' Comment.Multiline +'T' Comment.Multiline +'R' Comment.Multiline +'I' Comment.Multiline +'N' Comment.Multiline +'G' Comment.Multiline +'S' Comment.Multiline +' ' Comment.Multiline +'*/' Comment.Multiline +'\n\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'rand' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'Rand' Name.Class +'.' Operator +'new' Name +'(' Punctuation +'' Text.Whitespace +')' Punctuation +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'name' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"""' Literal.String +'D' Literal.String +'a' Literal.String +'v' Literal.String +'i' Literal.String +'d' Literal.String +'"""' Literal.String +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'fullName' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String +'%(' Literal.String.Interpol +'name' Name +'' Text.Whitespace +')' Literal.String.Interpol +' ' Literal.String +'S' Literal.String +'m' Literal.String +'i' Literal.String +'t' Literal.String +'h' Literal.String +'"' Literal.String +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'firstChild' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'Child' Name.Class +'.' Operator +'new' Name +'(' Punctuation +'"' Literal.String +'P' Literal.String +'h' Literal.String +'i' Literal.String +'l' Literal.String +'i' Literal.String +'p' Literal.String +' ' Literal.String +'S' Literal.String +'m' Literal.String +'i' Literal.String +'t' Literal.String +'h' Literal.String +'"' Literal.String +',' Punctuation +' ' Text.Whitespace +'fullName' Name +'' Text.Whitespace +')' Punctuation +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'age' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'21' Literal.Number.Float +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'weight' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'70.25' Literal.Number.Float +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'male' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'true' Keyword.Constant +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'sex' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'male' Name +' ' Text.Whitespace +'?' Operator +' ' Text.Whitespace +'"' Literal.String +'M' Literal.String +'"' Literal.String +' ' Text.Whitespace +':' Operator +' ' Text.Whitespace +'"' Literal.String +'F' Literal.String +'"' Literal.String +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'address' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"""' Literal.String +'\n' Literal.String + +' ' Literal.String +' ' Literal.String +' ' Literal.String +' ' Literal.String +'"' Literal.String +'H' Literal.String +'o' Literal.String +'u' Literal.String +'s' Literal.String +'e' Literal.String +' ' Literal.String +'n' Literal.String +'a' Literal.String +'m' Literal.String +'e' Literal.String +'"' Literal.String +' ' Literal.String +'1' Literal.String +'2' Literal.String +' ' Literal.String +'A' Literal.String +'n' Literal.String +'y' Literal.String +' ' Literal.String +'S' Literal.String +'t' Literal.String +'r' Literal.String +'e' Literal.String +'e' Literal.String +'t' Literal.String +'\n' Literal.String + +' ' Literal.String +' ' Literal.String +' ' Literal.String +' ' Literal.String +'S' Literal.String +'o' Literal.String +'m' Literal.String +'e' Literal.String +' ' Literal.String +'T' Literal.String +'o' Literal.String +'w' Literal.String +'n' Literal.String +'\n' Literal.String + +' ' Literal.String +' ' Literal.String +' ' Literal.String +' ' Literal.String +'\\' Literal.String +'t' Literal.String +' ' Literal.String +'%' Literal.String +'(' Literal.String +'"' Literal.String +'S' Literal.String +'o' Literal.String +'m' Literal.String +'e' Literal.String +' ' Literal.String +'C' Literal.String +'o' Literal.String +'u' Literal.String +'n' Literal.String +'t' Literal.String +'r' Literal.String +'y' Literal.String +'"' Literal.String +')' Literal.String +' ' Literal.String +'"' Literal.String +'\n' Literal.String + +'"""' Literal.String +'\n\n' Text.Whitespace + +'/*' Comment.Multiline +' ' Comment.Multiline +'L' Comment.Multiline +'O' Comment.Multiline +'O' Comment.Multiline +'P' Comment.Multiline +'S' Comment.Multiline +' ' Comment.Multiline +'&' Comment.Multiline +' ' Comment.Multiline +'C' Comment.Multiline +'O' Comment.Multiline +'N' Comment.Multiline +'D' Comment.Multiline +'I' Comment.Multiline +'T' Comment.Multiline +'I' Comment.Multiline +'O' Comment.Multiline +'N' Comment.Multiline +'A' Comment.Multiline +'L' Comment.Multiline +'S' Comment.Multiline +' ' Comment.Multiline +'*/' Comment.Multiline +'\n' Text.Whitespace + +'for' Keyword +' ' Text.Whitespace +'(' Punctuation +'i' Name +' ' Text.Whitespace +'in' Keyword +' ' Text.Whitespace +'1' Literal.Number.Float +'..' Operator +'5' Literal.Number.Float +'' Text.Whitespace +')' Punctuation +' ' Text.Whitespace +'{' Punctuation +'\n ' Text.Whitespace +'if' Keyword +' ' Text.Whitespace +'(' Punctuation +'i' Name +' ' Text.Whitespace +'==' Operator +' ' Text.Whitespace +'2' Literal.Number.Float +'' Text.Whitespace +')' Punctuation +' ' Text.Whitespace +'{' Punctuation +'\n ' Text.Whitespace +'continue' Keyword +'\n ' Text.Whitespace +'}' Punctuation +' ' Text.Whitespace +'else' Keyword +' ' Text.Whitespace +'if' Keyword +' ' Text.Whitespace +'(' Punctuation +'i' Name +' ' Text.Whitespace +'==' Operator +' ' Text.Whitespace +'4' Literal.Number.Float +'' Text.Whitespace +')' Punctuation +' ' Text.Whitespace +'{' Punctuation +'\n ' Text.Whitespace +'break' Keyword +'\n ' Text.Whitespace +'}' Punctuation +'\n ' Text.Whitespace +'System' Name.Class +'.' Operator +'print' Name +'(' Punctuation +'i' Name +'' Text.Whitespace +')' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'j' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'6' Literal.Number.Float +'\n' Text.Whitespace + +'while' Keyword +' ' Text.Whitespace +'(' Punctuation +'j' Name +' ' Text.Whitespace +'<=' Operator +' ' Text.Whitespace +'1e+1' Literal.Number.Float +'' Text.Whitespace +')' Punctuation +' ' Text.Whitespace +'{' Punctuation +'\n ' Text.Whitespace +'if' Keyword +' ' Text.Whitespace +'(' Punctuation +'j' Name +' ' Text.Whitespace +'==' Operator +' ' Text.Whitespace +'8' Literal.Number.Float +'' Text.Whitespace +')' Punctuation +' ' Text.Whitespace +'break' Keyword +'\n ' Text.Whitespace +'System' Name.Class +'.' Operator +'print' Name +'(' Punctuation +'j' Name +'' Text.Whitespace +')' Punctuation +'\n ' Text.Whitespace +'j' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'j' Name +' ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'1' Literal.Number.Float +'\n' Text.Whitespace + +'}' Punctuation +'\n\n' Text.Whitespace + +'/*' Comment.Multiline +' ' Comment.Multiline +'A' Comment.Multiline +'R' Comment.Multiline +'I' Comment.Multiline +'T' Comment.Multiline +'H' Comment.Multiline +'M' Comment.Multiline +'E' Comment.Multiline +'T' Comment.Multiline +'I' Comment.Multiline +'C' Comment.Multiline +' ' Comment.Multiline +'O' Comment.Multiline +'P' Comment.Multiline +'E' Comment.Multiline +'R' Comment.Multiline +'A' Comment.Multiline +'T' Comment.Multiline +'O' Comment.Multiline +'R' Comment.Multiline +'S' Comment.Multiline +' ' Comment.Multiline +'*/' Comment.Multiline +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'a' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'1' Literal.Number.Float +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'b' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'2' Literal.Number.Float +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'c' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'[' Punctuation +'-' Operator +'a' Name +',' Punctuation +' ' Text.Whitespace +'a' Name +' ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'b' Name +',' Punctuation +' ' Text.Whitespace +'a' Name +' ' Text.Whitespace +'-' Operator +' ' Text.Whitespace +'b' Name +',' Punctuation +' ' Text.Whitespace +'a' Name +' ' Text.Whitespace +'*' Operator +' ' Text.Whitespace +'b' Name +',' Punctuation +' ' Text.Whitespace +'a' Name +' ' Text.Whitespace +'/' Operator +' ' Text.Whitespace +'b' Name +',' Punctuation +' ' Text.Whitespace +'a' Name +' ' Text.Whitespace +'%' Operator +' ' Text.Whitespace +'b' Name +']' Punctuation +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'add' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'c' Name +'[' Punctuation +'1' Literal.Number.Float +']' Punctuation +'\n\n' Text.Whitespace + +'/*' Comment.Multiline +' ' Comment.Multiline +'B' Comment.Multiline +'I' Comment.Multiline +'T' Comment.Multiline +'W' Comment.Multiline +'I' Comment.Multiline +'S' Comment.Multiline +'E' Comment.Multiline +' ' Comment.Multiline +'O' Comment.Multiline +'P' Comment.Multiline +'E' Comment.Multiline +'R' Comment.Multiline +'A' Comment.Multiline +'T' Comment.Multiline +'O' Comment.Multiline +'R' Comment.Multiline +'S' Comment.Multiline +' ' Comment.Multiline +'*/' Comment.Multiline +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'d' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'3' Literal.Number.Float +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'e' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'4' Literal.Number.Float +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'f' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'[' Punctuation +'~' Operator +'d' Name +',' Punctuation +' ' Text.Whitespace +'d' Name +' ' Text.Whitespace +'&' Operator +' ' Text.Whitespace +'e' Name +',' Punctuation +' ' Text.Whitespace +'d' Name +' ' Text.Whitespace +'|' Operator +' ' Text.Whitespace +'e' Name +',' Punctuation +' ' Text.Whitespace +'d' Name +' ' Text.Whitespace +'^' Operator +' ' Text.Whitespace +'e' Name +',' Punctuation +' ' Text.Whitespace +'d' Name +' ' Text.Whitespace +'<<' Operator +' ' Text.Whitespace +'2' Literal.Number.Float +',' Punctuation +' ' Text.Whitespace +'e' Name +' ' Text.Whitespace +'>>' Operator +' ' Text.Whitespace +'1' Literal.Number.Float +']' Punctuation +'\n\n' Text.Whitespace + +'/*' Comment.Multiline +' ' Comment.Multiline +'C' Comment.Multiline +'O' Comment.Multiline +'M' Comment.Multiline +'P' Comment.Multiline +'A' Comment.Multiline +'R' Comment.Multiline +'I' Comment.Multiline +'S' Comment.Multiline +'O' Comment.Multiline +'N' Comment.Multiline +' ' Comment.Multiline +'O' Comment.Multiline +'P' Comment.Multiline +'E' Comment.Multiline +'R' Comment.Multiline +'A' Comment.Multiline +'T' Comment.Multiline +'O' Comment.Multiline +'R' Comment.Multiline +'S' Comment.Multiline +' ' Comment.Multiline +'*/' Comment.Multiline +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'g' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'5' Literal.Number.Float +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'h' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'6' Literal.Number.Float +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'i' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'[' Punctuation +'a' Name +' ' Text.Whitespace +'==' Operator +' ' Text.Whitespace +'b' Name +',' Punctuation +' ' Text.Whitespace +'a' Name +' ' Text.Whitespace +'!=' Operator +' ' Text.Whitespace +'b' Name +',' Punctuation +' ' Text.Whitespace +'a' Name +' ' Text.Whitespace +'<' Operator +' ' Text.Whitespace +'b' Name +',' Punctuation +' ' Text.Whitespace +'a' Name +' ' Text.Whitespace +'<=' Operator +' ' Text.Whitespace +'b' Name +',' Punctuation +' ' Text.Whitespace +'a' Name +' ' Text.Whitespace +'>' Operator +' ' Text.Whitespace +'b' Name +',' Punctuation +' ' Text.Whitespace +'a' Name +' ' Text.Whitespace +'>=' Operator +' ' Text.Whitespace +'b' Name +']' Punctuation +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'k' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'firstChild' Name +' ' Text.Whitespace +'is' Keyword +' ' Text.Whitespace +'Parent' Name.Class +'\n\n' Text.Whitespace + +'/*' Comment.Multiline +' ' Comment.Multiline +'F' Comment.Multiline +'U' Comment.Multiline +'N' Comment.Multiline +'C' Comment.Multiline +'T' Comment.Multiline +'I' Comment.Multiline +'O' Comment.Multiline +'N' Comment.Multiline +'S' Comment.Multiline +' ' Comment.Multiline +'*/' Comment.Multiline +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'func' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'Fn' Name.Class +'.' Operator +'new' Name +' ' Text.Whitespace +'{' Punctuation +' ' Text.Whitespace +'|' Operator +'param' Name +'|' Operator +'\n ' Text.Whitespace +'var' Keyword +' ' Text.Whitespace +'z' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String +'t' Literal.String +'h' Literal.String +'i' Literal.String +'s' Literal.String +'"' Literal.String +'\n ' Text.Whitespace +'System' Name.Class +'.' Operator +'print' Name +'(' Punctuation +'z' Name +' ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'"' Literal.String +' ' Literal.String +'"' Literal.String +' ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'param' Name +'' Text.Whitespace +')' Punctuation +'\n' Text.Whitespace + +'}' Punctuation +'\n' Text.Whitespace + +'func' Name +'.' Operator +'call' Name +'(' Punctuation +'"' Literal.String +'f' Literal.String +'u' Literal.String +'n' Literal.String +'c' Literal.String +'t' Literal.String +'i' Literal.String +'o' Literal.String +'n' Literal.String +'"' Literal.String +'' Text.Whitespace +')' Punctuation +'\n\n' Text.Whitespace + +'/*' Comment.Multiline +' ' Comment.Multiline +'M' Comment.Multiline +'I' Comment.Multiline +'S' Comment.Multiline +'C' Comment.Multiline +'E' Comment.Multiline +'L' Comment.Multiline +'L' Comment.Multiline +'A' Comment.Multiline +'N' Comment.Multiline +'E' Comment.Multiline +'O' Comment.Multiline +'U' Comment.Multiline +'S' Comment.Multiline +' ' Comment.Multiline +'*/' Comment.Multiline +'\n\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'hex' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'0x12ac' Literal.Number.Hex +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'nul' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'null' Keyword.Constant +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'l' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'false' Keyword.Constant +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'m' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'true' Keyword.Constant +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'n' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'l' Name +' ' Text.Whitespace +'&&' Operator +' ' Text.Whitespace +'m' Name +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'o' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'l' Name +' ' Text.Whitespace +'||' Operator +' ' Text.Whitespace +'m' Name +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'esc' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String +'\\\\' Literal.String.Escape +' ' Literal.String +'\\%' Literal.String.Escape +' ' Literal.String +'\\"' Literal.String.Escape +' ' Literal.String +'\\0' Literal.String.Escape +' ' Literal.String +'\\a' Literal.String.Escape +' ' Literal.String +'\\b' Literal.String.Escape +' ' Literal.String +'\\t' Literal.String.Escape +' ' Literal.String +'\\f' Literal.String.Escape +' ' Literal.String +'\\n' Literal.String.Escape +' ' Literal.String +'\\r' Literal.String.Escape +' ' Literal.String +'\\v' Literal.String.Escape +' ' Literal.String +'\\e' Literal.String.Escape +' ' Literal.String +'\\x01' Literal.String.Escape +' ' Literal.String +'\\uabcd' Literal.String.Escape +' ' Literal.String +'\\Uabcdef01' Literal.String.Escape +'"' Literal.String +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'uni' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String +'£' Literal.String +' ' Literal.String +'é' Literal.String +' ' Literal.String +'😀' Literal.String +' ' Literal.String +' ' Literal.String +'\u200e' Literal.String +'🎷' Literal.String +'"' Literal.String +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'map' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'{' Punctuation +'"' Literal.String +'a' Literal.String +'"' Literal.String +':' Operator +' ' Text.Whitespace +'1' Literal.Number.Float +',' Punctuation +' ' Text.Whitespace +'"' Literal.String +'b' Literal.String +'"' Literal.String +':' Operator +' ' Text.Whitespace +'2' Literal.Number.Float +'}' Punctuation +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'iex' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String +'%(' Literal.String.Interpol +'map' Name +'[' Punctuation +'"' Literal.String +'a' Literal.String +'"' Literal.String +']' Punctuation +' ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'map' Name +'[' Punctuation +'"' Literal.String +'b' Literal.String +'"' Literal.String +']' Punctuation +'' Text.Whitespace +')' Literal.String.Interpol +'"' Literal.String +'\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'odd' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'(' Punctuation +'1' Literal.Number.Float +'...' Operator +'h' Name +'' Text.Whitespace +')' Punctuation +'.' Operator +'where' Name +' ' Text.Whitespace +'{' Punctuation +' ' Text.Whitespace +'|' Operator +'i' Name +'|' Operator +' ' Text.Whitespace +'i' Name +' ' Text.Whitespace +'%' Operator +' ' Text.Whitespace +'2' Literal.Number.Float +' ' Text.Whitespace +'==' Operator +' ' Text.Whitespace +'1' Literal.Number.Float +' ' Text.Whitespace +'}' Punctuation +'\n ' Text.Whitespace +'.' Operator +'toList' Name +'\n' Text.Whitespace From 48ae5a49a94d9897396767b3843288828eb3815c Mon Sep 17 00:00:00 2001 From: PureFox48 <64583745+PureFox48@users.noreply.github.com> Date: Sat, 5 Nov 2022 10:17:22 +0000 Subject: [PATCH 04/16] Update _mapping.py --- pygments/lexers/_mapping.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index cfe139ffae..6ac65f9458 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -528,6 +528,7 @@ 'WebIDLLexer': ('pygments.lexers.webidl', 'Web IDL', ('webidl',), ('*.webidl',), ()), 'WhileyLexer': ('pygments.lexers.whiley', 'Whiley', ('whiley',), ('*.whiley',), ('text/x-whiley',)), 'WoWTocLexer': ('pygments.lexers.wowtoc', 'World of Warcraft TOC', ('wowtoc',), ('*.toc',), ()), + 'WrenLexer': ('pygments.lexers.wren', 'Wren', ('wren',), ('*.wren',), ()), 'X10Lexer': ('pygments.lexers.x10', 'X10', ('x10', 'xten'), ('*.x10',), ('text/x-x10',)), 'XMLUL4Lexer': ('pygments.lexers.ul4', 'XML+UL4', ('xml+ul4',), ('*.xmlul4',), ()), 'XQueryLexer': ('pygments.lexers.webmisc', 'XQuery', ('xquery', 'xqy', 'xq', 'xql', 'xqm'), ('*.xqy', '*.xquery', '*.xq', '*.xql', '*.xqm'), ('text/xquery', 'application/xquery')), From ad55df7d0ee60a01f25f285b7925486349cc1e89 Mon Sep 17 00:00:00 2001 From: PureFox48 <64583745+PureFox48@users.noreply.github.com> Date: Sat, 5 Nov 2022 21:10:28 +0000 Subject: [PATCH 05/16] Update wren.py See individual comments. --- pygments/lexers/wren.py | 43 +++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/pygments/lexers/wren.py b/pygments/lexers/wren.py index fee867e3de..469e67f810 100644 --- a/pygments/lexers/wren.py +++ b/pygments/lexers/wren.py @@ -8,7 +8,7 @@ import re -from pygments.lexer import RegexLexer +from pygments.lexer import RegexLexer, words, include from pygments.token import Whitespace, Punctuation, Keyword, Name, Comment, \ Operator, Number, String @@ -16,6 +16,7 @@ class WrenLexer(RegexLexer): name = 'Wren' + url = 'https://wren.io' aliases = ['wren'] filenames = ['*.wren'] @@ -35,15 +36,25 @@ class WrenLexer(RegexLexer): # parenthesized expression or the end of an interpolation. So, do # a non-consuming match and let the parent state (either # 'parenthesized' or 'interpolation') decide. - (r'(?=\))', Whitespace, '#pop'), + (r'(?=\))', Punctuation, '#pop'), # Keywords. - (r'(as|break|class|construct|continue|else|for|foreign|if|import|' - r'in|is|return|static|super|var|while)\b', Keyword), - - (r'(true|false|null)\b', Keyword.Constant), - - (r'this\b', Name.Builtin), + (words(( + 'as', 'break', 'class', 'construct', 'continue', 'else', + 'for', 'foreign', 'if', 'import', 'return', 'static', 'super', + 'var', 'while'), prefix = r'(? Date: Sat, 5 Nov 2022 21:16:06 +0000 Subject: [PATCH 06/16] Update example.wren --- tests/examplefiles/wren/example.wren | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/examplefiles/wren/example.wren b/tests/examplefiles/wren/example.wren index a0a79be979..550fcc54c8 100644 --- a/tests/examplefiles/wren/example.wren +++ b/tests/examplefiles/wren/example.wren @@ -119,3 +119,4 @@ var map = {"a": 1, "b": 2} var iex = "%(map["a"] + map["b"])" var odd = (1...h).where { |i| i % 2 == 1 } .toList +var emp = "" From 234a4fb6e9216a8cabd7517a29b406c262cf0f72 Mon Sep 17 00:00:00 2001 From: PureFox48 <64583745+PureFox48@users.noreply.github.com> Date: Sat, 5 Nov 2022 21:18:59 +0000 Subject: [PATCH 07/16] Update example.wren.output --- tests/examplefiles/wren/example.wren.output | 471 +++----------------- 1 file changed, 64 insertions(+), 407 deletions(-) diff --git a/tests/examplefiles/wren/example.wren.output b/tests/examplefiles/wren/example.wren.output index d114c61ada..37e8afb437 100644 --- a/tests/examplefiles/wren/example.wren.output +++ b/tests/examplefiles/wren/example.wren.output @@ -2,27 +2,14 @@ '\n\n' Text.Whitespace '/*' Comment.Multiline -' ' Comment.Multiline -'I' Comment.Multiline -'M' Comment.Multiline -'P' Comment.Multiline -'O' Comment.Multiline -'R' Comment.Multiline -'T' Comment.Multiline -'S' Comment.Multiline -' ' Comment.Multiline +' IMPORTS ' Comment.Multiline '*/' Comment.Multiline '\n' Text.Whitespace 'import' Keyword ' ' Text.Whitespace '"' Literal.String -'r' Literal.String -'a' Literal.String -'n' Literal.String -'d' Literal.String -'o' Literal.String -'m' Literal.String +'random' Literal.String '"' Literal.String ' ' Text.Whitespace 'for' Keyword @@ -35,16 +22,7 @@ '\n\n' Text.Whitespace '/*' Comment.Multiline -' ' Comment.Multiline -'C' Comment.Multiline -'O' Comment.Multiline -'M' Comment.Multiline -'M' Comment.Multiline -'E' Comment.Multiline -'N' Comment.Multiline -'T' Comment.Multiline -'S' Comment.Multiline -' ' Comment.Multiline +' COMMENTS ' Comment.Multiline '*/' Comment.Multiline '\n\n' Text.Whitespace @@ -52,105 +30,17 @@ '\n\n' Text.Whitespace '/*' Comment.Multiline -'\n' Comment.Multiline - -' ' Comment.Multiline -' ' Comment.Multiline -' ' Comment.Multiline -' ' Comment.Multiline -'m' Comment.Multiline -'u' Comment.Multiline -'l' Comment.Multiline -'t' Comment.Multiline -'i' Comment.Multiline -'l' Comment.Multiline -'i' Comment.Multiline -'n' Comment.Multiline -'e' Comment.Multiline -' ' Comment.Multiline -'c' Comment.Multiline -'o' Comment.Multiline -'m' Comment.Multiline -'m' Comment.Multiline -'e' Comment.Multiline -'n' Comment.Multiline -'t' Comment.Multiline -'\n' Comment.Multiline - -' ' Comment.Multiline -' ' Comment.Multiline -' ' Comment.Multiline -' ' Comment.Multiline -'/*' Comment.Multiline -'\n' Comment.Multiline - -' ' Comment.Multiline -' ' Comment.Multiline -' ' Comment.Multiline -' ' Comment.Multiline -' ' Comment.Multiline -' ' Comment.Multiline -' ' Comment.Multiline -' ' Comment.Multiline -'n' Comment.Multiline -'e' Comment.Multiline -'s' Comment.Multiline -'t' Comment.Multiline -'e' Comment.Multiline -'d' Comment.Multiline -' ' Comment.Multiline -'m' Comment.Multiline -'u' Comment.Multiline -'l' Comment.Multiline -'t' Comment.Multiline -'i' Comment.Multiline -'l' Comment.Multiline -'i' Comment.Multiline -'n' Comment.Multiline -'e' Comment.Multiline -' ' Comment.Multiline -'c' Comment.Multiline -'o' Comment.Multiline -'m' Comment.Multiline -'m' Comment.Multiline -'e' Comment.Multiline -'n' Comment.Multiline -'t' Comment.Multiline -'\n' Comment.Multiline - -' ' Comment.Multiline -' ' Comment.Multiline -' ' Comment.Multiline -' ' Comment.Multiline +'\n multiline comment\n /' Comment.Multiline +'*' Comment.Multiline +'\n nested multiline comment\n ' Comment.Multiline '*/' Comment.Multiline -'\n' Comment.Multiline +'\n' Text.Whitespace -'*/' Comment.Multiline +'*/' Operator '\n\n' Text.Whitespace '/*' Comment.Multiline -' ' Comment.Multiline -'C' Comment.Multiline -'L' Comment.Multiline -'A' Comment.Multiline -'S' Comment.Multiline -'S' Comment.Multiline -'E' Comment.Multiline -'S' Comment.Multiline -' ' Comment.Multiline -'&' Comment.Multiline -' ' Comment.Multiline -'A' Comment.Multiline -'T' Comment.Multiline -'T' Comment.Multiline -'R' Comment.Multiline -'I' Comment.Multiline -'B' Comment.Multiline -'U' Comment.Multiline -'T' Comment.Multiline -'E' Comment.Multiline -'S' Comment.Multiline -' ' Comment.Multiline +' CLASSES & ATTRIBUTES ' Comment.Multiline '*/' Comment.Multiline '\n\n' Text.Whitespace @@ -171,7 +61,7 @@ ' ' Text.Whitespace '(' Punctuation 'field' Name -'' Text.Whitespace +'' Punctuation ')' Punctuation ' ' Text.Whitespace '{' Punctuation @@ -189,7 +79,7 @@ 'new' Name '(' Punctuation 'parent' Name -'' Text.Whitespace +'' Punctuation ')' Punctuation ' ' Text.Whitespace '{' Punctuation @@ -216,7 +106,7 @@ ' ' Text.Whitespace 'method' Name '(' Punctuation -'' Text.Whitespace +'' Punctuation ')' Punctuation '\n' Text.Whitespace @@ -233,7 +123,7 @@ ' ' Text.Whitespace 'Child' Name.Class ' ' Text.Whitespace -'is' Keyword +'is' Operator.Word ' ' Text.Whitespace 'Parent' Name.Class ' ' Text.Whitespace @@ -247,7 +137,7 @@ ',' Punctuation ' ' Text.Whitespace 'child' Name -'' Text.Whitespace +'' Punctuation ')' Punctuation ' ' Text.Whitespace '{' Punctuation @@ -255,7 +145,7 @@ 'super' Keyword '(' Punctuation 'parent' Name -'' Text.Whitespace +'' Punctuation ')' Punctuation '\n ' Text.Whitespace '_child' Name.Variable.Instance @@ -278,7 +168,7 @@ ' ' Text.Whitespace '{' Punctuation ' ' Text.Whitespace -'this' Name.Builtin +'this' Name '.' Operator 'parent' Name ' ' Text.Whitespace @@ -289,27 +179,7 @@ '\n\n' Text.Whitespace '/*' Comment.Multiline -' ' Comment.Multiline -'V' Comment.Multiline -'A' Comment.Multiline -'R' Comment.Multiline -'I' Comment.Multiline -'A' Comment.Multiline -'B' Comment.Multiline -'L' Comment.Multiline -'E' Comment.Multiline -'S' Comment.Multiline -' ' Comment.Multiline -'&' Comment.Multiline -' ' Comment.Multiline -'S' Comment.Multiline -'T' Comment.Multiline -'R' Comment.Multiline -'I' Comment.Multiline -'N' Comment.Multiline -'G' Comment.Multiline -'S' Comment.Multiline -' ' Comment.Multiline +' VARIABLES & STRINGS ' Comment.Multiline '*/' Comment.Multiline '\n\n' Text.Whitespace @@ -323,7 +193,7 @@ '.' Operator 'new' Name '(' Punctuation -'' Text.Whitespace +'' Punctuation ')' Punctuation '\n' Text.Whitespace @@ -333,13 +203,7 @@ ' ' Text.Whitespace '=' Operator ' ' Text.Whitespace -'"""' Literal.String -'D' Literal.String -'a' Literal.String -'v' Literal.String -'i' Literal.String -'d' Literal.String -'"""' Literal.String +'"""David"""' Literal.String '\n' Text.Whitespace 'var' Keyword @@ -351,14 +215,9 @@ '"' Literal.String '%(' Literal.String.Interpol 'name' Name -'' Text.Whitespace +'' Punctuation ')' Literal.String.Interpol -' ' Literal.String -'S' Literal.String -'m' Literal.String -'i' Literal.String -'t' Literal.String -'h' Literal.String +' Smith' Literal.String '"' Literal.String '\n' Text.Whitespace @@ -373,23 +232,12 @@ 'new' Name '(' Punctuation '"' Literal.String -'P' Literal.String -'h' Literal.String -'i' Literal.String -'l' Literal.String -'i' Literal.String -'p' Literal.String -' ' Literal.String -'S' Literal.String -'m' Literal.String -'i' Literal.String -'t' Literal.String -'h' Literal.String +'Philip Smith' Literal.String '"' Literal.String ',' Punctuation ' ' Text.Whitespace 'fullName' Name -'' Text.Whitespace +'' Punctuation ')' Punctuation '\n' Text.Whitespace @@ -447,124 +295,25 @@ ' ' Text.Whitespace '=' Operator ' ' Text.Whitespace -'"""' Literal.String -'\n' Literal.String - -' ' Literal.String -' ' Literal.String -' ' Literal.String -' ' Literal.String -'"' Literal.String -'H' Literal.String -'o' Literal.String -'u' Literal.String -'s' Literal.String -'e' Literal.String -' ' Literal.String -'n' Literal.String -'a' Literal.String -'m' Literal.String -'e' Literal.String -'"' Literal.String -' ' Literal.String -'1' Literal.String -'2' Literal.String -' ' Literal.String -'A' Literal.String -'n' Literal.String -'y' Literal.String -' ' Literal.String -'S' Literal.String -'t' Literal.String -'r' Literal.String -'e' Literal.String -'e' Literal.String -'t' Literal.String -'\n' Literal.String - -' ' Literal.String -' ' Literal.String -' ' Literal.String -' ' Literal.String -'S' Literal.String -'o' Literal.String -'m' Literal.String -'e' Literal.String -' ' Literal.String -'T' Literal.String -'o' Literal.String -'w' Literal.String -'n' Literal.String -'\n' Literal.String - -' ' Literal.String -' ' Literal.String -' ' Literal.String -' ' Literal.String -'\\' Literal.String -'t' Literal.String -' ' Literal.String -'%' Literal.String -'(' Literal.String -'"' Literal.String -'S' Literal.String -'o' Literal.String -'m' Literal.String -'e' Literal.String -' ' Literal.String -'C' Literal.String -'o' Literal.String -'u' Literal.String -'n' Literal.String -'t' Literal.String -'r' Literal.String -'y' Literal.String -'"' Literal.String -')' Literal.String -' ' Literal.String -'"' Literal.String -'\n' Literal.String - -'"""' Literal.String +'"""\n "House name" 12 Any Street\n Some Town\n \\t %("Some Country") "\n"""' Literal.String '\n\n' Text.Whitespace '/*' Comment.Multiline -' ' Comment.Multiline -'L' Comment.Multiline -'O' Comment.Multiline -'O' Comment.Multiline -'P' Comment.Multiline -'S' Comment.Multiline -' ' Comment.Multiline -'&' Comment.Multiline -' ' Comment.Multiline -'C' Comment.Multiline -'O' Comment.Multiline -'N' Comment.Multiline -'D' Comment.Multiline -'I' Comment.Multiline -'T' Comment.Multiline -'I' Comment.Multiline -'O' Comment.Multiline -'N' Comment.Multiline -'A' Comment.Multiline -'L' Comment.Multiline -'S' Comment.Multiline -' ' Comment.Multiline +' LOOPS & CONDITIONALS ' Comment.Multiline '*/' Comment.Multiline '\n' Text.Whitespace 'for' Keyword ' ' Text.Whitespace '(' Punctuation -'i' Name +'i' Name.Builtin ' ' Text.Whitespace -'in' Keyword +'in' Operator.Word ' ' Text.Whitespace '1' Literal.Number.Float '..' Operator '5' Literal.Number.Float -'' Text.Whitespace +'' Punctuation ')' Punctuation ' ' Text.Whitespace '{' Punctuation @@ -572,12 +321,12 @@ 'if' Keyword ' ' Text.Whitespace '(' Punctuation -'i' Name +'i' Name.Builtin ' ' Text.Whitespace '==' Operator ' ' Text.Whitespace '2' Literal.Number.Float -'' Text.Whitespace +'' Punctuation ')' Punctuation ' ' Text.Whitespace '{' Punctuation @@ -591,12 +340,12 @@ 'if' Keyword ' ' Text.Whitespace '(' Punctuation -'i' Name +'i' Name.Builtin ' ' Text.Whitespace '==' Operator ' ' Text.Whitespace '4' Literal.Number.Float -'' Text.Whitespace +'' Punctuation ')' Punctuation ' ' Text.Whitespace '{' Punctuation @@ -609,8 +358,8 @@ '.' Operator 'print' Name '(' Punctuation -'i' Name -'' Text.Whitespace +'i' Name.Builtin +'' Punctuation ')' Punctuation '\n' Text.Whitespace @@ -634,7 +383,7 @@ '<=' Operator ' ' Text.Whitespace '1e+1' Literal.Number.Float -'' Text.Whitespace +'' Punctuation ')' Punctuation ' ' Text.Whitespace '{' Punctuation @@ -647,7 +396,7 @@ '==' Operator ' ' Text.Whitespace '8' Literal.Number.Float -'' Text.Whitespace +'' Punctuation ')' Punctuation ' ' Text.Whitespace 'break' Keyword @@ -657,7 +406,7 @@ 'print' Name '(' Punctuation 'j' Name -'' Text.Whitespace +'' Punctuation ')' Punctuation '\n ' Text.Whitespace 'j' Name @@ -675,28 +424,7 @@ '\n\n' Text.Whitespace '/*' Comment.Multiline -' ' Comment.Multiline -'A' Comment.Multiline -'R' Comment.Multiline -'I' Comment.Multiline -'T' Comment.Multiline -'H' Comment.Multiline -'M' Comment.Multiline -'E' Comment.Multiline -'T' Comment.Multiline -'I' Comment.Multiline -'C' Comment.Multiline -' ' Comment.Multiline -'O' Comment.Multiline -'P' Comment.Multiline -'E' Comment.Multiline -'R' Comment.Multiline -'A' Comment.Multiline -'T' Comment.Multiline -'O' Comment.Multiline -'R' Comment.Multiline -'S' Comment.Multiline -' ' Comment.Multiline +' ARITHMETIC OPERATORS ' Comment.Multiline '*/' Comment.Multiline '\n' Text.Whitespace @@ -778,25 +506,7 @@ '\n\n' Text.Whitespace '/*' Comment.Multiline -' ' Comment.Multiline -'B' Comment.Multiline -'I' Comment.Multiline -'T' Comment.Multiline -'W' Comment.Multiline -'I' Comment.Multiline -'S' Comment.Multiline -'E' Comment.Multiline -' ' Comment.Multiline -'O' Comment.Multiline -'P' Comment.Multiline -'E' Comment.Multiline -'R' Comment.Multiline -'A' Comment.Multiline -'T' Comment.Multiline -'O' Comment.Multiline -'R' Comment.Multiline -'S' Comment.Multiline -' ' Comment.Multiline +' BITWISE OPERATORS ' Comment.Multiline '*/' Comment.Multiline '\n' Text.Whitespace @@ -866,28 +576,7 @@ '\n\n' Text.Whitespace '/*' Comment.Multiline -' ' Comment.Multiline -'C' Comment.Multiline -'O' Comment.Multiline -'M' Comment.Multiline -'P' Comment.Multiline -'A' Comment.Multiline -'R' Comment.Multiline -'I' Comment.Multiline -'S' Comment.Multiline -'O' Comment.Multiline -'N' Comment.Multiline -' ' Comment.Multiline -'O' Comment.Multiline -'P' Comment.Multiline -'E' Comment.Multiline -'R' Comment.Multiline -'A' Comment.Multiline -'T' Comment.Multiline -'O' Comment.Multiline -'R' Comment.Multiline -'S' Comment.Multiline -' ' Comment.Multiline +' COMPARISON OPERATORS ' Comment.Multiline '*/' Comment.Multiline '\n' Text.Whitespace @@ -902,7 +591,7 @@ 'var' Keyword ' ' Text.Whitespace -'h' Name +'h' Name.Builtin ' ' Text.Whitespace '=' Operator ' ' Text.Whitespace @@ -911,7 +600,7 @@ 'var' Keyword ' ' Text.Whitespace -'i' Name +'i' Name.Builtin ' ' Text.Whitespace '=' Operator ' ' Text.Whitespace @@ -967,23 +656,13 @@ ' ' Text.Whitespace 'firstChild' Name ' ' Text.Whitespace -'is' Keyword +'is' Operator.Word ' ' Text.Whitespace 'Parent' Name.Class '\n\n' Text.Whitespace '/*' Comment.Multiline -' ' Comment.Multiline -'F' Comment.Multiline -'U' Comment.Multiline -'N' Comment.Multiline -'C' Comment.Multiline -'T' Comment.Multiline -'I' Comment.Multiline -'O' Comment.Multiline -'N' Comment.Multiline -'S' Comment.Multiline -' ' Comment.Multiline +' FUNCTIONS ' Comment.Multiline '*/' Comment.Multiline '\n' Text.Whitespace @@ -1010,10 +689,7 @@ '=' Operator ' ' Text.Whitespace '"' Literal.String -'t' Literal.String -'h' Literal.String -'i' Literal.String -'s' Literal.String +'this' Literal.String '"' Literal.String '\n ' Text.Whitespace 'System' Name.Class @@ -1031,7 +707,7 @@ '+' Operator ' ' Text.Whitespace 'param' Name -'' Text.Whitespace +'' Punctuation ')' Punctuation '\n' Text.Whitespace @@ -1043,35 +719,14 @@ 'call' Name '(' Punctuation '"' Literal.String -'f' Literal.String -'u' Literal.String -'n' Literal.String -'c' Literal.String -'t' Literal.String -'i' Literal.String -'o' Literal.String -'n' Literal.String +'function' Literal.String '"' Literal.String -'' Text.Whitespace +'' Punctuation ')' Punctuation '\n\n' Text.Whitespace '/*' Comment.Multiline -' ' Comment.Multiline -'M' Comment.Multiline -'I' Comment.Multiline -'S' Comment.Multiline -'C' Comment.Multiline -'E' Comment.Multiline -'L' Comment.Multiline -'L' Comment.Multiline -'A' Comment.Multiline -'N' Comment.Multiline -'E' Comment.Multiline -'O' Comment.Multiline -'U' Comment.Multiline -'S' Comment.Multiline -' ' Comment.Multiline +' MISCELLANEOUS ' Comment.Multiline '*/' Comment.Multiline '\n\n' Text.Whitespace @@ -1183,15 +838,7 @@ '=' Operator ' ' Text.Whitespace '"' Literal.String -'£' Literal.String -' ' Literal.String -'é' Literal.String -' ' Literal.String -'😀' Literal.String -' ' Literal.String -' ' Literal.String -'\u200e' Literal.String -'🎷' Literal.String +'£ é 😀 \u200e🎷' Literal.String '"' Literal.String '\n' Text.Whitespace @@ -1242,7 +889,7 @@ 'b' Literal.String '"' Literal.String ']' Punctuation -'' Text.Whitespace +'' Punctuation ')' Literal.String.Interpol '"' Literal.String '\n' Text.Whitespace @@ -1257,7 +904,7 @@ '1' Literal.Number.Float '...' Operator 'h' Name -'' Text.Whitespace +'' Punctuation ')' Punctuation '.' Operator 'where' Name @@ -1265,10 +912,10 @@ '{' Punctuation ' ' Text.Whitespace '|' Operator -'i' Name +'i' Name.Builtin '|' Operator ' ' Text.Whitespace -'i' Name +'i' Name.Builtin ' ' Text.Whitespace '%' Operator ' ' Text.Whitespace @@ -1283,3 +930,13 @@ '.' Operator 'toList' Name '\n' Text.Whitespace + +'var' Keyword +' ' Text.Whitespace +'emp' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String +'"' Literal.String +'\n' Text.Whitespace From 43a876eeddb4f20aea3cfad3c3444a13ef93e857 Mon Sep 17 00:00:00 2001 From: PureFox48 <64583745+PureFox48@users.noreply.github.com> Date: Sat, 5 Nov 2022 21:30:49 +0000 Subject: [PATCH 08/16] Update wren.py --- pygments/lexers/wren.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/wren.py b/pygments/lexers/wren.py index 469e67f810..8ce4721150 100644 --- a/pygments/lexers/wren.py +++ b/pygments/lexers/wren.py @@ -8,7 +8,7 @@ import re -from pygments.lexer import RegexLexer, words, include +from pygments.lexer import RegexLexer, words from pygments.token import Whitespace, Punctuation, Keyword, Name, Comment, \ Operator, Number, String From 71ed0448efa54d08b9cbe7916cd9bd135660e85e Mon Sep 17 00:00:00 2001 From: PureFox48 <64583745+PureFox48@users.noreply.github.com> Date: Sun, 6 Nov 2022 10:01:55 +0000 Subject: [PATCH 09/16] Update wren.py 'prefixes' weren't quite right before. --- pygments/lexers/wren.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pygments/lexers/wren.py b/pygments/lexers/wren.py index 8ce4721150..e958768f2b 100644 --- a/pygments/lexers/wren.py +++ b/pygments/lexers/wren.py @@ -42,18 +42,18 @@ class WrenLexer(RegexLexer): (words(( 'as', 'break', 'class', 'construct', 'continue', 'else', 'for', 'foreign', 'if', 'import', 'return', 'static', 'super', - 'var', 'while'), prefix = r'(? Date: Sun, 6 Nov 2022 10:08:16 +0000 Subject: [PATCH 10/16] Update example.wren.output From ef321bb203a170ed074ed2003948b120155cf6e0 Mon Sep 17 00:00:00 2001 From: PureFox48 <64583745+PureFox48@users.noreply.github.com> Date: Sun, 6 Nov 2022 12:37:36 +0000 Subject: [PATCH 11/16] Update wren.py --- pygments/lexers/wren.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pygments/lexers/wren.py b/pygments/lexers/wren.py index e958768f2b..06ecea05a4 100644 --- a/pygments/lexers/wren.py +++ b/pygments/lexers/wren.py @@ -1,3 +1,4 @@ + """ pygments.lexers.wren ~~~~~~~~~~~~~~~~~~~~ @@ -15,6 +16,11 @@ __all__ = ['WrenLexer'] class WrenLexer(RegexLexer): + """ + For Wren source code, version 0.4.0. + + .. versionadded:: 2.14.0 + """ name = 'Wren' url = 'https://wren.io' aliases = ['wren'] @@ -77,9 +83,9 @@ class WrenLexer(RegexLexer): (r'"', String, 'string'), # Other string ], 'comment': [ + (r'/\*', Comment.Multiline, '#push'), (r'\*/', Comment.Multiline, '#pop'), - (r'[^*]+', Comment.Multiline), - (r'\*', Comment.Multiline), + (r'([^*/]|\*(?!/)|/(?!\*))+', Comment.Multiline), ], 'string': [ (r'"', String, '#pop'), @@ -89,7 +95,7 @@ class WrenLexer(RegexLexer): (r'\\U[a-fA-F0-9]{8}', String.Escape), # Long Unicode escape. (r'%\(', String.Interpol, ('interpolation', 'root')), - (r'[^\\"%]*', String), # All remaining characters. + (r'[^\\"%]+', String), # All remaining characters. ], 'parenthesized': [ # We only get to this state when we're at a ')'. From b0df3d7cd076c8c23f1b452ddaa1ab806b845e8a Mon Sep 17 00:00:00 2001 From: PureFox48 <64583745+PureFox48@users.noreply.github.com> Date: Sun, 6 Nov 2022 12:44:06 +0000 Subject: [PATCH 12/16] Update example.wren.output --- tests/examplefiles/wren/example.wren.output | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/examplefiles/wren/example.wren.output b/tests/examplefiles/wren/example.wren.output index 37e8afb437..759a3df887 100644 --- a/tests/examplefiles/wren/example.wren.output +++ b/tests/examplefiles/wren/example.wren.output @@ -30,13 +30,13 @@ '\n\n' Text.Whitespace '/*' Comment.Multiline -'\n multiline comment\n /' Comment.Multiline -'*' Comment.Multiline +'\n multiline comment\n ' Comment.Multiline +'/*' Comment.Multiline '\n nested multiline comment\n ' Comment.Multiline '*/' Comment.Multiline -'\n' Text.Whitespace +'\n' Comment.Multiline -'*/' Operator +'*/' Comment.Multiline '\n\n' Text.Whitespace '/*' Comment.Multiline From f982b0ae6e5f1151473a15c9163b5fac7f81bc50 Mon Sep 17 00:00:00 2001 From: PureFox48 <64583745+PureFox48@users.noreply.github.com> Date: Mon, 14 Nov 2022 09:32:05 +0000 Subject: [PATCH 13/16] Update wren.py Corrected module docstring. --- pygments/lexers/wren.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pygments/lexers/wren.py b/pygments/lexers/wren.py index 06ecea05a4..89d2cee3b2 100644 --- a/pygments/lexers/wren.py +++ b/pygments/lexers/wren.py @@ -1,8 +1,9 @@ - """ pygments.lexers.wren ~~~~~~~~~~~~~~~~~~~~ + Lexer for Wren. + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ From 9d09ed40cc05a4ead18a1a41f907785f0e9d627c Mon Sep 17 00:00:00 2001 From: PureFox48 <64583745+PureFox48@users.noreply.github.com> Date: Mon, 14 Nov 2022 12:29:34 +0000 Subject: [PATCH 14/16] Update wren.py Hopefully, fixes the problem with the random test. --- pygments/lexers/wren.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pygments/lexers/wren.py b/pygments/lexers/wren.py index 89d2cee3b2..690b943867 100644 --- a/pygments/lexers/wren.py +++ b/pygments/lexers/wren.py @@ -12,7 +12,7 @@ from pygments.lexer import RegexLexer, words from pygments.token import Whitespace, Punctuation, Keyword, Name, Comment, \ - Operator, Number, String + Operator, Number, String, Error __all__ = ['WrenLexer'] @@ -49,16 +49,13 @@ class WrenLexer(RegexLexer): (words(( 'as', 'break', 'class', 'construct', 'continue', 'else', 'for', 'foreign', 'if', 'import', 'return', 'static', 'super', - 'var', 'while'), prefix = r'(? Date: Mon, 14 Nov 2022 12:38:18 +0000 Subject: [PATCH 15/16] Update example.wren.output --- tests/examplefiles/wren/example.wren.output | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/examplefiles/wren/example.wren.output b/tests/examplefiles/wren/example.wren.output index 759a3df887..f38a3281a1 100644 --- a/tests/examplefiles/wren/example.wren.output +++ b/tests/examplefiles/wren/example.wren.output @@ -168,7 +168,7 @@ ' ' Text.Whitespace '{' Punctuation ' ' Text.Whitespace -'this' Name +'this' Keyword '.' Operator 'parent' Name ' ' Text.Whitespace @@ -306,7 +306,7 @@ 'for' Keyword ' ' Text.Whitespace '(' Punctuation -'i' Name.Builtin +'i' Name ' ' Text.Whitespace 'in' Operator.Word ' ' Text.Whitespace @@ -321,7 +321,7 @@ 'if' Keyword ' ' Text.Whitespace '(' Punctuation -'i' Name.Builtin +'i' Name ' ' Text.Whitespace '==' Operator ' ' Text.Whitespace @@ -340,7 +340,7 @@ 'if' Keyword ' ' Text.Whitespace '(' Punctuation -'i' Name.Builtin +'i' Name ' ' Text.Whitespace '==' Operator ' ' Text.Whitespace @@ -358,7 +358,7 @@ '.' Operator 'print' Name '(' Punctuation -'i' Name.Builtin +'i' Name '' Punctuation ')' Punctuation '\n' Text.Whitespace @@ -591,7 +591,7 @@ 'var' Keyword ' ' Text.Whitespace -'h' Name.Builtin +'h' Name ' ' Text.Whitespace '=' Operator ' ' Text.Whitespace @@ -600,7 +600,7 @@ 'var' Keyword ' ' Text.Whitespace -'i' Name.Builtin +'i' Name ' ' Text.Whitespace '=' Operator ' ' Text.Whitespace @@ -912,10 +912,10 @@ '{' Punctuation ' ' Text.Whitespace '|' Operator -'i' Name.Builtin +'i' Name '|' Operator ' ' Text.Whitespace -'i' Name.Builtin +'i' Name ' ' Text.Whitespace '%' Operator ' ' Text.Whitespace From 9417349f64d6015ab7f38bf4011ae36c68861af5 Mon Sep 17 00:00:00 2001 From: Jean Abou Samra Date: Tue, 29 Nov 2022 01:34:29 +0100 Subject: [PATCH 16/16] Fix non-termination on unmatched paren --- pygments/lexers/wren.py | 32 +++++++-------------- tests/examplefiles/wren/example.wren | 2 +- tests/examplefiles/wren/example.wren.output | 21 ++------------ tests/snippets/wren/lonely-paren.txt | 10 +++++++ 4 files changed, 24 insertions(+), 41 deletions(-) create mode 100644 tests/snippets/wren/lonely-paren.txt diff --git a/pygments/lexers/wren.py b/pygments/lexers/wren.py index 690b943867..f0f3793f88 100644 --- a/pygments/lexers/wren.py +++ b/pygments/lexers/wren.py @@ -1,16 +1,16 @@ """ pygments.lexers.wren ~~~~~~~~~~~~~~~~~~~~ - + Lexer for Wren. - + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ import re -from pygments.lexer import RegexLexer, words +from pygments.lexer import include, RegexLexer, words from pygments.token import Whitespace, Punctuation, Keyword, Name, Comment, \ Operator, Number, String, Error @@ -35,15 +35,11 @@ class WrenLexer(RegexLexer): (r'\s+', Whitespace), (r'[,\\\[\]{}]', Punctuation), - # Push a parenthesized state so that we know the corresponding ')' - # is for a parenthesized expression and not interpolation. - (r'\(', Punctuation, ('parenthesized', 'root')), - - # In this state, we don't know whether a closing ')' is for a - # parenthesized expression or the end of an interpolation. So, do - # a non-consuming match and let the parent state (either - # 'parenthesized' or 'interpolation') decide. - (r'(?=\))', Punctuation, '#pop'), + # Really 'root', not '#push': in 'interpolation', + # parentheses inside the interpolation expression are + # Punctuation, not String.Interpol. + (r'\(', Punctuation, 'root'), + (r'\)', Punctuation, '#pop'), # Keywords. (words(( @@ -79,9 +75,6 @@ class WrenLexer(RegexLexer): # Strings. (r'""".*?"""', String), # Raw string (r'"', String, 'string'), # Other string - - # Errors. - (r'.+$', Error), ], 'comment': [ (r'/\*', Comment.Multiline, '#push'), @@ -95,15 +88,12 @@ class WrenLexer(RegexLexer): (r'\\u[a-fA-F0-9]{4}', String.Escape), # Unicode escape. (r'\\U[a-fA-F0-9]{8}', String.Escape), # Long Unicode escape. - (r'%\(', String.Interpol, ('interpolation', 'root')), + (r'%\(', String.Interpol, 'interpolation'), (r'[^\\"%]+', String), # All remaining characters. ], - 'parenthesized': [ - # We only get to this state when we're at a ')'. - (r'\)', Punctuation, '#pop'), - ], 'interpolation': [ - # We only get to this state when we're at a ')'. + # redefine closing paren to be String.Interpol (r'\)', String.Interpol, '#pop'), + include('root'), ], } diff --git a/tests/examplefiles/wren/example.wren b/tests/examplefiles/wren/example.wren index 550fcc54c8..272d3cb94e 100644 --- a/tests/examplefiles/wren/example.wren +++ b/tests/examplefiles/wren/example.wren @@ -116,7 +116,7 @@ var o = l || m var esc = "\\ \% \" \0 \a \b \t \f \n \r \v \e \x01 \uabcd \Uabcdef01" var uni = "£ é 😀 ‎🎷" var map = {"a": 1, "b": 2} -var iex = "%(map["a"] + map["b"])" +var iex = "%(map["a"] + map[("b")])" var odd = (1...h).where { |i| i % 2 == 1 } .toList var emp = "" diff --git a/tests/examplefiles/wren/example.wren.output b/tests/examplefiles/wren/example.wren.output index f38a3281a1..2b76dd6313 100644 --- a/tests/examplefiles/wren/example.wren.output +++ b/tests/examplefiles/wren/example.wren.output @@ -61,7 +61,6 @@ ' ' Text.Whitespace '(' Punctuation 'field' Name -'' Punctuation ')' Punctuation ' ' Text.Whitespace '{' Punctuation @@ -79,7 +78,6 @@ 'new' Name '(' Punctuation 'parent' Name -'' Punctuation ')' Punctuation ' ' Text.Whitespace '{' Punctuation @@ -106,7 +104,6 @@ ' ' Text.Whitespace 'method' Name '(' Punctuation -'' Punctuation ')' Punctuation '\n' Text.Whitespace @@ -137,7 +134,6 @@ ',' Punctuation ' ' Text.Whitespace 'child' Name -'' Punctuation ')' Punctuation ' ' Text.Whitespace '{' Punctuation @@ -145,7 +141,6 @@ 'super' Keyword '(' Punctuation 'parent' Name -'' Punctuation ')' Punctuation '\n ' Text.Whitespace '_child' Name.Variable.Instance @@ -193,7 +188,6 @@ '.' Operator 'new' Name '(' Punctuation -'' Punctuation ')' Punctuation '\n' Text.Whitespace @@ -215,7 +209,6 @@ '"' Literal.String '%(' Literal.String.Interpol 'name' Name -'' Punctuation ')' Literal.String.Interpol ' Smith' Literal.String '"' Literal.String @@ -237,7 +230,6 @@ ',' Punctuation ' ' Text.Whitespace 'fullName' Name -'' Punctuation ')' Punctuation '\n' Text.Whitespace @@ -313,7 +305,6 @@ '1' Literal.Number.Float '..' Operator '5' Literal.Number.Float -'' Punctuation ')' Punctuation ' ' Text.Whitespace '{' Punctuation @@ -326,7 +317,6 @@ '==' Operator ' ' Text.Whitespace '2' Literal.Number.Float -'' Punctuation ')' Punctuation ' ' Text.Whitespace '{' Punctuation @@ -345,7 +335,6 @@ '==' Operator ' ' Text.Whitespace '4' Literal.Number.Float -'' Punctuation ')' Punctuation ' ' Text.Whitespace '{' Punctuation @@ -359,7 +348,6 @@ 'print' Name '(' Punctuation 'i' Name -'' Punctuation ')' Punctuation '\n' Text.Whitespace @@ -383,7 +371,6 @@ '<=' Operator ' ' Text.Whitespace '1e+1' Literal.Number.Float -'' Punctuation ')' Punctuation ' ' Text.Whitespace '{' Punctuation @@ -396,7 +383,6 @@ '==' Operator ' ' Text.Whitespace '8' Literal.Number.Float -'' Punctuation ')' Punctuation ' ' Text.Whitespace 'break' Keyword @@ -406,7 +392,6 @@ 'print' Name '(' Punctuation 'j' Name -'' Punctuation ')' Punctuation '\n ' Text.Whitespace 'j' Name @@ -707,7 +692,6 @@ '+' Operator ' ' Text.Whitespace 'param' Name -'' Punctuation ')' Punctuation '\n' Text.Whitespace @@ -721,7 +705,6 @@ '"' Literal.String 'function' Literal.String '"' Literal.String -'' Punctuation ')' Punctuation '\n\n' Text.Whitespace @@ -885,11 +868,12 @@ ' ' Text.Whitespace 'map' Name '[' Punctuation +'(' Punctuation '"' Literal.String 'b' Literal.String '"' Literal.String +')' Punctuation ']' Punctuation -'' Punctuation ')' Literal.String.Interpol '"' Literal.String '\n' Text.Whitespace @@ -904,7 +888,6 @@ '1' Literal.Number.Float '...' Operator 'h' Name -'' Punctuation ')' Punctuation '.' Operator 'where' Name diff --git a/tests/snippets/wren/lonely-paren.txt b/tests/snippets/wren/lonely-paren.txt new file mode 100644 index 0000000000..5236e601b7 --- /dev/null +++ b/tests/snippets/wren/lonely-paren.txt @@ -0,0 +1,10 @@ +---input--- +// This invalid input should terminate. +) + +---tokens--- +'// This invalid input should terminate.' Comment.Single +'\n' Text.Whitespace + +')' Punctuation +'\n' Text.Whitespace