From a77968aa24e0b1675c2819d6c026baa0d0ed420b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20R=C3=B6mer?= <16097755+hroemer@users.noreply.github.com> Date: Tue, 9 Apr 2024 18:03:08 +0200 Subject: [PATCH 1/7] #2515: adding basic mojo lexer --- AUTHORS | 2 +- pygments/lexers/_mapping.py | 1 + pygments/lexers/mojo.py | 701 ++++++++++++++++++++++++++++++++++++ 3 files changed, 703 insertions(+), 1 deletion(-) create mode 100644 pygments/lexers/mojo.py diff --git a/AUTHORS b/AUTHORS index bdb69cccc5..a7928ea88b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -276,6 +276,6 @@ Other contributors, listed alphabetically, are: * vanillajonathan -- PRQL lexer * Nikolay Antipov -- OpenSCAD lexer * Markus Meyer, Nextron Systems -- YARA lexer - +* Hannes Römer -- Mojo lexer Many thanks for all contributions! diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index ace820fe55..400064535d 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -294,6 +294,7 @@ 'MCSchemaLexer': ('pygments.lexers.minecraft', 'MCSchema', ('mcschema',), ('*.mcschema',), ('text/mcschema',)), 'MIMELexer': ('pygments.lexers.mime', 'MIME', ('mime',), (), ('multipart/mixed', 'multipart/related', 'multipart/alternative')), 'MIPSLexer': ('pygments.lexers.mips', 'MIPS', ('mips',), ('*.mips', '*.MIPS'), ()), + 'MojoLexer': ('pygments.lexers.mojo', 'Mojo', ('mojo',), ('*.mojo', '*.🔥'), ('text/x-mojo', 'application/x-mojo')), 'MOOCodeLexer': ('pygments.lexers.scripting', 'MOOCode', ('moocode', 'moo'), ('*.moo',), ('text/x-moocode',)), 'MSDOSSessionLexer': ('pygments.lexers.shell', 'MSDOS Session', ('doscon',), (), ()), 'Macaulay2Lexer': ('pygments.lexers.macaulay2', 'Macaulay2', ('macaulay2',), ('*.m2',), ()), diff --git a/pygments/lexers/mojo.py b/pygments/lexers/mojo.py new file mode 100644 index 0000000000..e432b4a48f --- /dev/null +++ b/pygments/lexers/mojo.py @@ -0,0 +1,701 @@ +""" + pygments.lexers.mojo + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexers for Mojo and related languages. + + :copyright: Copyright 2024-2024 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import keyword + +from pygments import unistring as uni +from pygments.lexer import ( + RegexLexer, + bygroups, + combined, + default, + include, + this, + using, + words, +) +from pygments.token import ( + Comment, + Error, + Keyword, + Name, + Number, + Operator, + Punctuation, + String, + Text, + Whitespace, +) +from pygments.util import shebang_matches + +__all__ = ["MojoLexer"] + + +class MojoLexer(RegexLexer): + """ + For Mojo source code (version 24.2). + """ + + name = "Mojo" + url = "https://docs.modular.com/mojo/" + aliases = ["mojo", "🔥"] + filenames = [ + "*.mojo", + "*.🔥", + ] + mimetypes = [ + "text/x-mojo", + "application/x-mojo", + ] + version_added = "2.18" + + uni_name = f"[{uni.xid_start}][{uni.xid_continue}]*" + + def innerstring_rules(ttype): + return [ + # the old style '%s' % (...) string formatting (still valid in Py3) + ( + r"%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?" + "[hlL]?[E-GXc-giorsaux%]", + String.Interpol, + ), + # the new style '{}'.format(...) string formatting + ( + r"\{" + r"((\w+)((\.\w+)|(\[[^\]]+\]))*)?" # field name + r"(\![sra])?" # conversion + r"(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?" + r"\}", + String.Interpol, + ), + # backslashes, quotes and formatting signs must be parsed one at a time + (r'[^\\\'"%{\n]+', ttype), + (r'[\'"\\]', ttype), + # unhandled string formatting sign + (r"%|(\{{1,2})", ttype), + # newlines are an error (use "nl" state) + ] + + def fstring_rules(ttype): + return [ + # Assuming that a '}' is the closing brace after format specifier. + # Sadly, this means that we won't detect syntax error. But it's + # more important to parse correct syntax correctly, than to + # highlight invalid syntax. + (r"\}", String.Interpol), + (r"\{", String.Interpol, "expr-inside-fstring"), + # backslashes, quotes and formatting signs must be parsed one at a time + (r'[^\\\'"{}\n]+', ttype), + (r'[\'"\\]', ttype), + # newlines are an error (use "nl" state) + ] + + tokens = { + "root": [ + (r"\n", Whitespace), + ( + r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")', + bygroups(Whitespace, String.Affix, String.Doc), + ), + ( + r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')", + bygroups(Whitespace, String.Affix, String.Doc), + ), + (r"\A#!.+$", Comment.Hashbang), + (r"#.*$", Comment.Single), + (r"\\\n", Text), + (r"\\", Text), + include("keywords"), + include("soft-keywords"), + ( + r"(alias)((?:\s|\\\s)+)", + bygroups(Keyword, Text), + "varname", # TODO varname the right fit? + ), + (r"(var)((?:\s|\\\s)+)", bygroups(Keyword, Text), "varname"), + (r"(def)((?:\s|\\\s)+)", bygroups(Keyword, Text), "funcname"), + (r"(fn)((?:\s|\\\s)+)", bygroups(Keyword, Text), "funcname"), + ( + r"(class)((?:\s|\\\s)+)", + bygroups(Keyword, Text), + "classname", + ), # not implemented yet + (r"(struct)((?:\s|\\\s)+)", bygroups(Keyword, Text), "structname"), + (r"(trait)((?:\s|\\\s)+)", bygroups(Keyword, Text), "structname"), + (r"(from)((?:\s|\\\s)+)", bygroups(Keyword.Namespace, Text), "fromimport"), + (r"(import)((?:\s|\\\s)+)", bygroups(Keyword.Namespace, Text), "import"), + include("expr"), + ], + "expr": [ + # raw f-strings + ( + '(?i)(rf|fr)(""")', + bygroups(String.Affix, String.Double), + combined("rfstringescape", "tdqf"), + ), + ( + "(?i)(rf|fr)(''')", + bygroups(String.Affix, String.Single), + combined("rfstringescape", "tsqf"), + ), + ( + '(?i)(rf|fr)(")', + bygroups(String.Affix, String.Double), + combined("rfstringescape", "dqf"), + ), + ( + "(?i)(rf|fr)(')", + bygroups(String.Affix, String.Single), + combined("rfstringescape", "sqf"), + ), + # non-raw f-strings + ( + '([fF])(""")', + bygroups(String.Affix, String.Double), + combined("fstringescape", "tdqf"), + ), + ( + "([fF])(''')", + bygroups(String.Affix, String.Single), + combined("fstringescape", "tsqf"), + ), + ( + '([fF])(")', + bygroups(String.Affix, String.Double), + combined("fstringescape", "dqf"), + ), + ( + "([fF])(')", + bygroups(String.Affix, String.Single), + combined("fstringescape", "sqf"), + ), + # raw bytes and strings + ('(?i)(rb|br|r)(""")', bygroups(String.Affix, String.Double), "tdqs"), + ("(?i)(rb|br|r)(''')", bygroups(String.Affix, String.Single), "tsqs"), + ('(?i)(rb|br|r)(")', bygroups(String.Affix, String.Double), "dqs"), + ("(?i)(rb|br|r)(')", bygroups(String.Affix, String.Single), "sqs"), + # non-raw strings + ( + '([uU]?)(""")', + bygroups(String.Affix, String.Double), + combined("stringescape", "tdqs"), + ), + ( + "([uU]?)(''')", + bygroups(String.Affix, String.Single), + combined("stringescape", "tsqs"), + ), + ( + '([uU]?)(")', + bygroups(String.Affix, String.Double), + combined("stringescape", "dqs"), + ), + ( + "([uU]?)(')", + bygroups(String.Affix, String.Single), + combined("stringescape", "sqs"), + ), + # non-raw bytes + ( + '([bB])(""")', + bygroups(String.Affix, String.Double), + combined("bytesescape", "tdqs"), + ), + ( + "([bB])(''')", + bygroups(String.Affix, String.Single), + combined("bytesescape", "tsqs"), + ), + ( + '([bB])(")', + bygroups(String.Affix, String.Double), + combined("bytesescape", "dqs"), + ), + ( + "([bB])(')", + bygroups(String.Affix, String.Single), + combined("bytesescape", "sqs"), + ), + (r"[^\S\n]+", Text), + include("numbers"), + (r"!=|==|<<|>>|:=|[-~+/*%=<>&^|.]", Operator), + (r"[]{}:(),;[]", Punctuation), + (r"(in|is|and|or|not)\b", Operator.Word), + include("expr-keywords"), + include("builtins"), + include("magicfuncs"), + include("magicvars"), + include("name"), + ], + "expr-inside-fstring": [ + (r"[{([]", Punctuation, "expr-inside-fstring-inner"), + # without format specifier + ( + r"(=\s*)?" # debug (https://bugs.python.org/issue36817) + r"(\![sraf])?" # conversion + r"\}", + String.Interpol, + "#pop", + ), + # with format specifier + # we'll catch the remaining '}' in the outer scope + ( + r"(=\s*)?" # debug (https://bugs.python.org/issue36817) + r"(\![sraf])?" # conversion + r":", + String.Interpol, + "#pop", + ), + (r"\s+", Whitespace), # allow new lines + include("expr"), + ], + "expr-inside-fstring-inner": [ + (r"[{([]", Punctuation, "expr-inside-fstring-inner"), + (r"[])}]", Punctuation, "#pop"), + (r"\s+", Whitespace), # allow new lines + include("expr"), + ], + "expr-keywords": [ + # Based on https://docs.python.org/3/reference/expressions.html + ( + words( + ( + "async for", # TODO https://docs.modular.com/mojo/roadmap#no-async-for-or-async-with + "async with", # TODO https://docs.modular.com/mojo/roadmap#no-async-for-or-async-with + "await", + "else", + "for", + "if", + "lambda", + "yield", + "yield from", + ), + suffix=r"\b", + ), + Keyword, + ), + (words(("True", "False", "None"), suffix=r"\b"), Keyword.Constant), + ], + "keywords": [ + ( + words( + ( + "assert", + "async", + "await", + "borrowed", + "break", + "continue", + "del", + "elif", + "else", + "except", + "finally", + "for", + "global", + "if", + "lambda", + "pass", + "raise", + "nonlocal", + "return", + "try", + "while", + "yield", + "yield from", + "as", + "with", + ), + suffix=r"\b", + ), + Keyword, + ), + (words(("True", "False", "None"), suffix=r"\b"), Keyword.Constant), + ], + "soft-keywords": [ + # `match`, `case` and `_` soft keywords + ( + r"(^[ \t]*)" # at beginning of line + possible indentation + r"(match|case)\b" # a possible keyword + r"(?![ \t]*(?:" # not followed by... + r"[:,;=^&|@~)\]}]|(?:" + # characters and keywords that mean this isn't + # pattern matching (but None/True/False is ok) + r"|".join(k for k in keyword.kwlist if k[0].islower()) + + r")\b))", + bygroups(Text, Keyword), + "soft-keywords-inner", + ), + ], + "soft-keywords-inner": [ + # optional `_` keyword + (r"(\s+)([^\n_]*)(_\b)", bygroups(Whitespace, using(this), Keyword)), + default("#pop"), + ], + "builtins": [ + ( + words( + ( + "__import__", + "abs", + "aiter", + "all", + "any", + "bin", + "bool", + "bytearray", + "breakpoint", + "bytes", + "callable", + "chr", + "classmethod", + "compile", + "complex", + "delattr", + "dict", + "dir", + "divmod", + "enumerate", + "eval", + "filter", + "float", + "format", + "frozenset", + "getattr", + "globals", + "hasattr", + "hash", + "hex", + "id", + "input", + "int", + "isinstance", + "issubclass", + "iter", + "len", + "list", + "locals", + "map", + "max", + "memoryview", + "min", + "next", + "object", + "oct", + "open", + "ord", + "pow", + "print", + "property", + "range", + "repr", + "reversed", + "round", + "set", + "setattr", + "slice", + "sorted", + "staticmethod", + "str", + "sum", + "super", + "tuple", + "type", + "vars", + "zip", + # Mojo builtin types: https://docs.modular.com/mojo/stdlib/builtin/ + "AnyType", + "Coroutine", + "DType", + "Error", + "Int", + "List", + "ListLiteral", + "Scalar", + "Int8", + "UInt8", + "Int16", + "UInt16", + "Int32", + "UInt32", + "Int64", + "UInt64", + "BFloat16", + "Float16", + "Float32", + "Float64", + "SIMD", + "String", + "Tensor", + "Tuple", + "Movable", + "Copyable", + "CollectionElement", + ), + prefix=r"(? Date: Tue, 9 Apr 2024 18:20:34 +0200 Subject: [PATCH 2/7] #2515: adding basic mojo lexer --- pygments/lexers/mojo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/mojo.py b/pygments/lexers/mojo.py index e432b4a48f..d26090b71d 100644 --- a/pygments/lexers/mojo.py +++ b/pygments/lexers/mojo.py @@ -697,5 +697,5 @@ def fstring_rules(ttype): def analyse_text(text): return ( - shebang_matches(text, r"mojo?") or "import " in text[:1000] + shebang_matches(text, r"mojo?") or "import " in text[:1000] ) # TODO supported? From f0a2b5835c5d34b5cd45844f1d1f22deca1e9aed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20R=C3=B6mer?= <16097755+hroemer@users.noreply.github.com> Date: Sun, 14 Apr 2024 11:51:02 +0200 Subject: [PATCH 3/7] #2515: adding mojo snippets and example file --- pygments/lexers/_mapping.py | 2 +- pygments/lexers/mojo.py | 4 +- tests/examplefiles/mojo/sample.mojo | 124 ++++ tests/examplefiles/mojo/sample.mojo.output | 820 +++++++++++++++++++++ tests/snippets/mojo/test_floats.txt | 75 ++ tests/snippets/mojo/test_kw.txt | 35 + tests/snippets/mojo/test_needs_name.txt | 55 ++ tests/snippets/mojo/test_soft_kwds.txt | 67 ++ 8 files changed, 1179 insertions(+), 3 deletions(-) create mode 100644 tests/examplefiles/mojo/sample.mojo create mode 100644 tests/examplefiles/mojo/sample.mojo.output create mode 100644 tests/snippets/mojo/test_floats.txt create mode 100644 tests/snippets/mojo/test_kw.txt create mode 100644 tests/snippets/mojo/test_needs_name.txt create mode 100644 tests/snippets/mojo/test_soft_kwds.txt diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index 400064535d..eaca6e21ae 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -294,7 +294,6 @@ 'MCSchemaLexer': ('pygments.lexers.minecraft', 'MCSchema', ('mcschema',), ('*.mcschema',), ('text/mcschema',)), 'MIMELexer': ('pygments.lexers.mime', 'MIME', ('mime',), (), ('multipart/mixed', 'multipart/related', 'multipart/alternative')), 'MIPSLexer': ('pygments.lexers.mips', 'MIPS', ('mips',), ('*.mips', '*.MIPS'), ()), - 'MojoLexer': ('pygments.lexers.mojo', 'Mojo', ('mojo',), ('*.mojo', '*.🔥'), ('text/x-mojo', 'application/x-mojo')), 'MOOCodeLexer': ('pygments.lexers.scripting', 'MOOCode', ('moocode', 'moo'), ('*.moo',), ('text/x-moocode',)), 'MSDOSSessionLexer': ('pygments.lexers.shell', 'MSDOS Session', ('doscon',), (), ()), 'Macaulay2Lexer': ('pygments.lexers.macaulay2', 'Macaulay2', ('macaulay2',), ('*.m2',), ()), @@ -318,6 +317,7 @@ 'ModelicaLexer': ('pygments.lexers.modeling', 'Modelica', ('modelica',), ('*.mo',), ('text/x-modelica',)), 'Modula2Lexer': ('pygments.lexers.modula2', 'Modula-2', ('modula2', 'm2'), ('*.def', '*.mod'), ('text/x-modula2',)), 'MoinWikiLexer': ('pygments.lexers.markup', 'MoinMoin/Trac Wiki markup', ('trac-wiki', 'moin'), (), ('text/x-trac-wiki',)), + 'MojoLexer': ('pygments.lexers.mojo', 'Mojo', ('mojo', '🔥'), ('*.mojo', '*.🔥'), ('text/x-mojo', 'application/x-mojo')), 'MonkeyLexer': ('pygments.lexers.basic', 'Monkey', ('monkey',), ('*.monkey',), ('text/x-monkey',)), 'MonteLexer': ('pygments.lexers.monte', 'Monte', ('monte',), ('*.mt',), ()), 'MoonScriptLexer': ('pygments.lexers.scripting', 'MoonScript', ('moonscript', 'moon'), ('*.moon',), ('text/x-moonscript', 'application/x-moonscript')), diff --git a/pygments/lexers/mojo.py b/pygments/lexers/mojo.py index d26090b71d..1a12561fba 100644 --- a/pygments/lexers/mojo.py +++ b/pygments/lexers/mojo.py @@ -40,7 +40,7 @@ class MojoLexer(RegexLexer): """ - For Mojo source code (version 24.2). + For Mojo source code (version 24.2.1). """ name = "Mojo" @@ -697,5 +697,5 @@ def fstring_rules(ttype): def analyse_text(text): return ( - shebang_matches(text, r"mojo?") or "import " in text[:1000] + shebang_matches(text, r"mojo?") or "import " in text[:1000] ) # TODO supported? diff --git a/tests/examplefiles/mojo/sample.mojo b/tests/examplefiles/mojo/sample.mojo new file mode 100644 index 0000000000..71ac795676 --- /dev/null +++ b/tests/examplefiles/mojo/sample.mojo @@ -0,0 +1,124 @@ +from sys.info import ( + os_is_linux, + os_is_macos, + has_sse4, + has_avx, + has_avx2, + num_physical_cores, + num_logical_cores, +) +from sys.info import _current_cpu, _current_target, _triple_attr + + +def func() -> None: + variable = "implicit variable" + print("def function", variable) + + +fn strict_func(): + var variable = "explicit variable" + print("fn function", variable) + + +fn generic_func[a: Int = 3, msg: StringLiteral = "woof"](): + """ + A generic function. + """ + print(msg, a) + + +fn get_sys_info() -> String: + """ + Retrieves system information (based upon: https://github.com/modularml/mojo/blob/main/examples/deviceinfo.mojo). + """ + var os = "" + if os_is_linux(): + os = "linux" + elif os_is_macos(): + os = "macOS" + else: + os = "windows" + var cpu = String(_current_cpu()) + var arch = String(_triple_attr()) + var cpu_features = String("") + if has_sse4(): + cpu_features += "sse4" + if has_avx(): + cpu_features += " avx" + if has_avx2(): + cpu_features += " avx2" + + var info = ( + """System information: + OS : """ + + os + + """ + CPU : """ + + cpu + + """ + Arch : """ + + arch + + """ + Physical Cores : """ + + num_physical_cores() + + """ + Logical Cores : """ + + num_logical_cores() + + """ + CPU Features : """ + + cpu_features + ) + + return info + + +trait Greet: + """ + A trait example. + """ + + fn say_hello(self) -> None: + """Says hello.""" + ... + + +@value +struct Person(Greet): + """ + A person. + """ + + var name: String + + fn __init__(inout self, name: String): + self.name = name + + fn __str__(self) -> String: + """ + Overridden __str__ method + """ + return "Person named " + self.name + + fn say_hello(self) -> None: + print(self.name, "says hello!") + + +fn main(): + for i in range(3): + var pers = Person("Per_" + String(i)) + print(pers) + pers.say_hello() + + generic_func() # prints 'woof 3' + generic_func[5]() # prints 'woof 5' + generic_func[7, "meow"]() # prints 'meow 7' + + try: + func() + strict_func() + + raise Error("fail") + except: + print("error was raised") + + print(get_sys_info()) diff --git a/tests/examplefiles/mojo/sample.mojo.output b/tests/examplefiles/mojo/sample.mojo.output new file mode 100644 index 0000000000..dd30b6c8c1 --- /dev/null +++ b/tests/examplefiles/mojo/sample.mojo.output @@ -0,0 +1,820 @@ +'from' Keyword.Namespace +' ' Text +'sys' Name.Namespace +'.' Name.Namespace +'info' Name.Namespace +' ' Text +'import' Keyword.Namespace +' ' Text +'(' Punctuation +'\n' Text.Whitespace + +' ' Text +'os_is_linux' Name +',' Punctuation +'\n' Text.Whitespace + +' ' Text +'os_is_macos' Name +',' Punctuation +'\n' Text.Whitespace + +' ' Text +'has_sse4' Name +',' Punctuation +'\n' Text.Whitespace + +' ' Text +'has_avx' Name +',' Punctuation +'\n' Text.Whitespace + +' ' Text +'has_avx2' Name +',' Punctuation +'\n' Text.Whitespace + +' ' Text +'num_physical_cores' Name +',' Punctuation +'\n' Text.Whitespace + +' ' Text +'num_logical_cores' Name +',' Punctuation +'\n' Text.Whitespace + +')' Punctuation +'\n' Text.Whitespace + +'from' Keyword.Namespace +' ' Text +'sys' Name.Namespace +'.' Name.Namespace +'info' Name.Namespace +' ' Text +'import' Keyword.Namespace +' ' Text +'_current_cpu' Name +',' Punctuation +' ' Text +'_current_target' Name +',' Punctuation +' ' Text +'_triple_attr' Name +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'def' Keyword +' ' Text +'func' Name.Function +'(' Punctuation +')' Punctuation +' ' Text +'-' Operator +'>' Operator +' ' Text +'None' Keyword.Constant +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'variable' Name +' ' Text +'=' Operator +' ' Text +'"' Literal.String.Double +'implicit variable' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +' ' Text +'print' Name.Builtin +'(' Punctuation +'"' Literal.String.Double +'def function' Literal.String.Double +'"' Literal.String.Double +',' Punctuation +' ' Text +'variable' Name +')' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'fn' Keyword +' ' Text +'strict_func' Name.Function +'(' Punctuation +')' Punctuation +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'var' Keyword +' ' Text +'variable' Name.Variable +' ' Text +'=' Operator +' ' Text +'"' Literal.String.Double +'explicit variable' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +' ' Text +'print' Name.Builtin +'(' Punctuation +'"' Literal.String.Double +'fn function' Literal.String.Double +'"' Literal.String.Double +',' Punctuation +' ' Text +'variable' Name +')' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'fn' Keyword +' ' Text +'generic_func' Name.Function +'[' Punctuation +'a' Name +':' Punctuation +' ' Text +'Int' Name.Builtin +' ' Text +'=' Operator +' ' Text +'3' Literal.Number.Integer +',' Punctuation +' ' Text +'msg' Name +':' Punctuation +' ' Text +'StringLiteral' Name +' ' Text +'=' Operator +' ' Text +'"' Literal.String.Double +'woof' Literal.String.Double +'"' Literal.String.Double +']' Punctuation +'(' Punctuation +')' Punctuation +':' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'"""\n A generic function.\n """' Literal.String.Doc +'\n' Text.Whitespace + +' ' Text +'print' Name.Builtin +'(' Punctuation +'msg' Name +',' Punctuation +' ' Text +'a' Name +')' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'fn' Keyword +' ' Text +'get_sys_info' Name.Function +'(' Punctuation +')' Punctuation +' ' Text +'-' Operator +'>' Operator +' ' Text +'String' Name.Builtin +':' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'"""\n Retrieves system information (based upon: https://github.com/modularml/mojo/blob/main/examples/deviceinfo.mojo).\n """' Literal.String.Doc +'\n' Text.Whitespace + +' ' Text +'var' Keyword +' ' Text +'os' Name.Variable +' ' Text +'=' Operator +' ' Text +'"' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +' ' Text +'if' Keyword +' ' Text +'os_is_linux' Name +'(' Punctuation +')' Punctuation +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'os' Name +' ' Text +'=' Operator +' ' Text +'"' Literal.String.Double +'linux' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +' ' Text +'elif' Keyword +' ' Text +'os_is_macos' Name +'(' Punctuation +')' Punctuation +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'os' Name +' ' Text +'=' Operator +' ' Text +'"' Literal.String.Double +'macOS' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +' ' Text +'else' Keyword +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'os' Name +' ' Text +'=' Operator +' ' Text +'"' Literal.String.Double +'windows' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +' ' Text +'var' Keyword +' ' Text +'cpu' Name.Variable +' ' Text +'=' Operator +' ' Text +'String' Name.Builtin +'(' Punctuation +'_current_cpu' Name +'(' Punctuation +')' Punctuation +')' Punctuation +'\n' Text.Whitespace + +' ' Text +'var' Keyword +' ' Text +'arch' Name.Variable +' ' Text +'=' Operator +' ' Text +'String' Name.Builtin +'(' Punctuation +'_triple_attr' Name +'(' Punctuation +')' Punctuation +')' Punctuation +'\n' Text.Whitespace + +' ' Text +'var' Keyword +' ' Text +'cpu_features' Name.Variable +' ' Text +'=' Operator +' ' Text +'String' Name.Builtin +'(' Punctuation +'"' Literal.String.Double +'"' Literal.String.Double +')' Punctuation +'\n' Text.Whitespace + +' ' Text +'if' Keyword +' ' Text +'has_sse4' Name +'(' Punctuation +')' Punctuation +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'cpu_features' Name +' ' Text +'+' Operator +'=' Operator +' ' Text +'"' Literal.String.Double +'sse4' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +' ' Text +'if' Keyword +' ' Text +'has_avx' Name +'(' Punctuation +')' Punctuation +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'cpu_features' Name +' ' Text +'+' Operator +'=' Operator +' ' Text +'"' Literal.String.Double +' avx' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +' ' Text +'if' Keyword +' ' Text +'has_avx2' Name +'(' Punctuation +')' Punctuation +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'cpu_features' Name +' ' Text +'+' Operator +'=' Operator +' ' Text +'"' Literal.String.Double +' avx2' Literal.String.Double +'"' Literal.String.Double +'\n' Text.Whitespace + +'\n' Text.Whitespace + +' ' Text +'var' Keyword +' ' Text +'info' Name.Variable +' ' Text +'=' Operator +' ' Text +'(' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'"""System information:\n OS : """' Literal.String.Doc +'\n' Text.Whitespace + +' ' Text +'+' Operator +' ' Text +'os' Name +'\n' Text.Whitespace + +' ' Text +'+' Operator +' ' Text +'"""' Literal.String.Double +'\n' Literal.String.Double + +' CPU : ' Literal.String.Double +'"""' Literal.String.Double +'\n' Text.Whitespace + +' ' Text +'+' Operator +' ' Text +'cpu' Name +'\n' Text.Whitespace + +' ' Text +'+' Operator +' ' Text +'"""' Literal.String.Double +'\n' Literal.String.Double + +' Arch : ' Literal.String.Double +'"""' Literal.String.Double +'\n' Text.Whitespace + +' ' Text +'+' Operator +' ' Text +'arch' Name +'\n' Text.Whitespace + +' ' Text +'+' Operator +' ' Text +'"""' Literal.String.Double +'\n' Literal.String.Double + +' Physical Cores : ' Literal.String.Double +'"""' Literal.String.Double +'\n' Text.Whitespace + +' ' Text +'+' Operator +' ' Text +'num_physical_cores' Name +'(' Punctuation +')' Punctuation +'\n' Text.Whitespace + +' ' Text +'+' Operator +' ' Text +'"""' Literal.String.Double +'\n' Literal.String.Double + +' Logical Cores : ' Literal.String.Double +'"""' Literal.String.Double +'\n' Text.Whitespace + +' ' Text +'+' Operator +' ' Text +'num_logical_cores' Name +'(' Punctuation +')' Punctuation +'\n' Text.Whitespace + +' ' Text +'+' Operator +' ' Text +'"""' Literal.String.Double +'\n' Literal.String.Double + +' CPU Features : ' Literal.String.Double +'"""' Literal.String.Double +'\n' Text.Whitespace + +' ' Text +'+' Operator +' ' Text +'cpu_features' Name +'\n' Text.Whitespace + +' ' Text +')' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +' ' Text +'return' Keyword +' ' Text +'info' Name +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'trait' Keyword +' ' Text +'Greet' Name.Struct +':' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'"""\n A trait example.\n """' Literal.String.Doc +'\n' Text.Whitespace + +'\n' Text.Whitespace + +' ' Text +'fn' Keyword +' ' Text +'say_hello' Name.Function +'(' Punctuation +'self' Name.Builtin.Pseudo +')' Punctuation +' ' Text +'-' Operator +'>' Operator +' ' Text +'None' Keyword.Constant +':' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'"""Says hello."""' Literal.String.Doc +'\n' Text.Whitespace + +' ' Text +'.' Operator +'.' Operator +'.' Operator +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'@value' Name.Decorator +'\n' Text.Whitespace + +'struct' Keyword +' ' Text +'Person' Name.Struct +'(' Punctuation +'Greet' Name +')' Punctuation +':' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'"""\n A person.\n """' Literal.String.Doc +'\n' Text.Whitespace + +'\n' Text.Whitespace + +' ' Text +'var' Keyword +' ' Text +'name' Name.Variable +':' Punctuation +' ' Text +'String' Name.Builtin +'\n' Text.Whitespace + +'\n' Text.Whitespace + +' ' Text +'fn' Keyword +' ' Text +'__init__' Name.Function.Magic +'(' Punctuation +'inout' Name +' ' Text +'self' Name.Builtin.Pseudo +',' Punctuation +' ' Text +'name' Name +':' Punctuation +' ' Text +'String' Name.Builtin +')' Punctuation +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'self' Name.Builtin.Pseudo +'.' Operator +'name' Name +' ' Text +'=' Operator +' ' Text +'name' Name +'\n' Text.Whitespace + +'\n' Text.Whitespace + +' ' Text +'fn' Keyword +' ' Text +'__str__' Name.Function.Magic +'(' Punctuation +'self' Name.Builtin.Pseudo +')' Punctuation +' ' Text +'-' Operator +'>' Operator +' ' Text +'String' Name.Builtin +':' Punctuation +'\n' Text.Whitespace + +' ' Text.Whitespace +'"""\n Overridden __str__ method\n """' Literal.String.Doc +'\n' Text.Whitespace + +' ' Text +'return' Keyword +' ' Text +'"' Literal.String.Double +'Person named ' Literal.String.Double +'"' Literal.String.Double +' ' Text +'+' Operator +' ' Text +'self' Name.Builtin.Pseudo +'.' Operator +'name' Name +'\n' Text.Whitespace + +'\n' Text.Whitespace + +' ' Text +'fn' Keyword +' ' Text +'say_hello' Name.Function +'(' Punctuation +'self' Name.Builtin.Pseudo +')' Punctuation +' ' Text +'-' Operator +'>' Operator +' ' Text +'None' Keyword.Constant +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'print' Name.Builtin +'(' Punctuation +'self' Name.Builtin.Pseudo +'.' Operator +'name' Name +',' Punctuation +' ' Text +'"' Literal.String.Double +'says hello!' Literal.String.Double +'"' Literal.String.Double +')' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'\n' Text.Whitespace + +'fn' Keyword +' ' Text +'main' Name.Function +'(' Punctuation +')' Punctuation +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'for' Keyword +' ' Text +'i' Name +' ' Text +'in' Operator.Word +' ' Text +'range' Name.Builtin +'(' Punctuation +'3' Literal.Number.Integer +')' Punctuation +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'var' Keyword +' ' Text +'pers' Name.Variable +' ' Text +'=' Operator +' ' Text +'Person' Name +'(' Punctuation +'"' Literal.String.Double +'Per_' Literal.String.Double +'"' Literal.String.Double +' ' Text +'+' Operator +' ' Text +'String' Name.Builtin +'(' Punctuation +'i' Name +')' Punctuation +')' Punctuation +'\n' Text.Whitespace + +' ' Text +'print' Name.Builtin +'(' Punctuation +'pers' Name +')' Punctuation +'\n' Text.Whitespace + +' ' Text +'pers' Name +'.' Operator +'say_hello' Name +'(' Punctuation +')' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +' ' Text +'generic_func' Name +'(' Punctuation +')' Punctuation +' ' Text +"# prints 'woof 3'" Comment.Single +'\n' Text.Whitespace + +' ' Text +'generic_func' Name +'[' Punctuation +'5' Literal.Number.Integer +']' Punctuation +'(' Punctuation +')' Punctuation +' ' Text +"# prints 'woof 5'" Comment.Single +'\n' Text.Whitespace + +' ' Text +'generic_func' Name +'[' Punctuation +'7' Literal.Number.Integer +',' Punctuation +' ' Text +'"' Literal.String.Double +'meow' Literal.String.Double +'"' Literal.String.Double +']' Punctuation +'(' Punctuation +')' Punctuation +' ' Text +"# prints 'meow 7'" Comment.Single +'\n' Text.Whitespace + +'\n' Text.Whitespace + +' ' Text +'try' Keyword +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'func' Name +'(' Punctuation +')' Punctuation +'\n' Text.Whitespace + +' ' Text +'strict_func' Name +'(' Punctuation +')' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +' ' Text +'raise' Keyword +' ' Text +'Error' Name.Builtin +'(' Punctuation +'"' Literal.String.Double +'fail' Literal.String.Double +'"' Literal.String.Double +')' Punctuation +'\n' Text.Whitespace + +' ' Text +'except' Keyword +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'print' Name.Builtin +'(' Punctuation +'"' Literal.String.Double +'error was raised' Literal.String.Double +'"' Literal.String.Double +')' Punctuation +'\n' Text.Whitespace + +'\n' Text.Whitespace + +' ' Text +'print' Name.Builtin +'(' Punctuation +'get_sys_info' Name +'(' Punctuation +')' Punctuation +')' Punctuation +'\n' Text.Whitespace diff --git a/tests/snippets/mojo/test_floats.txt b/tests/snippets/mojo/test_floats.txt new file mode 100644 index 0000000000..3ae9dc96e4 --- /dev/null +++ b/tests/snippets/mojo/test_floats.txt @@ -0,0 +1,75 @@ +---input--- +123 -11 0 -0 0.5 .5 1. -0.5 +0.5 -.5 -1. 2e1 -2e1 2e -2e +2e e.3 -e.3 11.2e-3 -11.2e-3 5_6 5__6 _5 6_ 5.6_7 5.67_ + +---tokens--- +'123' Literal.Number.Integer +' ' Text +'-' Operator +'11' Literal.Number.Integer +' ' Text +'0' Literal.Number.Integer +' ' Text +'-' Operator +'0' Literal.Number.Integer +' ' Text +'0.5' Literal.Number.Float +' ' Text +'.5' Literal.Number.Float +' ' Text +'1.' Literal.Number.Float +' ' Text +'-' Operator +'0.5' Literal.Number.Float +' ' Text +'+' Operator +'0.5' Literal.Number.Float +' ' Text +'-' Operator +'.5' Literal.Number.Float +' ' Text +'-' Operator +'1.' Literal.Number.Float +' ' Text +'2e1' Literal.Number.Float +' ' Text +'-' Operator +'2e1' Literal.Number.Float +' ' Text +'2' Literal.Number.Integer +'e' Name +' ' Text +'-' Operator +'2' Literal.Number.Integer +'e' Name +' ' Text +'+' Operator +'2' Literal.Number.Integer +'e' Name +' ' Text +'e' Name +'.3' Literal.Number.Float +' ' Text +'-' Operator +'e' Name +'.3' Literal.Number.Float +' ' Text +'11.2e-3' Literal.Number.Float +' ' Text +'-' Operator +'11.2e-3' Literal.Number.Float +' ' Text +'5_6' Literal.Number.Integer +' ' Text +'5' Literal.Number.Integer +'__6' Name +' ' Text +'_5' Name +' ' Text +'6' Literal.Number.Integer +'_' Name +' ' Text +'5.6_7' Literal.Number.Float +' ' Text +'5.67' Literal.Number.Float +'_' Name +'\n' Text.Whitespace diff --git a/tests/snippets/mojo/test_kw.txt b/tests/snippets/mojo/test_kw.txt new file mode 100644 index 0000000000..d4fb3ec519 --- /dev/null +++ b/tests/snippets/mojo/test_kw.txt @@ -0,0 +1,35 @@ +---input--- +fn +def +struct +trait +class +if +elif +else +with + +---tokens--- +'fn' Keyword +'\n' Text + +'def' Name.Function +'\n' Text.Whitespace + +'struct' Keyword +'\n' Text + +'trait' Name.Struct +'\n' Text.Whitespace + +'class' Keyword +'\n' Text + +'if' Name.Class +'\n' Text.Whitespace + +'elif' Keyword +'\n' Text.Whitespace + +'else' Keyword +'\n' Text.Whitespace diff --git a/tests/snippets/mojo/test_needs_name.txt b/tests/snippets/mojo/test_needs_name.txt new file mode 100644 index 0000000000..f121da03f2 --- /dev/null +++ b/tests/snippets/mojo/test_needs_name.txt @@ -0,0 +1,55 @@ +# Tests that '@' is recognized as an Operator + +---input--- +S = (H @ beta - r).T @ inv(H @ V @ H.T) @ (H @ beta - r) + +---tokens--- +'S' Name +' ' Text +'=' Operator +' ' Text +'(' Punctuation +'H' Name +' ' Text +'@' Operator +' ' Text +'beta' Name +' ' Text +'-' Operator +' ' Text +'r' Name +')' Punctuation +'.' Operator +'T' Name +' ' Text +'@' Operator +' ' Text +'inv' Name +'(' Punctuation +'H' Name +' ' Text +'@' Operator +' ' Text +'V' Name +' ' Text +'@' Operator +' ' Text +'H' Name +'.' Operator +'T' Name +')' Punctuation +' ' Text +'@' Operator +' ' Text +'(' Punctuation +'H' Name +' ' Text +'@' Operator +' ' Text +'beta' Name +' ' Text +'-' Operator +' ' Text +'r' Name +')' Punctuation +'\n' Text.Whitespace diff --git a/tests/snippets/mojo/test_soft_kwds.txt b/tests/snippets/mojo/test_soft_kwds.txt new file mode 100644 index 0000000000..af9c8954ad --- /dev/null +++ b/tests/snippets/mojo/test_soft_kwds.txt @@ -0,0 +1,67 @@ +---input--- +match spam: + case Some(x): + print(f"found {x}") + case None: + print("found nothing") + case _: + assert False + +---tokens--- +'match' Keyword +' ' Text +'spam' Name +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'case' Keyword +' ' Text +'Some' Name +'(' Punctuation +'x' Name +')' Punctuation +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'print' Name.Builtin +'(' Punctuation +'f' Literal.String.Affix +'"' Literal.String.Double +'found ' Literal.String.Double +'{' Literal.String.Interpol +'x' Name +'}' Literal.String.Interpol +'"' Literal.String.Double +')' Punctuation +'\n' Text.Whitespace + +' ' Text +'case' Keyword +' ' Text +'None' Keyword.Constant +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'print' Name.Builtin +'(' Punctuation +'"' Literal.String.Double +'found nothing' Literal.String.Double +'"' Literal.String.Double +')' Punctuation +'\n' Text.Whitespace + +' ' Text +'case' Keyword +' ' Text.Whitespace +'_' Keyword +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'assert' Keyword +' ' Text +'False' Keyword.Constant +'\n' Text.Whitespace From 2737da1cf076b1e90cb7bb0baf4e2a98ce86d173 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20R=C3=B6mer?= <16097755+hroemer@users.noreply.github.com> Date: Sun, 14 Apr 2024 11:53:16 +0200 Subject: [PATCH 4/7] #2515: updating mojo snippets --- tests/snippets/mojo/test_kw.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/snippets/mojo/test_kw.txt b/tests/snippets/mojo/test_kw.txt index d4fb3ec519..ed1eaf14fd 100644 --- a/tests/snippets/mojo/test_kw.txt +++ b/tests/snippets/mojo/test_kw.txt @@ -33,3 +33,6 @@ with 'else' Keyword '\n' Text.Whitespace + +'with' Keyword +'\n' Text.Whitespace From 2fd5fee7b337086d5826018c75e891c2b0733664 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20R=C3=B6mer?= <16097755+hroemer@users.noreply.github.com> Date: Mon, 15 Apr 2024 18:55:28 +0200 Subject: [PATCH 5/7] #2515: ruff fixes --- pygments/lexers/mojo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/mojo.py b/pygments/lexers/mojo.py index 1a12561fba..3698e02754 100644 --- a/pygments/lexers/mojo.py +++ b/pygments/lexers/mojo.py @@ -23,7 +23,7 @@ ) from pygments.token import ( Comment, - Error, + # Error, Keyword, Name, Number, From b1b0942f8b13bf910df5937e6db4223485651c72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hannes=20R=C3=B6mer?= <16097755+hroemer@users.noreply.github.com> Date: Mon, 15 Apr 2024 18:58:39 +0200 Subject: [PATCH 6/7] #2515: ruff header fixes --- pygments/lexers/mojo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pygments/lexers/mojo.py b/pygments/lexers/mojo.py index 3698e02754..2532279485 100644 --- a/pygments/lexers/mojo.py +++ b/pygments/lexers/mojo.py @@ -1,10 +1,10 @@ """ pygments.lexers.mojo - ~~~~~~~~~~~~~~~~~~~~~~ + ~~~~~~~~~~~~~~~~~~~~ Lexers for Mojo and related languages. - :copyright: Copyright 2024-2024 by the Pygments team, see AUTHORS. + :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ From 35bda7a56928358fbff9eea6c38b6ffe6c56f11b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matth=C3=A4us=20G=2E=20Chajdas?= Date: Sat, 27 Apr 2024 14:41:40 +0200 Subject: [PATCH 7/7] Fix whitespace issues. --- pygments/lexers/mojo.py | 41 +- tests/examplefiles/mojo/sample.mojo.output | 699 ++++++++------------- 2 files changed, 276 insertions(+), 464 deletions(-) diff --git a/pygments/lexers/mojo.py b/pygments/lexers/mojo.py index 2532279485..0191dce0aa 100644 --- a/pygments/lexers/mojo.py +++ b/pygments/lexers/mojo.py @@ -99,7 +99,7 @@ def fstring_rules(ttype): tokens = { "root": [ - (r"\n", Whitespace), + (r"\s+", Whitespace), ( r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")', bygroups(Whitespace, String.Affix, String.Doc), @@ -110,27 +110,30 @@ def fstring_rules(ttype): ), (r"\A#!.+$", Comment.Hashbang), (r"#.*$", Comment.Single), - (r"\\\n", Text), - (r"\\", Text), + (r"\\\n", Whitespace), + (r"\\", Whitespace), include("keywords"), include("soft-keywords"), + # In the original PR, all the below here used ((?:\s|\\\s)+) to + # designate whitespace, but I can't find any example of this being + # needed in the example file, so we're replacing it with `\s+`. ( - r"(alias)((?:\s|\\\s)+)", - bygroups(Keyword, Text), + r"(alias)(\s+)", + bygroups(Keyword, Whitespace), "varname", # TODO varname the right fit? ), - (r"(var)((?:\s|\\\s)+)", bygroups(Keyword, Text), "varname"), - (r"(def)((?:\s|\\\s)+)", bygroups(Keyword, Text), "funcname"), - (r"(fn)((?:\s|\\\s)+)", bygroups(Keyword, Text), "funcname"), + (r"(var)(\s+)", bygroups(Keyword, Whitespace), "varname"), + (r"(def)(\s+)", bygroups(Keyword, Whitespace), "funcname"), + (r"(fn)(\s+)", bygroups(Keyword, Whitespace), "funcname"), ( - r"(class)((?:\s|\\\s)+)", - bygroups(Keyword, Text), + r"(class)(\s+)", + bygroups(Keyword, Whitespace), "classname", ), # not implemented yet - (r"(struct)((?:\s|\\\s)+)", bygroups(Keyword, Text), "structname"), - (r"(trait)((?:\s|\\\s)+)", bygroups(Keyword, Text), "structname"), - (r"(from)((?:\s|\\\s)+)", bygroups(Keyword.Namespace, Text), "fromimport"), - (r"(import)((?:\s|\\\s)+)", bygroups(Keyword.Namespace, Text), "import"), + (r"(struct)(\s+)", bygroups(Keyword, Whitespace), "structname"), + (r"(trait)(\s+)", bygroups(Keyword, Whitespace), "structname"), + (r"(from)(\s+)", bygroups(Keyword.Namespace, Whitespace), "fromimport"), + (r"(import)(\s+)", bygroups(Keyword.Namespace, Whitespace), "import"), include("expr"), ], "expr": [ @@ -226,7 +229,7 @@ def fstring_rules(ttype): (r"[^\S\n]+", Text), include("numbers"), (r"!=|==|<<|>>|:=|[-~+/*%=<>&^|.]", Operator), - (r"[]{}:(),;[]", Punctuation), + (r"([]{}:\(\),;[])+", Punctuation), (r"(in|is|and|or|not)\b", Operator.Word), include("expr-keywords"), include("builtins"), @@ -329,7 +332,7 @@ def fstring_rules(ttype): # pattern matching (but None/True/False is ok) r"|".join(k for k in keyword.kwlist if k[0].islower()) + r")\b))", - bygroups(Text, Keyword), + bygroups(Whitespace, Keyword), "soft-keywords-inner", ), ], @@ -619,14 +622,14 @@ def fstring_rules(ttype): (uni_name, Name.Struct, "#pop"), ], "import": [ - (r"(\s+)(as)(\s+)", bygroups(Text, Keyword, Text)), + (r"(\s+)(as)(\s+)", bygroups(Whitespace, Keyword, Whitespace)), (r"\.", Name.Namespace), (uni_name, Name.Namespace), - (r"(\s*)(,)(\s*)", bygroups(Text, Operator, Text)), + (r"(\s*)(,)(\s*)", bygroups(Whitespace, Operator, Whitespace)), default("#pop"), # all else: go back ], "fromimport": [ - (r"(\s+)(import)\b", bygroups(Text, Keyword.Namespace), "#pop"), + (r"(\s+)(import)\b", bygroups(Whitespace, Keyword.Namespace), "#pop"), (r"\.", Name.Namespace), # if None occurs here, it's "raise x from None", since None can # never be a module name diff --git a/tests/examplefiles/mojo/sample.mojo.output b/tests/examplefiles/mojo/sample.mojo.output index dd30b6c8c1..f41913d10d 100644 --- a/tests/examplefiles/mojo/sample.mojo.output +++ b/tests/examplefiles/mojo/sample.mojo.output @@ -1,45 +1,31 @@ 'from' Keyword.Namespace -' ' Text +' ' Text.Whitespace 'sys' Name.Namespace '.' Name.Namespace 'info' Name.Namespace -' ' Text +' ' Text.Whitespace 'import' Keyword.Namespace -' ' Text +' ' Text.Whitespace '(' Punctuation -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'os_is_linux' Name ',' Punctuation -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'os_is_macos' Name ',' Punctuation -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'has_sse4' Name ',' Punctuation -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'has_avx' Name ',' Punctuation -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'has_avx2' Name ',' Punctuation -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'num_physical_cores' Name ',' Punctuation -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'num_logical_cores' Name ',' Punctuation '\n' Text.Whitespace @@ -48,773 +34,596 @@ '\n' Text.Whitespace 'from' Keyword.Namespace -' ' Text +' ' Text.Whitespace 'sys' Name.Namespace '.' Name.Namespace 'info' Name.Namespace -' ' Text +' ' Text.Whitespace 'import' Keyword.Namespace -' ' Text +' ' Text.Whitespace '_current_cpu' Name ',' Punctuation -' ' Text +' ' Text.Whitespace '_current_target' Name ',' Punctuation -' ' Text +' ' Text.Whitespace '_triple_attr' Name -'\n' Text.Whitespace - -'\n' Text.Whitespace - -'\n' Text.Whitespace +'\n\n\n' Text.Whitespace 'def' Keyword -' ' Text +' ' Text.Whitespace 'func' Name.Function -'(' Punctuation -')' Punctuation -' ' Text +'()' Punctuation +' ' Text.Whitespace '-' Operator '>' Operator -' ' Text +' ' Text.Whitespace 'None' Keyword.Constant ':' Punctuation -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'variable' Name -' ' Text +' ' Text.Whitespace '=' Operator -' ' Text +' ' Text.Whitespace '"' Literal.String.Double 'implicit variable' Literal.String.Double '"' Literal.String.Double -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'print' Name.Builtin '(' Punctuation '"' Literal.String.Double 'def function' Literal.String.Double '"' Literal.String.Double ',' Punctuation -' ' Text +' ' Text.Whitespace 'variable' Name ')' Punctuation -'\n' Text.Whitespace - -'\n' Text.Whitespace - -'\n' Text.Whitespace +'\n\n\n' Text.Whitespace 'fn' Keyword -' ' Text +' ' Text.Whitespace 'strict_func' Name.Function -'(' Punctuation -')' Punctuation -':' Punctuation -'\n' Text.Whitespace - -' ' Text +'():' Punctuation +'\n ' Text.Whitespace 'var' Keyword -' ' Text +' ' Text.Whitespace 'variable' Name.Variable -' ' Text +' ' Text.Whitespace '=' Operator -' ' Text +' ' Text.Whitespace '"' Literal.String.Double 'explicit variable' Literal.String.Double '"' Literal.String.Double -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'print' Name.Builtin '(' Punctuation '"' Literal.String.Double 'fn function' Literal.String.Double '"' Literal.String.Double ',' Punctuation -' ' Text +' ' Text.Whitespace 'variable' Name ')' Punctuation -'\n' Text.Whitespace - -'\n' Text.Whitespace - -'\n' Text.Whitespace +'\n\n\n' Text.Whitespace 'fn' Keyword -' ' Text +' ' Text.Whitespace 'generic_func' Name.Function '[' Punctuation 'a' Name ':' Punctuation -' ' Text +' ' Text.Whitespace 'Int' Name.Builtin -' ' Text +' ' Text.Whitespace '=' Operator -' ' Text +' ' Text.Whitespace '3' Literal.Number.Integer ',' Punctuation -' ' Text +' ' Text.Whitespace 'msg' Name ':' Punctuation -' ' Text +' ' Text.Whitespace 'StringLiteral' Name -' ' Text +' ' Text.Whitespace '=' Operator -' ' Text +' ' Text.Whitespace '"' Literal.String.Double 'woof' Literal.String.Double '"' Literal.String.Double -']' Punctuation -'(' Punctuation -')' Punctuation -':' Punctuation -'\n' Text.Whitespace +']():' Punctuation +'\n ' Text.Whitespace +'"""' Literal.String.Double +'\n' Literal.String.Double -' ' Text.Whitespace -'"""\n A generic function.\n """' Literal.String.Doc -'\n' Text.Whitespace +' A generic function.' Literal.String.Double +'\n' Literal.String.Double -' ' Text +' ' Literal.String.Double +'"""' Literal.String.Double +'\n ' Text.Whitespace 'print' Name.Builtin '(' Punctuation 'msg' Name ',' Punctuation -' ' Text +' ' Text.Whitespace 'a' Name ')' Punctuation -'\n' Text.Whitespace - -'\n' Text.Whitespace - -'\n' Text.Whitespace +'\n\n\n' Text.Whitespace 'fn' Keyword -' ' Text +' ' Text.Whitespace 'get_sys_info' Name.Function -'(' Punctuation -')' Punctuation -' ' Text +'()' Punctuation +' ' Text.Whitespace '-' Operator '>' Operator -' ' Text +' ' Text.Whitespace 'String' Name.Builtin ':' Punctuation -'\n' Text.Whitespace +'\n ' Text.Whitespace +'"""' Literal.String.Double +'\n' Literal.String.Double -' ' Text.Whitespace -'"""\n Retrieves system information (based upon: https://github.com/modularml/mojo/blob/main/examples/deviceinfo.mojo).\n """' Literal.String.Doc -'\n' Text.Whitespace +' Retrieves system information (based upon: https://github.com/modularml/mojo/blob/main/examples/deviceinfo.mojo).' Literal.String.Double +'\n' Literal.String.Double -' ' Text +' ' Literal.String.Double +'"""' Literal.String.Double +'\n ' Text.Whitespace 'var' Keyword -' ' Text +' ' Text.Whitespace 'os' Name.Variable -' ' Text +' ' Text.Whitespace '=' Operator -' ' Text +' ' Text.Whitespace '"' Literal.String.Double '"' Literal.String.Double -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'if' Keyword -' ' Text +' ' Text.Whitespace 'os_is_linux' Name -'(' Punctuation -')' Punctuation -':' Punctuation -'\n' Text.Whitespace - -' ' Text +'():' Punctuation +'\n ' Text.Whitespace 'os' Name -' ' Text +' ' Text.Whitespace '=' Operator -' ' Text +' ' Text.Whitespace '"' Literal.String.Double 'linux' Literal.String.Double '"' Literal.String.Double -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'elif' Keyword -' ' Text +' ' Text.Whitespace 'os_is_macos' Name -'(' Punctuation -')' Punctuation -':' Punctuation -'\n' Text.Whitespace - -' ' Text +'():' Punctuation +'\n ' Text.Whitespace 'os' Name -' ' Text +' ' Text.Whitespace '=' Operator -' ' Text +' ' Text.Whitespace '"' Literal.String.Double 'macOS' Literal.String.Double '"' Literal.String.Double -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'else' Keyword ':' Punctuation -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'os' Name -' ' Text +' ' Text.Whitespace '=' Operator -' ' Text +' ' Text.Whitespace '"' Literal.String.Double 'windows' Literal.String.Double '"' Literal.String.Double -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'var' Keyword -' ' Text +' ' Text.Whitespace 'cpu' Name.Variable -' ' Text +' ' Text.Whitespace '=' Operator -' ' Text +' ' Text.Whitespace 'String' Name.Builtin '(' Punctuation '_current_cpu' Name -'(' Punctuation -')' Punctuation -')' Punctuation -'\n' Text.Whitespace - -' ' Text +'())' Punctuation +'\n ' Text.Whitespace 'var' Keyword -' ' Text +' ' Text.Whitespace 'arch' Name.Variable -' ' Text +' ' Text.Whitespace '=' Operator -' ' Text +' ' Text.Whitespace 'String' Name.Builtin '(' Punctuation '_triple_attr' Name -'(' Punctuation -')' Punctuation -')' Punctuation -'\n' Text.Whitespace - -' ' Text +'())' Punctuation +'\n ' Text.Whitespace 'var' Keyword -' ' Text +' ' Text.Whitespace 'cpu_features' Name.Variable -' ' Text +' ' Text.Whitespace '=' Operator -' ' Text +' ' Text.Whitespace 'String' Name.Builtin '(' Punctuation '"' Literal.String.Double '"' Literal.String.Double ')' Punctuation -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'if' Keyword -' ' Text +' ' Text.Whitespace 'has_sse4' Name -'(' Punctuation -')' Punctuation -':' Punctuation -'\n' Text.Whitespace - -' ' Text +'():' Punctuation +'\n ' Text.Whitespace 'cpu_features' Name -' ' Text +' ' Text.Whitespace '+' Operator '=' Operator -' ' Text +' ' Text.Whitespace '"' Literal.String.Double 'sse4' Literal.String.Double '"' Literal.String.Double -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'if' Keyword -' ' Text +' ' Text.Whitespace 'has_avx' Name -'(' Punctuation -')' Punctuation -':' Punctuation -'\n' Text.Whitespace - -' ' Text +'():' Punctuation +'\n ' Text.Whitespace 'cpu_features' Name -' ' Text +' ' Text.Whitespace '+' Operator '=' Operator -' ' Text +' ' Text.Whitespace '"' Literal.String.Double ' avx' Literal.String.Double '"' Literal.String.Double -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'if' Keyword -' ' Text +' ' Text.Whitespace 'has_avx2' Name -'(' Punctuation -')' Punctuation -':' Punctuation -'\n' Text.Whitespace - -' ' Text +'():' Punctuation +'\n ' Text.Whitespace 'cpu_features' Name -' ' Text +' ' Text.Whitespace '+' Operator '=' Operator -' ' Text +' ' Text.Whitespace '"' Literal.String.Double ' avx2' Literal.String.Double '"' Literal.String.Double -'\n' Text.Whitespace - -'\n' Text.Whitespace - -' ' Text +'\n\n ' Text.Whitespace 'var' Keyword -' ' Text +' ' Text.Whitespace 'info' Name.Variable -' ' Text +' ' Text.Whitespace '=' Operator -' ' Text +' ' Text.Whitespace '(' Punctuation -'\n' Text.Whitespace - -' ' Text.Whitespace -'"""System information:\n OS : """' Literal.String.Doc -'\n' Text.Whitespace +'\n ' Text.Whitespace +'"""' Literal.String.Double +'System information:' Literal.String.Double +'\n' Literal.String.Double -' ' Text +' OS : ' Literal.String.Double +'"""' Literal.String.Double +'\n ' Text.Whitespace '+' Operator -' ' Text +' ' Text.Whitespace 'os' Name -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace '+' Operator -' ' Text +' ' Text.Whitespace '"""' Literal.String.Double '\n' Literal.String.Double ' CPU : ' Literal.String.Double '"""' Literal.String.Double -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace '+' Operator -' ' Text +' ' Text.Whitespace 'cpu' Name -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace '+' Operator -' ' Text +' ' Text.Whitespace '"""' Literal.String.Double '\n' Literal.String.Double ' Arch : ' Literal.String.Double '"""' Literal.String.Double -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace '+' Operator -' ' Text +' ' Text.Whitespace 'arch' Name -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace '+' Operator -' ' Text +' ' Text.Whitespace '"""' Literal.String.Double '\n' Literal.String.Double ' Physical Cores : ' Literal.String.Double '"""' Literal.String.Double -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace '+' Operator -' ' Text +' ' Text.Whitespace 'num_physical_cores' Name -'(' Punctuation -')' Punctuation -'\n' Text.Whitespace - -' ' Text +'()' Punctuation +'\n ' Text.Whitespace '+' Operator -' ' Text +' ' Text.Whitespace '"""' Literal.String.Double '\n' Literal.String.Double ' Logical Cores : ' Literal.String.Double '"""' Literal.String.Double -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace '+' Operator -' ' Text +' ' Text.Whitespace 'num_logical_cores' Name -'(' Punctuation -')' Punctuation -'\n' Text.Whitespace - -' ' Text +'()' Punctuation +'\n ' Text.Whitespace '+' Operator -' ' Text +' ' Text.Whitespace '"""' Literal.String.Double '\n' Literal.String.Double ' CPU Features : ' Literal.String.Double '"""' Literal.String.Double -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace '+' Operator -' ' Text +' ' Text.Whitespace 'cpu_features' Name -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace ')' Punctuation -'\n' Text.Whitespace - -'\n' Text.Whitespace - -' ' Text +'\n\n ' Text.Whitespace 'return' Keyword -' ' Text +' ' Text.Whitespace 'info' Name -'\n' Text.Whitespace - -'\n' Text.Whitespace - -'\n' Text.Whitespace +'\n\n\n' Text.Whitespace 'trait' Keyword -' ' Text +' ' Text.Whitespace 'Greet' Name.Struct ':' Punctuation -'\n' Text.Whitespace - -' ' Text.Whitespace -'"""\n A trait example.\n """' Literal.String.Doc -'\n' Text.Whitespace +'\n ' Text.Whitespace +'"""' Literal.String.Double +'\n' Literal.String.Double -'\n' Text.Whitespace +' A trait example.' Literal.String.Double +'\n' Literal.String.Double -' ' Text +' ' Literal.String.Double +'"""' Literal.String.Double +'\n\n ' Text.Whitespace 'fn' Keyword -' ' Text +' ' Text.Whitespace 'say_hello' Name.Function '(' Punctuation 'self' Name.Builtin.Pseudo ')' Punctuation -' ' Text +' ' Text.Whitespace '-' Operator '>' Operator -' ' Text +' ' Text.Whitespace 'None' Keyword.Constant ':' Punctuation -'\n' Text.Whitespace - -' ' Text.Whitespace -'"""Says hello."""' Literal.String.Doc -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace +'"""' Literal.String.Double +'Says hello.' Literal.String.Double +'"""' Literal.String.Double +'\n ' Text.Whitespace '.' Operator '.' Operator '.' Operator -'\n' Text.Whitespace - -'\n' Text.Whitespace - -'\n' Text.Whitespace +'\n\n\n' Text.Whitespace '@value' Name.Decorator '\n' Text.Whitespace 'struct' Keyword -' ' Text +' ' Text.Whitespace 'Person' Name.Struct '(' Punctuation 'Greet' Name -')' Punctuation -':' Punctuation -'\n' Text.Whitespace - -' ' Text.Whitespace -'"""\n A person.\n """' Literal.String.Doc -'\n' Text.Whitespace +'):' Punctuation +'\n ' Text.Whitespace +'"""' Literal.String.Double +'\n' Literal.String.Double -'\n' Text.Whitespace +' A person.' Literal.String.Double +'\n' Literal.String.Double -' ' Text +' ' Literal.String.Double +'"""' Literal.String.Double +'\n\n ' Text.Whitespace 'var' Keyword -' ' Text +' ' Text.Whitespace 'name' Name.Variable ':' Punctuation -' ' Text +' ' Text.Whitespace 'String' Name.Builtin -'\n' Text.Whitespace - -'\n' Text.Whitespace - -' ' Text +'\n\n ' Text.Whitespace 'fn' Keyword -' ' Text +' ' Text.Whitespace '__init__' Name.Function.Magic '(' Punctuation 'inout' Name -' ' Text +' ' Text.Whitespace 'self' Name.Builtin.Pseudo ',' Punctuation -' ' Text +' ' Text.Whitespace 'name' Name ':' Punctuation -' ' Text +' ' Text.Whitespace 'String' Name.Builtin -')' Punctuation -':' Punctuation -'\n' Text.Whitespace - -' ' Text +'):' Punctuation +'\n ' Text.Whitespace 'self' Name.Builtin.Pseudo '.' Operator 'name' Name -' ' Text +' ' Text.Whitespace '=' Operator -' ' Text +' ' Text.Whitespace 'name' Name -'\n' Text.Whitespace - -'\n' Text.Whitespace - -' ' Text +'\n\n ' Text.Whitespace 'fn' Keyword -' ' Text +' ' Text.Whitespace '__str__' Name.Function.Magic '(' Punctuation 'self' Name.Builtin.Pseudo ')' Punctuation -' ' Text +' ' Text.Whitespace '-' Operator '>' Operator -' ' Text +' ' Text.Whitespace 'String' Name.Builtin ':' Punctuation -'\n' Text.Whitespace +'\n ' Text.Whitespace +'"""' Literal.String.Double +'\n' Literal.String.Double -' ' Text.Whitespace -'"""\n Overridden __str__ method\n """' Literal.String.Doc -'\n' Text.Whitespace +' Overridden __str__ method' Literal.String.Double +'\n' Literal.String.Double -' ' Text +' ' Literal.String.Double +'"""' Literal.String.Double +'\n ' Text.Whitespace 'return' Keyword -' ' Text +' ' Text.Whitespace '"' Literal.String.Double 'Person named ' Literal.String.Double '"' Literal.String.Double -' ' Text +' ' Text.Whitespace '+' Operator -' ' Text +' ' Text.Whitespace 'self' Name.Builtin.Pseudo '.' Operator 'name' Name -'\n' Text.Whitespace - -'\n' Text.Whitespace - -' ' Text +'\n\n ' Text.Whitespace 'fn' Keyword -' ' Text +' ' Text.Whitespace 'say_hello' Name.Function '(' Punctuation 'self' Name.Builtin.Pseudo ')' Punctuation -' ' Text +' ' Text.Whitespace '-' Operator '>' Operator -' ' Text +' ' Text.Whitespace 'None' Keyword.Constant ':' Punctuation -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'print' Name.Builtin '(' Punctuation 'self' Name.Builtin.Pseudo '.' Operator 'name' Name ',' Punctuation -' ' Text +' ' Text.Whitespace '"' Literal.String.Double 'says hello!' Literal.String.Double '"' Literal.String.Double ')' Punctuation -'\n' Text.Whitespace - -'\n' Text.Whitespace - -'\n' Text.Whitespace +'\n\n\n' Text.Whitespace 'fn' Keyword -' ' Text +' ' Text.Whitespace 'main' Name.Function -'(' Punctuation -')' Punctuation -':' Punctuation -'\n' Text.Whitespace - -' ' Text +'():' Punctuation +'\n ' Text.Whitespace 'for' Keyword -' ' Text +' ' Text.Whitespace 'i' Name -' ' Text +' ' Text.Whitespace 'in' Operator.Word -' ' Text +' ' Text.Whitespace 'range' Name.Builtin '(' Punctuation '3' Literal.Number.Integer -')' Punctuation -':' Punctuation -'\n' Text.Whitespace - -' ' Text +'):' Punctuation +'\n ' Text.Whitespace 'var' Keyword -' ' Text +' ' Text.Whitespace 'pers' Name.Variable -' ' Text +' ' Text.Whitespace '=' Operator -' ' Text +' ' Text.Whitespace 'Person' Name '(' Punctuation '"' Literal.String.Double 'Per_' Literal.String.Double '"' Literal.String.Double -' ' Text +' ' Text.Whitespace '+' Operator -' ' Text +' ' Text.Whitespace 'String' Name.Builtin '(' Punctuation 'i' Name -')' Punctuation -')' Punctuation -'\n' Text.Whitespace - -' ' Text +'))' Punctuation +'\n ' Text.Whitespace 'print' Name.Builtin '(' Punctuation 'pers' Name ')' Punctuation -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'pers' Name '.' Operator 'say_hello' Name -'(' Punctuation -')' Punctuation -'\n' Text.Whitespace - -'\n' Text.Whitespace - -' ' Text +'()' Punctuation +'\n\n ' Text.Whitespace 'generic_func' Name -'(' Punctuation -')' Punctuation -' ' Text +'()' Punctuation +' ' Text.Whitespace "# prints 'woof 3'" Comment.Single -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'generic_func' Name '[' Punctuation '5' Literal.Number.Integer -']' Punctuation -'(' Punctuation -')' Punctuation -' ' Text +']()' Punctuation +' ' Text.Whitespace "# prints 'woof 5'" Comment.Single -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'generic_func' Name '[' Punctuation '7' Literal.Number.Integer ',' Punctuation -' ' Text +' ' Text.Whitespace '"' Literal.String.Double 'meow' Literal.String.Double '"' Literal.String.Double -']' Punctuation -'(' Punctuation -')' Punctuation -' ' Text +']()' Punctuation +' ' Text.Whitespace "# prints 'meow 7'" Comment.Single -'\n' Text.Whitespace - -'\n' Text.Whitespace - -' ' Text +'\n\n ' Text.Whitespace 'try' Keyword ':' Punctuation -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'func' Name -'(' Punctuation -')' Punctuation -'\n' Text.Whitespace - -' ' Text +'()' Punctuation +'\n ' Text.Whitespace 'strict_func' Name -'(' Punctuation -')' Punctuation -'\n' Text.Whitespace - -'\n' Text.Whitespace - -' ' Text +'()' Punctuation +'\n\n ' Text.Whitespace 'raise' Keyword -' ' Text +' ' Text.Whitespace 'Error' Name.Builtin '(' Punctuation '"' Literal.String.Double 'fail' Literal.String.Double '"' Literal.String.Double ')' Punctuation -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'except' Keyword ':' Punctuation -'\n' Text.Whitespace - -' ' Text +'\n ' Text.Whitespace 'print' Name.Builtin '(' Punctuation '"' Literal.String.Double 'error was raised' Literal.String.Double '"' Literal.String.Double ')' Punctuation -'\n' Text.Whitespace - -'\n' Text.Whitespace - -' ' Text +'\n\n ' Text.Whitespace 'print' Name.Builtin '(' Punctuation 'get_sys_info' Name -'(' Punctuation -')' Punctuation -')' Punctuation +'())' Punctuation '\n' Text.Whitespace