diff --git a/AUTHORS b/AUTHORS index bdb69cccc5..a7928ea88b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -276,6 +276,6 @@ Other contributors, listed alphabetically, are: * vanillajonathan -- PRQL lexer * Nikolay Antipov -- OpenSCAD lexer * Markus Meyer, Nextron Systems -- YARA lexer - +* Hannes Römer -- Mojo lexer Many thanks for all contributions! diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index 94825f7e68..9c896642c0 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -317,6 +317,7 @@ 'ModelicaLexer': ('pygments.lexers.modeling', 'Modelica', ('modelica',), ('*.mo',), ('text/x-modelica',)), 'Modula2Lexer': ('pygments.lexers.modula2', 'Modula-2', ('modula2', 'm2'), ('*.def', '*.mod'), ('text/x-modula2',)), 'MoinWikiLexer': ('pygments.lexers.markup', 'MoinMoin/Trac Wiki markup', ('trac-wiki', 'moin'), (), ('text/x-trac-wiki',)), + 'MojoLexer': ('pygments.lexers.mojo', 'Mojo', ('mojo', '🔥'), ('*.mojo', '*.🔥'), ('text/x-mojo', 'application/x-mojo')), 'MonkeyLexer': ('pygments.lexers.basic', 'Monkey', ('monkey',), ('*.monkey',), ('text/x-monkey',)), 'MonteLexer': ('pygments.lexers.monte', 'Monte', ('monte',), ('*.mt',), ()), 'MoonScriptLexer': ('pygments.lexers.scripting', 'MoonScript', ('moonscript', 'moon'), ('*.moon',), ('text/x-moonscript', 'application/x-moonscript')), diff --git a/pygments/lexers/mojo.py b/pygments/lexers/mojo.py new file mode 100644 index 0000000000..0191dce0aa --- /dev/null +++ b/pygments/lexers/mojo.py @@ -0,0 +1,704 @@ +""" + pygments.lexers.mojo + ~~~~~~~~~~~~~~~~~~~~ + + Lexers for Mojo and related languages. + + :copyright: Copyright 2006-2024 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import keyword + +from pygments import unistring as uni +from pygments.lexer import ( + RegexLexer, + bygroups, + combined, + default, + include, + this, + using, + words, +) +from pygments.token import ( + Comment, + # Error, + Keyword, + Name, + Number, + Operator, + Punctuation, + String, + Text, + Whitespace, +) +from pygments.util import shebang_matches + +__all__ = ["MojoLexer"] + + +class MojoLexer(RegexLexer): + """ + For Mojo source code (version 24.2.1). + """ + + name = "Mojo" + url = "https://docs.modular.com/mojo/" + aliases = ["mojo", "🔥"] + filenames = [ + "*.mojo", + "*.🔥", + ] + mimetypes = [ + "text/x-mojo", + "application/x-mojo", + ] + version_added = "2.18" + + uni_name = f"[{uni.xid_start}][{uni.xid_continue}]*" + + def innerstring_rules(ttype): + return [ + # the old style '%s' % (...) string formatting (still valid in Py3) + ( + r"%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?" + "[hlL]?[E-GXc-giorsaux%]", + String.Interpol, + ), + # the new style '{}'.format(...) string formatting + ( + r"\{" + r"((\w+)((\.\w+)|(\[[^\]]+\]))*)?" # field name + r"(\![sra])?" # conversion + r"(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?" + r"\}", + String.Interpol, + ), + # backslashes, quotes and formatting signs must be parsed one at a time + (r'[^\\\'"%{\n]+', ttype), + (r'[\'"\\]', ttype), + # unhandled string formatting sign + (r"%|(\{{1,2})", ttype), + # newlines are an error (use "nl" state) + ] + + def fstring_rules(ttype): + return [ + # Assuming that a '}' is the closing brace after format specifier. + # Sadly, this means that we won't detect syntax error. But it's + # more important to parse correct syntax correctly, than to + # highlight invalid syntax. + (r"\}", String.Interpol), + (r"\{", String.Interpol, "expr-inside-fstring"), + # backslashes, quotes and formatting signs must be parsed one at a time + (r'[^\\\'"{}\n]+', ttype), + (r'[\'"\\]', ttype), + # newlines are an error (use "nl" state) + ] + + tokens = { + "root": [ + (r"\s+", Whitespace), + ( + r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")', + bygroups(Whitespace, String.Affix, String.Doc), + ), + ( + r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')", + bygroups(Whitespace, String.Affix, String.Doc), + ), + (r"\A#!.+$", Comment.Hashbang), + (r"#.*$", Comment.Single), + (r"\\\n", Whitespace), + (r"\\", Whitespace), + include("keywords"), + include("soft-keywords"), + # In the original PR, all the below here used ((?:\s|\\\s)+) to + # designate whitespace, but I can't find any example of this being + # needed in the example file, so we're replacing it with `\s+`. + ( + r"(alias)(\s+)", + bygroups(Keyword, Whitespace), + "varname", # TODO varname the right fit? + ), + (r"(var)(\s+)", bygroups(Keyword, Whitespace), "varname"), + (r"(def)(\s+)", bygroups(Keyword, Whitespace), "funcname"), + (r"(fn)(\s+)", bygroups(Keyword, Whitespace), "funcname"), + ( + r"(class)(\s+)", + bygroups(Keyword, Whitespace), + "classname", + ), # not implemented yet + (r"(struct)(\s+)", bygroups(Keyword, Whitespace), "structname"), + (r"(trait)(\s+)", bygroups(Keyword, Whitespace), "structname"), + (r"(from)(\s+)", bygroups(Keyword.Namespace, Whitespace), "fromimport"), + (r"(import)(\s+)", bygroups(Keyword.Namespace, Whitespace), "import"), + include("expr"), + ], + "expr": [ + # raw f-strings + ( + '(?i)(rf|fr)(""")', + bygroups(String.Affix, String.Double), + combined("rfstringescape", "tdqf"), + ), + ( + "(?i)(rf|fr)(''')", + bygroups(String.Affix, String.Single), + combined("rfstringescape", "tsqf"), + ), + ( + '(?i)(rf|fr)(")', + bygroups(String.Affix, String.Double), + combined("rfstringescape", "dqf"), + ), + ( + "(?i)(rf|fr)(')", + bygroups(String.Affix, String.Single), + combined("rfstringescape", "sqf"), + ), + # non-raw f-strings + ( + '([fF])(""")', + bygroups(String.Affix, String.Double), + combined("fstringescape", "tdqf"), + ), + ( + "([fF])(''')", + bygroups(String.Affix, String.Single), + combined("fstringescape", "tsqf"), + ), + ( + '([fF])(")', + bygroups(String.Affix, String.Double), + combined("fstringescape", "dqf"), + ), + ( + "([fF])(')", + bygroups(String.Affix, String.Single), + combined("fstringescape", "sqf"), + ), + # raw bytes and strings + ('(?i)(rb|br|r)(""")', bygroups(String.Affix, String.Double), "tdqs"), + ("(?i)(rb|br|r)(''')", bygroups(String.Affix, String.Single), "tsqs"), + ('(?i)(rb|br|r)(")', bygroups(String.Affix, String.Double), "dqs"), + ("(?i)(rb|br|r)(')", bygroups(String.Affix, String.Single), "sqs"), + # non-raw strings + ( + '([uU]?)(""")', + bygroups(String.Affix, String.Double), + combined("stringescape", "tdqs"), + ), + ( + "([uU]?)(''')", + bygroups(String.Affix, String.Single), + combined("stringescape", "tsqs"), + ), + ( + '([uU]?)(")', + bygroups(String.Affix, String.Double), + combined("stringescape", "dqs"), + ), + ( + "([uU]?)(')", + bygroups(String.Affix, String.Single), + combined("stringescape", "sqs"), + ), + # non-raw bytes + ( + '([bB])(""")', + bygroups(String.Affix, String.Double), + combined("bytesescape", "tdqs"), + ), + ( + "([bB])(''')", + bygroups(String.Affix, String.Single), + combined("bytesescape", "tsqs"), + ), + ( + '([bB])(")', + bygroups(String.Affix, String.Double), + combined("bytesescape", "dqs"), + ), + ( + "([bB])(')", + bygroups(String.Affix, String.Single), + combined("bytesescape", "sqs"), + ), + (r"[^\S\n]+", Text), + include("numbers"), + (r"!=|==|<<|>>|:=|[-~+/*%=<>&^|.]", Operator), + (r"([]{}:\(\),;[])+", Punctuation), + (r"(in|is|and|or|not)\b", Operator.Word), + include("expr-keywords"), + include("builtins"), + include("magicfuncs"), + include("magicvars"), + include("name"), + ], + "expr-inside-fstring": [ + (r"[{([]", Punctuation, "expr-inside-fstring-inner"), + # without format specifier + ( + r"(=\s*)?" # debug (https://bugs.python.org/issue36817) + r"(\![sraf])?" # conversion + r"\}", + String.Interpol, + "#pop", + ), + # with format specifier + # we'll catch the remaining '}' in the outer scope + ( + r"(=\s*)?" # debug (https://bugs.python.org/issue36817) + r"(\![sraf])?" # conversion + r":", + String.Interpol, + "#pop", + ), + (r"\s+", Whitespace), # allow new lines + include("expr"), + ], + "expr-inside-fstring-inner": [ + (r"[{([]", Punctuation, "expr-inside-fstring-inner"), + (r"[])}]", Punctuation, "#pop"), + (r"\s+", Whitespace), # allow new lines + include("expr"), + ], + "expr-keywords": [ + # Based on https://docs.python.org/3/reference/expressions.html + ( + words( + ( + "async for", # TODO https://docs.modular.com/mojo/roadmap#no-async-for-or-async-with + "async with", # TODO https://docs.modular.com/mojo/roadmap#no-async-for-or-async-with + "await", + "else", + "for", + "if", + "lambda", + "yield", + "yield from", + ), + suffix=r"\b", + ), + Keyword, + ), + (words(("True", "False", "None"), suffix=r"\b"), Keyword.Constant), + ], + "keywords": [ + ( + words( + ( + "assert", + "async", + "await", + "borrowed", + "break", + "continue", + "del", + "elif", + "else", + "except", + "finally", + "for", + "global", + "if", + "lambda", + "pass", + "raise", + "nonlocal", + "return", + "try", + "while", + "yield", + "yield from", + "as", + "with", + ), + suffix=r"\b", + ), + Keyword, + ), + (words(("True", "False", "None"), suffix=r"\b"), Keyword.Constant), + ], + "soft-keywords": [ + # `match`, `case` and `_` soft keywords + ( + r"(^[ \t]*)" # at beginning of line + possible indentation + r"(match|case)\b" # a possible keyword + r"(?![ \t]*(?:" # not followed by... + r"[:,;=^&|@~)\]}]|(?:" + # characters and keywords that mean this isn't + # pattern matching (but None/True/False is ok) + r"|".join(k for k in keyword.kwlist if k[0].islower()) + + r")\b))", + bygroups(Whitespace, Keyword), + "soft-keywords-inner", + ), + ], + "soft-keywords-inner": [ + # optional `_` keyword + (r"(\s+)([^\n_]*)(_\b)", bygroups(Whitespace, using(this), Keyword)), + default("#pop"), + ], + "builtins": [ + ( + words( + ( + "__import__", + "abs", + "aiter", + "all", + "any", + "bin", + "bool", + "bytearray", + "breakpoint", + "bytes", + "callable", + "chr", + "classmethod", + "compile", + "complex", + "delattr", + "dict", + "dir", + "divmod", + "enumerate", + "eval", + "filter", + "float", + "format", + "frozenset", + "getattr", + "globals", + "hasattr", + "hash", + "hex", + "id", + "input", + "int", + "isinstance", + "issubclass", + "iter", + "len", + "list", + "locals", + "map", + "max", + "memoryview", + "min", + "next", + "object", + "oct", + "open", + "ord", + "pow", + "print", + "property", + "range", + "repr", + "reversed", + "round", + "set", + "setattr", + "slice", + "sorted", + "staticmethod", + "str", + "sum", + "super", + "tuple", + "type", + "vars", + "zip", + # Mojo builtin types: https://docs.modular.com/mojo/stdlib/builtin/ + "AnyType", + "Coroutine", + "DType", + "Error", + "Int", + "List", + "ListLiteral", + "Scalar", + "Int8", + "UInt8", + "Int16", + "UInt16", + "Int32", + "UInt32", + "Int64", + "UInt64", + "BFloat16", + "Float16", + "Float32", + "Float64", + "SIMD", + "String", + "Tensor", + "Tuple", + "Movable", + "Copyable", + "CollectionElement", + ), + prefix=r"(? None: + variable = "implicit variable" + print("def function", variable) + + +fn strict_func(): + var variable = "explicit variable" + print("fn function", variable) + + +fn generic_func[a: Int = 3, msg: StringLiteral = "woof"](): + """ + A generic function. + """ + print(msg, a) + + +fn get_sys_info() -> String: + """ + Retrieves system information (based upon: https://github.com/modularml/mojo/blob/main/examples/deviceinfo.mojo). + """ + var os = "" + if os_is_linux(): + os = "linux" + elif os_is_macos(): + os = "macOS" + else: + os = "windows" + var cpu = String(_current_cpu()) + var arch = String(_triple_attr()) + var cpu_features = String("") + if has_sse4(): + cpu_features += "sse4" + if has_avx(): + cpu_features += " avx" + if has_avx2(): + cpu_features += " avx2" + + var info = ( + """System information: + OS : """ + + os + + """ + CPU : """ + + cpu + + """ + Arch : """ + + arch + + """ + Physical Cores : """ + + num_physical_cores() + + """ + Logical Cores : """ + + num_logical_cores() + + """ + CPU Features : """ + + cpu_features + ) + + return info + + +trait Greet: + """ + A trait example. + """ + + fn say_hello(self) -> None: + """Says hello.""" + ... + + +@value +struct Person(Greet): + """ + A person. + """ + + var name: String + + fn __init__(inout self, name: String): + self.name = name + + fn __str__(self) -> String: + """ + Overridden __str__ method + """ + return "Person named " + self.name + + fn say_hello(self) -> None: + print(self.name, "says hello!") + + +fn main(): + for i in range(3): + var pers = Person("Per_" + String(i)) + print(pers) + pers.say_hello() + + generic_func() # prints 'woof 3' + generic_func[5]() # prints 'woof 5' + generic_func[7, "meow"]() # prints 'meow 7' + + try: + func() + strict_func() + + raise Error("fail") + except: + print("error was raised") + + print(get_sys_info()) diff --git a/tests/examplefiles/mojo/sample.mojo.output b/tests/examplefiles/mojo/sample.mojo.output new file mode 100644 index 0000000000..f41913d10d --- /dev/null +++ b/tests/examplefiles/mojo/sample.mojo.output @@ -0,0 +1,629 @@ +'from' Keyword.Namespace +' ' Text.Whitespace +'sys' Name.Namespace +'.' Name.Namespace +'info' Name.Namespace +' ' Text.Whitespace +'import' Keyword.Namespace +' ' Text.Whitespace +'(' Punctuation +'\n ' Text.Whitespace +'os_is_linux' Name +',' Punctuation +'\n ' Text.Whitespace +'os_is_macos' Name +',' Punctuation +'\n ' Text.Whitespace +'has_sse4' Name +',' Punctuation +'\n ' Text.Whitespace +'has_avx' Name +',' Punctuation +'\n ' Text.Whitespace +'has_avx2' Name +',' Punctuation +'\n ' Text.Whitespace +'num_physical_cores' Name +',' Punctuation +'\n ' Text.Whitespace +'num_logical_cores' Name +',' Punctuation +'\n' Text.Whitespace + +')' Punctuation +'\n' Text.Whitespace + +'from' Keyword.Namespace +' ' Text.Whitespace +'sys' Name.Namespace +'.' Name.Namespace +'info' Name.Namespace +' ' Text.Whitespace +'import' Keyword.Namespace +' ' Text.Whitespace +'_current_cpu' Name +',' Punctuation +' ' Text.Whitespace +'_current_target' Name +',' Punctuation +' ' Text.Whitespace +'_triple_attr' Name +'\n\n\n' Text.Whitespace + +'def' Keyword +' ' Text.Whitespace +'func' Name.Function +'()' Punctuation +' ' Text.Whitespace +'-' Operator +'>' Operator +' ' Text.Whitespace +'None' Keyword.Constant +':' Punctuation +'\n ' Text.Whitespace +'variable' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String.Double +'implicit variable' Literal.String.Double +'"' Literal.String.Double +'\n ' Text.Whitespace +'print' Name.Builtin +'(' Punctuation +'"' Literal.String.Double +'def function' Literal.String.Double +'"' Literal.String.Double +',' Punctuation +' ' Text.Whitespace +'variable' Name +')' Punctuation +'\n\n\n' Text.Whitespace + +'fn' Keyword +' ' Text.Whitespace +'strict_func' Name.Function +'():' Punctuation +'\n ' Text.Whitespace +'var' Keyword +' ' Text.Whitespace +'variable' Name.Variable +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String.Double +'explicit variable' Literal.String.Double +'"' Literal.String.Double +'\n ' Text.Whitespace +'print' Name.Builtin +'(' Punctuation +'"' Literal.String.Double +'fn function' Literal.String.Double +'"' Literal.String.Double +',' Punctuation +' ' Text.Whitespace +'variable' Name +')' Punctuation +'\n\n\n' Text.Whitespace + +'fn' Keyword +' ' Text.Whitespace +'generic_func' Name.Function +'[' Punctuation +'a' Name +':' Punctuation +' ' Text.Whitespace +'Int' Name.Builtin +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'3' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'msg' Name +':' Punctuation +' ' Text.Whitespace +'StringLiteral' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String.Double +'woof' Literal.String.Double +'"' Literal.String.Double +']():' Punctuation +'\n ' Text.Whitespace +'"""' Literal.String.Double +'\n' Literal.String.Double + +' A generic function.' Literal.String.Double +'\n' Literal.String.Double + +' ' Literal.String.Double +'"""' Literal.String.Double +'\n ' Text.Whitespace +'print' Name.Builtin +'(' Punctuation +'msg' Name +',' Punctuation +' ' Text.Whitespace +'a' Name +')' Punctuation +'\n\n\n' Text.Whitespace + +'fn' Keyword +' ' Text.Whitespace +'get_sys_info' Name.Function +'()' Punctuation +' ' Text.Whitespace +'-' Operator +'>' Operator +' ' Text.Whitespace +'String' Name.Builtin +':' Punctuation +'\n ' Text.Whitespace +'"""' Literal.String.Double +'\n' Literal.String.Double + +' Retrieves system information (based upon: https://github.com/modularml/mojo/blob/main/examples/deviceinfo.mojo).' Literal.String.Double +'\n' Literal.String.Double + +' ' Literal.String.Double +'"""' Literal.String.Double +'\n ' Text.Whitespace +'var' Keyword +' ' Text.Whitespace +'os' Name.Variable +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String.Double +'"' Literal.String.Double +'\n ' Text.Whitespace +'if' Keyword +' ' Text.Whitespace +'os_is_linux' Name +'():' Punctuation +'\n ' Text.Whitespace +'os' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String.Double +'linux' Literal.String.Double +'"' Literal.String.Double +'\n ' Text.Whitespace +'elif' Keyword +' ' Text.Whitespace +'os_is_macos' Name +'():' Punctuation +'\n ' Text.Whitespace +'os' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String.Double +'macOS' Literal.String.Double +'"' Literal.String.Double +'\n ' Text.Whitespace +'else' Keyword +':' Punctuation +'\n ' Text.Whitespace +'os' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'"' Literal.String.Double +'windows' Literal.String.Double +'"' Literal.String.Double +'\n ' Text.Whitespace +'var' Keyword +' ' Text.Whitespace +'cpu' Name.Variable +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'String' Name.Builtin +'(' Punctuation +'_current_cpu' Name +'())' Punctuation +'\n ' Text.Whitespace +'var' Keyword +' ' Text.Whitespace +'arch' Name.Variable +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'String' Name.Builtin +'(' Punctuation +'_triple_attr' Name +'())' Punctuation +'\n ' Text.Whitespace +'var' Keyword +' ' Text.Whitespace +'cpu_features' Name.Variable +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'String' Name.Builtin +'(' Punctuation +'"' Literal.String.Double +'"' Literal.String.Double +')' Punctuation +'\n ' Text.Whitespace +'if' Keyword +' ' Text.Whitespace +'has_sse4' Name +'():' Punctuation +'\n ' Text.Whitespace +'cpu_features' Name +' ' Text.Whitespace +'+' Operator +'=' Operator +' ' Text.Whitespace +'"' Literal.String.Double +'sse4' Literal.String.Double +'"' Literal.String.Double +'\n ' Text.Whitespace +'if' Keyword +' ' Text.Whitespace +'has_avx' Name +'():' Punctuation +'\n ' Text.Whitespace +'cpu_features' Name +' ' Text.Whitespace +'+' Operator +'=' Operator +' ' Text.Whitespace +'"' Literal.String.Double +' avx' Literal.String.Double +'"' Literal.String.Double +'\n ' Text.Whitespace +'if' Keyword +' ' Text.Whitespace +'has_avx2' Name +'():' Punctuation +'\n ' Text.Whitespace +'cpu_features' Name +' ' Text.Whitespace +'+' Operator +'=' Operator +' ' Text.Whitespace +'"' Literal.String.Double +' avx2' Literal.String.Double +'"' Literal.String.Double +'\n\n ' Text.Whitespace +'var' Keyword +' ' Text.Whitespace +'info' Name.Variable +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'(' Punctuation +'\n ' Text.Whitespace +'"""' Literal.String.Double +'System information:' Literal.String.Double +'\n' Literal.String.Double + +' OS : ' Literal.String.Double +'"""' Literal.String.Double +'\n ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'os' Name +'\n ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'"""' Literal.String.Double +'\n' Literal.String.Double + +' CPU : ' Literal.String.Double +'"""' Literal.String.Double +'\n ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'cpu' Name +'\n ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'"""' Literal.String.Double +'\n' Literal.String.Double + +' Arch : ' Literal.String.Double +'"""' Literal.String.Double +'\n ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'arch' Name +'\n ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'"""' Literal.String.Double +'\n' Literal.String.Double + +' Physical Cores : ' Literal.String.Double +'"""' Literal.String.Double +'\n ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'num_physical_cores' Name +'()' Punctuation +'\n ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'"""' Literal.String.Double +'\n' Literal.String.Double + +' Logical Cores : ' Literal.String.Double +'"""' Literal.String.Double +'\n ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'num_logical_cores' Name +'()' Punctuation +'\n ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'"""' Literal.String.Double +'\n' Literal.String.Double + +' CPU Features : ' Literal.String.Double +'"""' Literal.String.Double +'\n ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'cpu_features' Name +'\n ' Text.Whitespace +')' Punctuation +'\n\n ' Text.Whitespace +'return' Keyword +' ' Text.Whitespace +'info' Name +'\n\n\n' Text.Whitespace + +'trait' Keyword +' ' Text.Whitespace +'Greet' Name.Struct +':' Punctuation +'\n ' Text.Whitespace +'"""' Literal.String.Double +'\n' Literal.String.Double + +' A trait example.' Literal.String.Double +'\n' Literal.String.Double + +' ' Literal.String.Double +'"""' Literal.String.Double +'\n\n ' Text.Whitespace +'fn' Keyword +' ' Text.Whitespace +'say_hello' Name.Function +'(' Punctuation +'self' Name.Builtin.Pseudo +')' Punctuation +' ' Text.Whitespace +'-' Operator +'>' Operator +' ' Text.Whitespace +'None' Keyword.Constant +':' Punctuation +'\n ' Text.Whitespace +'"""' Literal.String.Double +'Says hello.' Literal.String.Double +'"""' Literal.String.Double +'\n ' Text.Whitespace +'.' Operator +'.' Operator +'.' Operator +'\n\n\n' Text.Whitespace + +'@value' Name.Decorator +'\n' Text.Whitespace + +'struct' Keyword +' ' Text.Whitespace +'Person' Name.Struct +'(' Punctuation +'Greet' Name +'):' Punctuation +'\n ' Text.Whitespace +'"""' Literal.String.Double +'\n' Literal.String.Double + +' A person.' Literal.String.Double +'\n' Literal.String.Double + +' ' Literal.String.Double +'"""' Literal.String.Double +'\n\n ' Text.Whitespace +'var' Keyword +' ' Text.Whitespace +'name' Name.Variable +':' Punctuation +' ' Text.Whitespace +'String' Name.Builtin +'\n\n ' Text.Whitespace +'fn' Keyword +' ' Text.Whitespace +'__init__' Name.Function.Magic +'(' Punctuation +'inout' Name +' ' Text.Whitespace +'self' Name.Builtin.Pseudo +',' Punctuation +' ' Text.Whitespace +'name' Name +':' Punctuation +' ' Text.Whitespace +'String' Name.Builtin +'):' Punctuation +'\n ' Text.Whitespace +'self' Name.Builtin.Pseudo +'.' Operator +'name' Name +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'name' Name +'\n\n ' Text.Whitespace +'fn' Keyword +' ' Text.Whitespace +'__str__' Name.Function.Magic +'(' Punctuation +'self' Name.Builtin.Pseudo +')' Punctuation +' ' Text.Whitespace +'-' Operator +'>' Operator +' ' Text.Whitespace +'String' Name.Builtin +':' Punctuation +'\n ' Text.Whitespace +'"""' Literal.String.Double +'\n' Literal.String.Double + +' Overridden __str__ method' Literal.String.Double +'\n' Literal.String.Double + +' ' Literal.String.Double +'"""' Literal.String.Double +'\n ' Text.Whitespace +'return' Keyword +' ' Text.Whitespace +'"' Literal.String.Double +'Person named ' Literal.String.Double +'"' Literal.String.Double +' ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'self' Name.Builtin.Pseudo +'.' Operator +'name' Name +'\n\n ' Text.Whitespace +'fn' Keyword +' ' Text.Whitespace +'say_hello' Name.Function +'(' Punctuation +'self' Name.Builtin.Pseudo +')' Punctuation +' ' Text.Whitespace +'-' Operator +'>' Operator +' ' Text.Whitespace +'None' Keyword.Constant +':' Punctuation +'\n ' Text.Whitespace +'print' Name.Builtin +'(' Punctuation +'self' Name.Builtin.Pseudo +'.' Operator +'name' Name +',' Punctuation +' ' Text.Whitespace +'"' Literal.String.Double +'says hello!' Literal.String.Double +'"' Literal.String.Double +')' Punctuation +'\n\n\n' Text.Whitespace + +'fn' Keyword +' ' Text.Whitespace +'main' Name.Function +'():' Punctuation +'\n ' Text.Whitespace +'for' Keyword +' ' Text.Whitespace +'i' Name +' ' Text.Whitespace +'in' Operator.Word +' ' Text.Whitespace +'range' Name.Builtin +'(' Punctuation +'3' Literal.Number.Integer +'):' Punctuation +'\n ' Text.Whitespace +'var' Keyword +' ' Text.Whitespace +'pers' Name.Variable +' ' Text.Whitespace +'=' Operator +' ' Text.Whitespace +'Person' Name +'(' Punctuation +'"' Literal.String.Double +'Per_' Literal.String.Double +'"' Literal.String.Double +' ' Text.Whitespace +'+' Operator +' ' Text.Whitespace +'String' Name.Builtin +'(' Punctuation +'i' Name +'))' Punctuation +'\n ' Text.Whitespace +'print' Name.Builtin +'(' Punctuation +'pers' Name +')' Punctuation +'\n ' Text.Whitespace +'pers' Name +'.' Operator +'say_hello' Name +'()' Punctuation +'\n\n ' Text.Whitespace +'generic_func' Name +'()' Punctuation +' ' Text.Whitespace +"# prints 'woof 3'" Comment.Single +'\n ' Text.Whitespace +'generic_func' Name +'[' Punctuation +'5' Literal.Number.Integer +']()' Punctuation +' ' Text.Whitespace +"# prints 'woof 5'" Comment.Single +'\n ' Text.Whitespace +'generic_func' Name +'[' Punctuation +'7' Literal.Number.Integer +',' Punctuation +' ' Text.Whitespace +'"' Literal.String.Double +'meow' Literal.String.Double +'"' Literal.String.Double +']()' Punctuation +' ' Text.Whitespace +"# prints 'meow 7'" Comment.Single +'\n\n ' Text.Whitespace +'try' Keyword +':' Punctuation +'\n ' Text.Whitespace +'func' Name +'()' Punctuation +'\n ' Text.Whitespace +'strict_func' Name +'()' Punctuation +'\n\n ' Text.Whitespace +'raise' Keyword +' ' Text.Whitespace +'Error' Name.Builtin +'(' Punctuation +'"' Literal.String.Double +'fail' Literal.String.Double +'"' Literal.String.Double +')' Punctuation +'\n ' Text.Whitespace +'except' Keyword +':' Punctuation +'\n ' Text.Whitespace +'print' Name.Builtin +'(' Punctuation +'"' Literal.String.Double +'error was raised' Literal.String.Double +'"' Literal.String.Double +')' Punctuation +'\n\n ' Text.Whitespace +'print' Name.Builtin +'(' Punctuation +'get_sys_info' Name +'())' Punctuation +'\n' Text.Whitespace diff --git a/tests/snippets/mojo/test_floats.txt b/tests/snippets/mojo/test_floats.txt new file mode 100644 index 0000000000..3ae9dc96e4 --- /dev/null +++ b/tests/snippets/mojo/test_floats.txt @@ -0,0 +1,75 @@ +---input--- +123 -11 0 -0 0.5 .5 1. -0.5 +0.5 -.5 -1. 2e1 -2e1 2e -2e +2e e.3 -e.3 11.2e-3 -11.2e-3 5_6 5__6 _5 6_ 5.6_7 5.67_ + +---tokens--- +'123' Literal.Number.Integer +' ' Text +'-' Operator +'11' Literal.Number.Integer +' ' Text +'0' Literal.Number.Integer +' ' Text +'-' Operator +'0' Literal.Number.Integer +' ' Text +'0.5' Literal.Number.Float +' ' Text +'.5' Literal.Number.Float +' ' Text +'1.' Literal.Number.Float +' ' Text +'-' Operator +'0.5' Literal.Number.Float +' ' Text +'+' Operator +'0.5' Literal.Number.Float +' ' Text +'-' Operator +'.5' Literal.Number.Float +' ' Text +'-' Operator +'1.' Literal.Number.Float +' ' Text +'2e1' Literal.Number.Float +' ' Text +'-' Operator +'2e1' Literal.Number.Float +' ' Text +'2' Literal.Number.Integer +'e' Name +' ' Text +'-' Operator +'2' Literal.Number.Integer +'e' Name +' ' Text +'+' Operator +'2' Literal.Number.Integer +'e' Name +' ' Text +'e' Name +'.3' Literal.Number.Float +' ' Text +'-' Operator +'e' Name +'.3' Literal.Number.Float +' ' Text +'11.2e-3' Literal.Number.Float +' ' Text +'-' Operator +'11.2e-3' Literal.Number.Float +' ' Text +'5_6' Literal.Number.Integer +' ' Text +'5' Literal.Number.Integer +'__6' Name +' ' Text +'_5' Name +' ' Text +'6' Literal.Number.Integer +'_' Name +' ' Text +'5.6_7' Literal.Number.Float +' ' Text +'5.67' Literal.Number.Float +'_' Name +'\n' Text.Whitespace diff --git a/tests/snippets/mojo/test_kw.txt b/tests/snippets/mojo/test_kw.txt new file mode 100644 index 0000000000..ed1eaf14fd --- /dev/null +++ b/tests/snippets/mojo/test_kw.txt @@ -0,0 +1,38 @@ +---input--- +fn +def +struct +trait +class +if +elif +else +with + +---tokens--- +'fn' Keyword +'\n' Text + +'def' Name.Function +'\n' Text.Whitespace + +'struct' Keyword +'\n' Text + +'trait' Name.Struct +'\n' Text.Whitespace + +'class' Keyword +'\n' Text + +'if' Name.Class +'\n' Text.Whitespace + +'elif' Keyword +'\n' Text.Whitespace + +'else' Keyword +'\n' Text.Whitespace + +'with' Keyword +'\n' Text.Whitespace diff --git a/tests/snippets/mojo/test_needs_name.txt b/tests/snippets/mojo/test_needs_name.txt new file mode 100644 index 0000000000..f121da03f2 --- /dev/null +++ b/tests/snippets/mojo/test_needs_name.txt @@ -0,0 +1,55 @@ +# Tests that '@' is recognized as an Operator + +---input--- +S = (H @ beta - r).T @ inv(H @ V @ H.T) @ (H @ beta - r) + +---tokens--- +'S' Name +' ' Text +'=' Operator +' ' Text +'(' Punctuation +'H' Name +' ' Text +'@' Operator +' ' Text +'beta' Name +' ' Text +'-' Operator +' ' Text +'r' Name +')' Punctuation +'.' Operator +'T' Name +' ' Text +'@' Operator +' ' Text +'inv' Name +'(' Punctuation +'H' Name +' ' Text +'@' Operator +' ' Text +'V' Name +' ' Text +'@' Operator +' ' Text +'H' Name +'.' Operator +'T' Name +')' Punctuation +' ' Text +'@' Operator +' ' Text +'(' Punctuation +'H' Name +' ' Text +'@' Operator +' ' Text +'beta' Name +' ' Text +'-' Operator +' ' Text +'r' Name +')' Punctuation +'\n' Text.Whitespace diff --git a/tests/snippets/mojo/test_soft_kwds.txt b/tests/snippets/mojo/test_soft_kwds.txt new file mode 100644 index 0000000000..af9c8954ad --- /dev/null +++ b/tests/snippets/mojo/test_soft_kwds.txt @@ -0,0 +1,67 @@ +---input--- +match spam: + case Some(x): + print(f"found {x}") + case None: + print("found nothing") + case _: + assert False + +---tokens--- +'match' Keyword +' ' Text +'spam' Name +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'case' Keyword +' ' Text +'Some' Name +'(' Punctuation +'x' Name +')' Punctuation +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'print' Name.Builtin +'(' Punctuation +'f' Literal.String.Affix +'"' Literal.String.Double +'found ' Literal.String.Double +'{' Literal.String.Interpol +'x' Name +'}' Literal.String.Interpol +'"' Literal.String.Double +')' Punctuation +'\n' Text.Whitespace + +' ' Text +'case' Keyword +' ' Text +'None' Keyword.Constant +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'print' Name.Builtin +'(' Punctuation +'"' Literal.String.Double +'found nothing' Literal.String.Double +'"' Literal.String.Double +')' Punctuation +'\n' Text.Whitespace + +' ' Text +'case' Keyword +' ' Text.Whitespace +'_' Keyword +':' Punctuation +'\n' Text.Whitespace + +' ' Text +'assert' Keyword +' ' Text +'False' Keyword.Constant +'\n' Text.Whitespace