diff --git a/Contributing.md b/Contributing.md index 0d831f0b1f..93da428ca2 100644 --- a/Contributing.md +++ b/Contributing.md @@ -11,7 +11,7 @@ Goals & non-goals of Pygments Python support -------------- -Pygments supports all supported Python versions as per the [Python Developer's Guide](https://devguide.python.org/#status-of-python-branches). Additionally, the default Python version of the latest stable version of RHEL, Ubuntu LTS, and Debian are supported, even if they're officially EOL. Supporting other end-of-life versions is a non-goal of Pygments. +Pygments supports all supported Python versions as per the [Python Developer's Guide](https://devguide.python.org/versions/). Additionally, the default Python version of the latest stable version of RHEL, Ubuntu LTS, and Debian are supported, even if they're officially EOL. Supporting other end-of-life versions is a non-goal of Pygments. Validation ---------- diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index 89a286f931..12d9887f73 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -522,6 +522,7 @@ 'WatLexer': ('pygments.lexers.webassembly', 'WebAssembly', ('wast', 'wat'), ('*.wat', '*.wast'), ()), 'WebIDLLexer': ('pygments.lexers.webidl', 'Web IDL', ('webidl',), ('*.webidl',), ()), 'WhileyLexer': ('pygments.lexers.whiley', 'Whiley', ('whiley',), ('*.whiley',), ('text/x-whiley',)), + 'WoWTocLexer': ('pygments.lexers.wowtoc', 'World of Warcraft TOC', ('wowtoc',), ('*.toc',), ()), 'X10Lexer': ('pygments.lexers.x10', 'X10', ('x10', 'xten'), ('*.x10',), ('text/x-x10',)), 'XMLUL4Lexer': ('pygments.lexers.ul4', 'XML+UL4', ('xml+ul4',), ('*.xmlul4',), ()), 'XQueryLexer': ('pygments.lexers.webmisc', 'XQuery', ('xquery', 'xqy', 'xq', 'xql', 'xqm'), ('*.xqy', '*.xquery', '*.xq', '*.xql', '*.xqm'), ('text/xquery', 'application/xquery')), diff --git a/pygments/lexers/wowtoc.py b/pygments/lexers/wowtoc.py new file mode 100644 index 0000000000..615493cdf3 --- /dev/null +++ b/pygments/lexers/wowtoc.py @@ -0,0 +1,114 @@ +""" + pygments.lexers.wowtoc + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for World of Warcraft TOC files, which describe game addon metadata. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" +import re + +from pygments.lexer import RegexLexer, bygroups +from pygments.token import Comment, Name, Text, Punctuation, String, Keyword + +__all__ = ["WoWTocLexer"] + +def _create_tag_line_pattern(inner_pattern, ignore_case=False): + return ((r"(?i)" if ignore_case else r"") + + r"^(##)( *)" # groups 1, 2 + + inner_pattern # group 3 + + r"( *)(:)( *)(.*?)( *)$") # groups 4, 5, 6, 7, 8 + + +def _create_tag_line_token(inner_pattern, inner_token, ignore_case=False): + # this function template-izes the tag line for a specific type of tag, which will + # have a different pattern and different token. otherwise, everything about a tag + # line is the same + return ( + _create_tag_line_pattern(inner_pattern, ignore_case=ignore_case), + bygroups( + Keyword.Declaration, + Text.Whitespace, + inner_token, + Text.Whitespace, + Punctuation, + Text.Whitespace, + String, + Text.Whitespace, + ), + ) + + +class WoWTocLexer(RegexLexer): + """ + Lexer for World of Warcraft TOC files. + + .. versionadded:: 2.13 + """ + + name = "World of Warcraft TOC" + aliases = ["wowtoc"] + filenames = ["*.toc"] + + tokens = { + "root": [ + # official localized tags, Notes and Title + # (normal part is insensitive, locale part is sensitive) + _create_tag_line_token( + r"((?:[nN][oO][tT][eE][sS]|[tT][iI][tT][lL][eE])-(?:ptBR|zhCN|enCN|frFR|deDE|itIT|esMX|ptPT|koKR|ruRU|esES|zhTW|enTW|enGB|enUS))", + Name.Builtin, + ), + # other official tags + _create_tag_line_token( + r"(Interface|Title|Notes|RequiredDeps|Dep[^: ]*|OptionalDeps|LoadOnDemand|LoadWith|LoadManagers|SavedVariablesPerCharacter|SavedVariables|DefaultState|Secure|Author|Version)", + Name.Builtin, + ignore_case=True, + ), + # user-defined tags + _create_tag_line_token( + r"(X-[^: ]*)", + Name.Variable, + ignore_case=True, + ), + # non-conforming tags, but still valid + _create_tag_line_token( + r"([^: ]*)", + Name.Other, + ), + + # Comments + (r"^#.*$", Comment), + + # Addon Files + (r"^.+$", Name), + ] + } + + def analyse_text(text): + # at time of writing, this file suffix conflict's with one of Tex's in + # markup.py. Tex's anaylse_text() appears to be definitive (binary) and does not + # share any likeness to WoW TOCs, which means we wont have to compete with it by + # abitrary increments in score. + + result = 0 + + # while not required, an almost certain marker of WoW TOC's is the interface tag + # if this tag is omitted, players will need to opt-in to loading the addon with + # an options change ("Load out of date addons"). the value is also standardized: + # ``, with minor and patch being two-digit zero-padded. + interface_pattern = _create_tag_line_pattern(r"(Interface)", ignore_case=True) + match = re.search(interface_pattern, text) + if match and re.match(r"(\d+)(\d{2})(\d{2})", match.group(7)): + result += 0.8 + + casefolded = text.casefold() + # Lua file listing is good marker too, but probably conflicts with many other + # lexers + if ".lua" in casefolded: + result += 0.1 + # ditto for XML files, but they're less used in WoW TOCs + if ".xml" in casefolded: + result += 0.05 + + return result diff --git a/scripts/count_token_references.py b/scripts/count_token_references.py index e235cc40a8..8e798c2de0 100755 --- a/scripts/count_token_references.py +++ b/scripts/count_token_references.py @@ -72,7 +72,10 @@ def fetch_lexer_sources(): to a list of lines. """ lexer_dir = (pathlib.Path(__file__).parent / "../pygments/lexers").resolve() - lexer_sources = {fn: fn.read_text().splitlines(keepends=False) for fn in lexer_dir.glob("*.py")} + lexer_sources = { + fn: fn.read_text(encoding='utf-8').splitlines(keepends=False) + for fn in lexer_dir.glob("*.py") + } return lexer_sources diff --git a/tests/examplefiles/wowtoc/comments.toc b/tests/examplefiles/wowtoc/comments.toc new file mode 100644 index 0000000000..381fd8985b --- /dev/null +++ b/tests/examplefiles/wowtoc/comments.toc @@ -0,0 +1,6 @@ +# +#a +# a comment +# a comment with a # in it +## no comma, thus a comment +## has space: and is thus, a comment \ No newline at end of file diff --git a/tests/examplefiles/wowtoc/comments.toc.output b/tests/examplefiles/wowtoc/comments.toc.output new file mode 100644 index 0000000000..2d69add19e --- /dev/null +++ b/tests/examplefiles/wowtoc/comments.toc.output @@ -0,0 +1,17 @@ +'#' Comment +'\n' Text + +'#a' Comment +'\n' Text + +'# a comment' Comment +'\n' Text + +'# a comment with a # in it' Comment +'\n' Text + +'## no comma, thus a comment' Comment +'\n' Text + +'## has space: and is thus, a comment' Comment +'\n' Text diff --git a/tests/examplefiles/wowtoc/files.toc b/tests/examplefiles/wowtoc/files.toc new file mode 100644 index 0000000000..172cf1025e --- /dev/null +++ b/tests/examplefiles/wowtoc/files.toc @@ -0,0 +1,3 @@ +a +Foo.lua +Spaces allowed.lua \ No newline at end of file diff --git a/tests/examplefiles/wowtoc/files.toc.output b/tests/examplefiles/wowtoc/files.toc.output new file mode 100644 index 0000000000..135f22c3f0 --- /dev/null +++ b/tests/examplefiles/wowtoc/files.toc.output @@ -0,0 +1,8 @@ +'a' Name +'\n' Text + +'Foo.lua' Name +'\n' Text + +'Spaces allowed.lua' Name +'\n' Text diff --git a/tests/examplefiles/wowtoc/official_tags.toc b/tests/examplefiles/wowtoc/official_tags.toc new file mode 100644 index 0000000000..e8c017e56f --- /dev/null +++ b/tests/examplefiles/wowtoc/official_tags.toc @@ -0,0 +1,63 @@ +## TiTlE-ptBR: value +## TiTlE-zhCN: value +## TiTlE-enCN: value +## TiTlE-frFR: value +## TiTlE-deDE: value +## TiTlE-itIT: value +## TiTlE-esMX: value +## TiTlE-ptPT: value +## TiTlE-koKR: value +## TiTlE-ruRU: value +## TiTlE-esES: value +## TiTlE-zhTW: value +## TiTlE-enTW: value +## TiTlE-enGB: value +## TiTlE-enUS: value +## NoTeS-ptBR: value +## NoTeS-zhCN: value +## NoTeS-enCN: value +## NoTeS-frFR: value +## NoTeS-deDE: value +## NoTeS-itIT: value +## NoTeS-esMX: value +## NoTeS-ptPT: value +## NoTeS-koKR: value +## NoTeS-ruRU: value +## NoTeS-esES: value +## NoTeS-zhTW: value +## NoTeS-enTW: value +## NoTeS-enGB: value +## NoTeS-enUS: value +## Interface: value +## interface: value +## Title: value +## title: value +## Notes: value +## notes: value +## RequiredDeps: value +## requireddeps: value +## Dependencies: value +## dependencies: value +## OptionalDeps: value +## optionaldeps: value +## LoadOnDemand: value +## loadondemand: value +## LoadWith: value +## loadwith: value +## LoadManagers: value +## loadmanagers: value +## SavedVariablesPerCharacter: value +## savedvariablespercharacter: value +## SavedVariables: value +## savedvariables: value +## DefaultState: value +## defaultstate: value +## Secure: value +## secure: value +## Author: value +## author: value +## Version: value +## version: value +## Dep: value +## dep: value +## DepSomething: value diff --git a/tests/examplefiles/wowtoc/official_tags.toc.output b/tests/examplefiles/wowtoc/official_tags.toc.output new file mode 100644 index 0000000000..90037ea042 --- /dev/null +++ b/tests/examplefiles/wowtoc/official_tags.toc.output @@ -0,0 +1,503 @@ +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-ptBR' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-zhCN' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-enCN' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-frFR' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-deDE' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-itIT' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-esMX' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-ptPT' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-koKR' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-ruRU' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-esES' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-zhTW' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-enTW' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-enGB' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-enUS' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-ptBR' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-zhCN' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-enCN' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-frFR' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-deDE' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-itIT' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-esMX' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-ptPT' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-koKR' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-ruRU' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-esES' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-zhTW' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-enTW' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-enGB' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-enUS' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Interface' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'interface' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Title' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'title' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Notes' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'notes' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'RequiredDeps' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'requireddeps' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Dependencies' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'dependencies' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'OptionalDeps' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'optionaldeps' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'LoadOnDemand' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'loadondemand' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'LoadWith' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'loadwith' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'LoadManagers' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'loadmanagers' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'SavedVariablesPerCharacter' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'savedvariablespercharacter' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'SavedVariables' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'savedvariables' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'DefaultState' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'defaultstate' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Secure' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'secure' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Author' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'author' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Version' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'version' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Dep' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'dep' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'DepSomething' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text diff --git a/tests/examplefiles/wowtoc/other_tags.toc b/tests/examplefiles/wowtoc/other_tags.toc new file mode 100644 index 0000000000..30ed8478fd --- /dev/null +++ b/tests/examplefiles/wowtoc/other_tags.toc @@ -0,0 +1,6 @@ +## X-Foo: Value +## Non-Conforming: Value +##no:whitespace +## plenty : of whitespace +## NoValueOk: +## other intermediate non-space whitespace: is ok diff --git a/tests/examplefiles/wowtoc/other_tags.toc.output b/tests/examplefiles/wowtoc/other_tags.toc.output new file mode 100644 index 0000000000..15acbb525a --- /dev/null +++ b/tests/examplefiles/wowtoc/other_tags.toc.output @@ -0,0 +1,45 @@ +'##' Keyword.Declaration +' ' Text.Whitespace +'X-Foo' Name.Variable +':' Punctuation +' ' Text.Whitespace +'Value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Non-Conforming' Name.Other +':' Punctuation +' ' Text.Whitespace +'Value' Literal.String +'\n' Text + +'##' Keyword.Declaration +'no' Name.Other +':' Punctuation +'whitespace' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'plenty' Name.Other +' ' Text.Whitespace +':' Punctuation +' ' Text.Whitespace +'of whitespace' Literal.String +' ' Text.Whitespace +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoValueOk' Name.Other +':' Punctuation +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'other\tintermediate\tnon-space\twhitespace' Name.Other +':' Punctuation +' ' Text.Whitespace +'is ok' Literal.String +'\n' Text