From f627beca58e921733e6f230ad0c0d7b5a752cb0f Mon Sep 17 00:00:00 2001 From: Tim Martin Date: Tue, 20 Sep 2022 16:04:00 -0500 Subject: [PATCH 01/11] Add World of Warcraft TOC file lexer --- pygments/lexers/_mapping.py | 1 + pygments/lexers/wowtoc.py | 82 +++ tests/examplefiles/wowtoc/comments.toc | 6 + tests/examplefiles/wowtoc/comments.toc.output | 17 + tests/examplefiles/wowtoc/files.toc | 3 + tests/examplefiles/wowtoc/files.toc.output | 8 + tests/examplefiles/wowtoc/official_tags.toc | 63 +++ .../wowtoc/official_tags.toc.output | 503 ++++++++++++++++++ tests/examplefiles/wowtoc/other_tags.toc | 6 + .../examplefiles/wowtoc/other_tags.toc.output | 45 ++ 10 files changed, 734 insertions(+) create mode 100644 pygments/lexers/wowtoc.py create mode 100644 tests/examplefiles/wowtoc/comments.toc create mode 100644 tests/examplefiles/wowtoc/comments.toc.output create mode 100644 tests/examplefiles/wowtoc/files.toc create mode 100644 tests/examplefiles/wowtoc/files.toc.output create mode 100644 tests/examplefiles/wowtoc/official_tags.toc create mode 100644 tests/examplefiles/wowtoc/official_tags.toc.output create mode 100644 tests/examplefiles/wowtoc/other_tags.toc create mode 100644 tests/examplefiles/wowtoc/other_tags.toc.output diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index 89a286f931..12d9887f73 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -522,6 +522,7 @@ 'WatLexer': ('pygments.lexers.webassembly', 'WebAssembly', ('wast', 'wat'), ('*.wat', '*.wast'), ()), 'WebIDLLexer': ('pygments.lexers.webidl', 'Web IDL', ('webidl',), ('*.webidl',), ()), 'WhileyLexer': ('pygments.lexers.whiley', 'Whiley', ('whiley',), ('*.whiley',), ('text/x-whiley',)), + 'WoWTocLexer': ('pygments.lexers.wowtoc', 'World of Warcraft TOC', ('wowtoc',), ('*.toc',), ()), 'X10Lexer': ('pygments.lexers.x10', 'X10', ('x10', 'xten'), ('*.x10',), ('text/x-x10',)), 'XMLUL4Lexer': ('pygments.lexers.ul4', 'XML+UL4', ('xml+ul4',), ('*.xmlul4',), ()), 'XQueryLexer': ('pygments.lexers.webmisc', 'XQuery', ('xquery', 'xqy', 'xq', 'xql', 'xqm'), ('*.xqy', '*.xquery', '*.xq', '*.xql', '*.xqm'), ('text/xquery', 'application/xquery')), diff --git a/pygments/lexers/wowtoc.py b/pygments/lexers/wowtoc.py new file mode 100644 index 0000000000..711fe51c0a --- /dev/null +++ b/pygments/lexers/wowtoc.py @@ -0,0 +1,82 @@ +""" + pygments.lexers.wowtoc + ~~~~~~~~~~~~~~~~~~~~~~ + + Lexer for World of Warcraft TOC files, which describe game addon metadata. + + :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +from pygments.lexer import RegexLexer, bygroups +from pygments.token import Comment, Name, Text, Punctuation, String, Keyword + +__all__ = ["WoWTocLexer"] + +def _create_tag_line_token(inner_pattern, inner_token, ignore_case=False): + # this function template-izes the tag line for a specific type of tag, which will + # have a different pattern and different token. otherwise, everything about a tag + # line is the same + return ( + (r"(?i)" if ignore_case else r"") + + r"^(##)( *)" + + inner_pattern + + r"( *)(:)( *)([^\n]*?)( *)$", + bygroups( + Keyword.Declaration, + Text.Whitespace, + inner_token, + Text.Whitespace, + Punctuation, + Text.Whitespace, + String, + Text.Whitespace, + ), + ) + + +class WoWTocLexer(RegexLexer): + """ + Lexer for World of Warcraft TOC files. + + .. versionadded:: 2.13 + """ + + name = "World of Warcraft TOC" + aliases = ["wowtoc"] + filenames = ["*.toc"] + + tokens = { + "root": [ + # official localized tags, Notes and Title + # (normal part is insensitive, locale part is sensitive) + _create_tag_line_token( + r"((?:[nN][oO][tT][eE][sS]|[tT][iI][tT][lL][eE])-(?:ptBR|zhCN|enCN|frFR|deDE|itIT|esMX|ptPT|koKR|ruRU|esES|zhTW|enTW|enGB|enUS))", + Name.Builtin, + ), + # other official tags + _create_tag_line_token( + r"(Interface|Title|Notes|RequiredDeps|Dep[^: ]*|OptionalDeps|LoadOnDemand|LoadWith|LoadManagers|SavedVariablesPerCharacter|SavedVariables|DefaultState|Secure|Author|Version)", + Name.Builtin, + ignore_case=True + ), + # user-defined tags + _create_tag_line_token( + r"(x-[^: ]*)", + Name.Variable, + ignore_case=True + ), + # non-conforming tags, but still valid + _create_tag_line_token( + r"([^: ]*)", + Name.Other, + ignore_case=True + ), + + # Comments + (r"^#[^\n]*$", Comment), + + # Addon Files + (r"^[^\n]+$", Name), + ] + } diff --git a/tests/examplefiles/wowtoc/comments.toc b/tests/examplefiles/wowtoc/comments.toc new file mode 100644 index 0000000000..381fd8985b --- /dev/null +++ b/tests/examplefiles/wowtoc/comments.toc @@ -0,0 +1,6 @@ +# +#a +# a comment +# a comment with a # in it +## no comma, thus a comment +## has space: and is thus, a comment \ No newline at end of file diff --git a/tests/examplefiles/wowtoc/comments.toc.output b/tests/examplefiles/wowtoc/comments.toc.output new file mode 100644 index 0000000000..2d69add19e --- /dev/null +++ b/tests/examplefiles/wowtoc/comments.toc.output @@ -0,0 +1,17 @@ +'#' Comment +'\n' Text + +'#a' Comment +'\n' Text + +'# a comment' Comment +'\n' Text + +'# a comment with a # in it' Comment +'\n' Text + +'## no comma, thus a comment' Comment +'\n' Text + +'## has space: and is thus, a comment' Comment +'\n' Text diff --git a/tests/examplefiles/wowtoc/files.toc b/tests/examplefiles/wowtoc/files.toc new file mode 100644 index 0000000000..172cf1025e --- /dev/null +++ b/tests/examplefiles/wowtoc/files.toc @@ -0,0 +1,3 @@ +a +Foo.lua +Spaces allowed.lua \ No newline at end of file diff --git a/tests/examplefiles/wowtoc/files.toc.output b/tests/examplefiles/wowtoc/files.toc.output new file mode 100644 index 0000000000..135f22c3f0 --- /dev/null +++ b/tests/examplefiles/wowtoc/files.toc.output @@ -0,0 +1,8 @@ +'a' Name +'\n' Text + +'Foo.lua' Name +'\n' Text + +'Spaces allowed.lua' Name +'\n' Text diff --git a/tests/examplefiles/wowtoc/official_tags.toc b/tests/examplefiles/wowtoc/official_tags.toc new file mode 100644 index 0000000000..e8c017e56f --- /dev/null +++ b/tests/examplefiles/wowtoc/official_tags.toc @@ -0,0 +1,63 @@ +## TiTlE-ptBR: value +## TiTlE-zhCN: value +## TiTlE-enCN: value +## TiTlE-frFR: value +## TiTlE-deDE: value +## TiTlE-itIT: value +## TiTlE-esMX: value +## TiTlE-ptPT: value +## TiTlE-koKR: value +## TiTlE-ruRU: value +## TiTlE-esES: value +## TiTlE-zhTW: value +## TiTlE-enTW: value +## TiTlE-enGB: value +## TiTlE-enUS: value +## NoTeS-ptBR: value +## NoTeS-zhCN: value +## NoTeS-enCN: value +## NoTeS-frFR: value +## NoTeS-deDE: value +## NoTeS-itIT: value +## NoTeS-esMX: value +## NoTeS-ptPT: value +## NoTeS-koKR: value +## NoTeS-ruRU: value +## NoTeS-esES: value +## NoTeS-zhTW: value +## NoTeS-enTW: value +## NoTeS-enGB: value +## NoTeS-enUS: value +## Interface: value +## interface: value +## Title: value +## title: value +## Notes: value +## notes: value +## RequiredDeps: value +## requireddeps: value +## Dependencies: value +## dependencies: value +## OptionalDeps: value +## optionaldeps: value +## LoadOnDemand: value +## loadondemand: value +## LoadWith: value +## loadwith: value +## LoadManagers: value +## loadmanagers: value +## SavedVariablesPerCharacter: value +## savedvariablespercharacter: value +## SavedVariables: value +## savedvariables: value +## DefaultState: value +## defaultstate: value +## Secure: value +## secure: value +## Author: value +## author: value +## Version: value +## version: value +## Dep: value +## dep: value +## DepSomething: value diff --git a/tests/examplefiles/wowtoc/official_tags.toc.output b/tests/examplefiles/wowtoc/official_tags.toc.output new file mode 100644 index 0000000000..90037ea042 --- /dev/null +++ b/tests/examplefiles/wowtoc/official_tags.toc.output @@ -0,0 +1,503 @@ +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-ptBR' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-zhCN' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-enCN' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-frFR' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-deDE' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-itIT' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-esMX' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-ptPT' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-koKR' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-ruRU' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-esES' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-zhTW' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-enTW' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-enGB' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'TiTlE-enUS' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-ptBR' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-zhCN' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-enCN' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-frFR' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-deDE' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-itIT' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-esMX' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-ptPT' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-koKR' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-ruRU' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-esES' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-zhTW' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-enTW' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-enGB' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoTeS-enUS' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Interface' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'interface' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Title' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'title' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Notes' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'notes' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'RequiredDeps' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'requireddeps' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Dependencies' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'dependencies' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'OptionalDeps' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'optionaldeps' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'LoadOnDemand' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'loadondemand' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'LoadWith' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'loadwith' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'LoadManagers' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'loadmanagers' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'SavedVariablesPerCharacter' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'savedvariablespercharacter' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'SavedVariables' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'savedvariables' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'DefaultState' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'defaultstate' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Secure' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'secure' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Author' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'author' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Version' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'version' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Dep' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'dep' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'DepSomething' Name.Builtin +':' Punctuation +' ' Text.Whitespace +'value' Literal.String +'\n' Text diff --git a/tests/examplefiles/wowtoc/other_tags.toc b/tests/examplefiles/wowtoc/other_tags.toc new file mode 100644 index 0000000000..30ed8478fd --- /dev/null +++ b/tests/examplefiles/wowtoc/other_tags.toc @@ -0,0 +1,6 @@ +## X-Foo: Value +## Non-Conforming: Value +##no:whitespace +## plenty : of whitespace +## NoValueOk: +## other intermediate non-space whitespace: is ok diff --git a/tests/examplefiles/wowtoc/other_tags.toc.output b/tests/examplefiles/wowtoc/other_tags.toc.output new file mode 100644 index 0000000000..15acbb525a --- /dev/null +++ b/tests/examplefiles/wowtoc/other_tags.toc.output @@ -0,0 +1,45 @@ +'##' Keyword.Declaration +' ' Text.Whitespace +'X-Foo' Name.Variable +':' Punctuation +' ' Text.Whitespace +'Value' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'Non-Conforming' Name.Other +':' Punctuation +' ' Text.Whitespace +'Value' Literal.String +'\n' Text + +'##' Keyword.Declaration +'no' Name.Other +':' Punctuation +'whitespace' Literal.String +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'plenty' Name.Other +' ' Text.Whitespace +':' Punctuation +' ' Text.Whitespace +'of whitespace' Literal.String +' ' Text.Whitespace +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'NoValueOk' Name.Other +':' Punctuation +'\n' Text + +'##' Keyword.Declaration +' ' Text.Whitespace +'other\tintermediate\tnon-space\twhitespace' Name.Other +':' Punctuation +' ' Text.Whitespace +'is ok' Literal.String +'\n' Text From d3354dc37510f1a633f16b8104ae124594e3a88a Mon Sep 17 00:00:00 2001 From: Tim Martin Date: Wed, 21 Sep 2022 10:22:10 -0500 Subject: [PATCH 02/11] Apply suggestions from code review Co-authored-by: Jean Abou-Samra --- pygments/lexers/wowtoc.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pygments/lexers/wowtoc.py b/pygments/lexers/wowtoc.py index 711fe51c0a..831a9fb790 100644 --- a/pygments/lexers/wowtoc.py +++ b/pygments/lexers/wowtoc.py @@ -74,9 +74,9 @@ class WoWTocLexer(RegexLexer): ), # Comments - (r"^#[^\n]*$", Comment), + (r"^#.*$", Comment), # Addon Files - (r"^[^\n]+$", Name), + (r"^.+$", Name), ] } From 5653bad90d7fdc2da9e68a096d68f17dbd3b2196 Mon Sep 17 00:00:00 2001 From: Tim Martin Date: Wed, 21 Sep 2022 10:31:33 -0500 Subject: [PATCH 03/11] Update pygments/lexers/wowtoc.py Co-authored-by: Jean Abou-Samra --- pygments/lexers/wowtoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/wowtoc.py b/pygments/lexers/wowtoc.py index 831a9fb790..a204853f71 100644 --- a/pygments/lexers/wowtoc.py +++ b/pygments/lexers/wowtoc.py @@ -21,7 +21,7 @@ def _create_tag_line_token(inner_pattern, inner_token, ignore_case=False): (r"(?i)" if ignore_case else r"") + r"^(##)( *)" + inner_pattern - + r"( *)(:)( *)([^\n]*?)( *)$", + + r"( *)(:)( *)(.*?)( *)$", bygroups( Keyword.Declaration, Text.Whitespace, From 1e31dd983398715c6b2d479331929f7e7662f1f4 Mon Sep 17 00:00:00 2001 From: Tim Martin Date: Wed, 21 Sep 2022 10:29:20 -0500 Subject: [PATCH 04/11] Formatting --- pygments/lexers/wowtoc.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pygments/lexers/wowtoc.py b/pygments/lexers/wowtoc.py index a204853f71..ad5a0af113 100644 --- a/pygments/lexers/wowtoc.py +++ b/pygments/lexers/wowtoc.py @@ -13,6 +13,7 @@ __all__ = ["WoWTocLexer"] + def _create_tag_line_token(inner_pattern, inner_token, ignore_case=False): # this function template-izes the tag line for a specific type of tag, which will # have a different pattern and different token. otherwise, everything about a tag @@ -58,19 +59,18 @@ class WoWTocLexer(RegexLexer): _create_tag_line_token( r"(Interface|Title|Notes|RequiredDeps|Dep[^: ]*|OptionalDeps|LoadOnDemand|LoadWith|LoadManagers|SavedVariablesPerCharacter|SavedVariables|DefaultState|Secure|Author|Version)", Name.Builtin, - ignore_case=True + ignore_case=True, ), # user-defined tags _create_tag_line_token( r"(x-[^: ]*)", Name.Variable, - ignore_case=True + ignore_case=True, ), # non-conforming tags, but still valid _create_tag_line_token( r"([^: ]*)", Name.Other, - ignore_case=True ), # Comments From b9624660138dbaf8ea6e190cebd02e2c13946d99 Mon Sep 17 00:00:00 2001 From: Tim Martin Date: Wed, 21 Sep 2022 10:41:04 -0500 Subject: [PATCH 05/11] Upper case X more canonical --- pygments/lexers/wowtoc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/wowtoc.py b/pygments/lexers/wowtoc.py index ad5a0af113..9d58916a85 100644 --- a/pygments/lexers/wowtoc.py +++ b/pygments/lexers/wowtoc.py @@ -63,7 +63,7 @@ class WoWTocLexer(RegexLexer): ), # user-defined tags _create_tag_line_token( - r"(x-[^: ]*)", + r"(X-[^: ]*)", Name.Variable, ignore_case=True, ), From f2262ac3f9c6fa80a6fefca13ff3da2aeb5e0186 Mon Sep 17 00:00:00 2001 From: Tim Martin Date: Wed, 21 Sep 2022 12:02:03 -0500 Subject: [PATCH 06/11] Add WoW TOC analyse_text --- pygments/lexers/wowtoc.py | 40 +++++++++++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/pygments/lexers/wowtoc.py b/pygments/lexers/wowtoc.py index 9d58916a85..cc3e0f6cd5 100644 --- a/pygments/lexers/wowtoc.py +++ b/pygments/lexers/wowtoc.py @@ -7,22 +7,26 @@ :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. """ +import re from pygments.lexer import RegexLexer, bygroups from pygments.token import Comment, Name, Text, Punctuation, String, Keyword __all__ = ["WoWTocLexer"] +def _create_tag_line_pattern(inner_pattern, ignore_case=False): + return ((r"(?i)" if ignore_case else r"") + + r"^(##)( *)" + + inner_pattern + + r"( *)(:)( *)(.*?)( *)$") + def _create_tag_line_token(inner_pattern, inner_token, ignore_case=False): # this function template-izes the tag line for a specific type of tag, which will # have a different pattern and different token. otherwise, everything about a tag # line is the same return ( - (r"(?i)" if ignore_case else r"") - + r"^(##)( *)" - + inner_pattern - + r"( *)(:)( *)(.*?)( *)$", + _create_tag_line_pattern(inner_pattern, ignore_case=ignore_case), bygroups( Keyword.Declaration, Text.Whitespace, @@ -80,3 +84,31 @@ class WoWTocLexer(RegexLexer): (r"^.+$", Name), ] } + + def analyse_text(text): + # at time of writing, this file suffix conflict's with one of Tex's in + # markup.py. Tex's anaylse_text() appears to be definitive (binary) and does not + # share any likeness to WoW TOCs, which means we wont have to compete with it by + # abitrary increments in score. + + result = 0 + + # while not required, an almost certain marker of WoW TOC's is the interface tag + # if this tag is omitted, players will need to opt-in to loading the addon with + # an options change ("Load out of date addons"). the value is also standardized: + # ``, with minor and patch being two-digit zero-padded. + interface_pattern = _create_tag_line_pattern(r"(Interface)", ignore_case=True) + match = re.search(interface_pattern, text) + if match and re.match(r"(\d+)(\d{2})(\d{2})", match.group(7) or ""): + result += 0.8 + + casefolded = text.casefold() + # Lua file listing is good marker too, but probably conflicts with many other + # lexers + if ".lua" in casefolded: + result += 0.1 + # ditto for XML files, but they're less used in WoW TOCs + if ".xml" in casefolded: + result += 0.05 + + return result From a6a0749d6a8738966af5e8452d6a787b9e87c39c Mon Sep 17 00:00:00 2001 From: Tim Martin Date: Wed, 21 Sep 2022 12:02:53 -0500 Subject: [PATCH 07/11] Use "python" in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 427e5a47b0..87c05c7b4b 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ # :license: BSD, see LICENSE for details. # -PYTHON ?= python3 +PYTHON ?= python export PYTHONPATH = $(shell echo "$$PYTHONPATH"):$(shell python -c 'import os; print ":".join(os.path.abspath(line.strip()) for line in file("PYTHONPATH"))' 2>/dev/null) From 39ee98d162c7447f2e28a4d3628e410bf56528d4 Mon Sep 17 00:00:00 2001 From: Tim Martin Date: Wed, 21 Sep 2022 12:03:14 -0500 Subject: [PATCH 08/11] Decode utf-8 in count_token_references --- scripts/count_token_references.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/count_token_references.py b/scripts/count_token_references.py index e235cc40a8..8e798c2de0 100755 --- a/scripts/count_token_references.py +++ b/scripts/count_token_references.py @@ -72,7 +72,10 @@ def fetch_lexer_sources(): to a list of lines. """ lexer_dir = (pathlib.Path(__file__).parent / "../pygments/lexers").resolve() - lexer_sources = {fn: fn.read_text().splitlines(keepends=False) for fn in lexer_dir.glob("*.py")} + lexer_sources = { + fn: fn.read_text(encoding='utf-8').splitlines(keepends=False) + for fn in lexer_dir.glob("*.py") + } return lexer_sources From 35586304c9dbbc0e3e1606dc6d4bc57e54d0e13b Mon Sep 17 00:00:00 2001 From: Tim Martin Date: Wed, 21 Sep 2022 12:03:58 -0500 Subject: [PATCH 09/11] Update broken Contributing.md URL --- Contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Contributing.md b/Contributing.md index 0d831f0b1f..93da428ca2 100644 --- a/Contributing.md +++ b/Contributing.md @@ -11,7 +11,7 @@ Goals & non-goals of Pygments Python support -------------- -Pygments supports all supported Python versions as per the [Python Developer's Guide](https://devguide.python.org/#status-of-python-branches). Additionally, the default Python version of the latest stable version of RHEL, Ubuntu LTS, and Debian are supported, even if they're officially EOL. Supporting other end-of-life versions is a non-goal of Pygments. +Pygments supports all supported Python versions as per the [Python Developer's Guide](https://devguide.python.org/versions/). Additionally, the default Python version of the latest stable version of RHEL, Ubuntu LTS, and Debian are supported, even if they're officially EOL. Supporting other end-of-life versions is a non-goal of Pygments. Validation ---------- From a5fdd70a6b79308dbb2f834cab472a3953eb440c Mon Sep 17 00:00:00 2001 From: Tim Martin Date: Wed, 21 Sep 2022 12:52:00 -0500 Subject: [PATCH 10/11] Revert "Use "python" in Makefile" This reverts commit a6a0749d6a8738966af5e8452d6a787b9e87c39c. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 87c05c7b4b..427e5a47b0 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ # :license: BSD, see LICENSE for details. # -PYTHON ?= python +PYTHON ?= python3 export PYTHONPATH = $(shell echo "$$PYTHONPATH"):$(shell python -c 'import os; print ":".join(os.path.abspath(line.strip()) for line in file("PYTHONPATH"))' 2>/dev/null) From 102de2745554c09c3525b1d400062e1d2c675889 Mon Sep 17 00:00:00 2001 From: Tim Martin Date: Wed, 21 Sep 2022 12:52:31 -0500 Subject: [PATCH 11/11] Group 7 will always be present --- pygments/lexers/wowtoc.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pygments/lexers/wowtoc.py b/pygments/lexers/wowtoc.py index cc3e0f6cd5..615493cdf3 100644 --- a/pygments/lexers/wowtoc.py +++ b/pygments/lexers/wowtoc.py @@ -16,9 +16,9 @@ def _create_tag_line_pattern(inner_pattern, ignore_case=False): return ((r"(?i)" if ignore_case else r"") - + r"^(##)( *)" - + inner_pattern - + r"( *)(:)( *)(.*?)( *)$") + + r"^(##)( *)" # groups 1, 2 + + inner_pattern # group 3 + + r"( *)(:)( *)(.*?)( *)$") # groups 4, 5, 6, 7, 8 def _create_tag_line_token(inner_pattern, inner_token, ignore_case=False): @@ -99,7 +99,7 @@ def analyse_text(text): # ``, with minor and patch being two-digit zero-padded. interface_pattern = _create_tag_line_pattern(r"(Interface)", ignore_case=True) match = re.search(interface_pattern, text) - if match and re.match(r"(\d+)(\d{2})(\d{2})", match.group(7) or ""): + if match and re.match(r"(\d+)(\d{2})(\d{2})", match.group(7)): result += 0.8 casefolded = text.casefold()