Add org-mode lexer (#2636)

Co-authored-by: Gosha Tcherednitchenko <git@gosha.net>

pygments/lexers/_mapping.py

@@ -365,6 +365,7 @@
     'OpaLexer': ('pygments.lexers.ml', 'Opa', ('opa',), ('*.opa',), ('text/x-opa',)),
     'OpenEdgeLexer': ('pygments.lexers.business', 'OpenEdge ABL', ('openedge', 'abl', 'progress'), ('*.p', '*.cls'), ('text/x-openedge', 'application/x-openedge')),
     'OpenScadLexer': ('pygments.lexers.openscad', 'OpenSCAD', ('openscad',), ('*.scad',), ('application/x-openscad',)),
+    'OrgLexer': ('pygments.lexers.markup', 'Org Mode', ('org', 'orgmode', 'org-mode'), ('*.org',), ('text/org',)),
     'OutputLexer': ('pygments.lexers.special', 'Text output', ('output',), (), ()),
     'PacmanConfLexer': ('pygments.lexers.configs', 'PacmanConf', ('pacmanconf',), ('pacman.conf',), ()),
     'PanLexer': ('pygments.lexers.dsls', 'Pan', ('pan',), ('*.pan',), ()),

pygments/lexers/markup.py

@@ -19,13 +19,14 @@
 from pygments.lexer import RegexLexer, DelegatingLexer, include, bygroups, \
     using, this, do_insertions, default, words
 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
-    Number, Punctuation, Generic, Other, Whitespace
+    Number, Punctuation, Generic, Other, Whitespace, Literal
 from pygments.util import get_bool_opt, ClassNotFound
 
 __all__ = ['BBCodeLexer', 'MoinWikiLexer', 'RstLexer', 'TexLexer', 'GroffLexer',
            'MozPreprocHashLexer', 'MozPreprocPercentLexer',
            'MozPreprocXulLexer', 'MozPreprocJavascriptLexer',
-           'MozPreprocCssLexer', 'MarkdownLexer', 'TiddlyWiki5Lexer', 'WikitextLexer']
+           'MozPreprocCssLexer', 'MarkdownLexer', 'OrgLexer', 'TiddlyWiki5Lexer',
+           'WikitextLexer']
 
 
 class BBCodeLexer(RegexLexer):
@@ -617,6 +618,114 @@ def __init__(self, **options):
         self.handlecodeblocks = get_bool_opt(options, 'handlecodeblocks', True)
         RegexLexer.__init__(self, **options)
 
+class OrgLexer(RegexLexer):
+    """
+    For Org Mode markup.
+    """
+    name = 'Org Mode'
+    url = 'https://orgmode.org'
+    aliases = ['org', 'orgmode', 'org-mode']
+    filenames = ['*.org']
+    mimetypes = ["text/org"]
+    version_added = '2.18'
+
+    def _inline(start, end):
+        return rf'(?<!\w){start}(.|\n(?!\n))+?{end}(?!\w)'
+
+    tokens = {
+        'root': [
+            (r'^# .*', Comment.Single),
+
+            # Headings
+            (r'^(\* )(COMMENT)( .*)',
+             bygroups(Generic.Heading, Comment.Preproc, Generic.Heading)),
+            (r'^(\*\*+ )(COMMENT)( .*)',
+             bygroups(Generic.Subheading, Comment.Preproc, Generic.Subheading)),
+            (r'^(\* )(DONE)( .*)',
+             bygroups(Generic.Heading, Generic.Deleted, Generic.Heading)),
+            (r'^(\*\*+ )(DONE)( .*)',
+             bygroups(Generic.Subheading, Generic.Deleted, Generic.Subheading)),
+            (r'^(\* )(TODO)( .*)',
+             bygroups(Generic.Heading, Generic.Error, Generic.Heading)),
+            (r'^(\*\*+ )(TODO)( .*)',
+             bygroups(Generic.Subheading, Generic.Error, Generic.Subheading)),
+
+            (r'^(\* .+?)( :[a-zA-Z0-9_@:]+:)?$', bygroups(Generic.Heading, Generic.Emph)),
+            (r'^(\*\*+ .+?)( :[a-zA-Z0-9_@:]+:)?$', bygroups(Generic.Subheading, Generic.Emph)),
+
+            # Unordered lists items, including TODO items and description items
+            (r'^(?:( *)([+-] )|( +)(\* ))(\[[ X-]\])?(.+ ::)?',
+             bygroups(Whitespace, Keyword, Whitespace, Keyword, Generic.Prompt, Name.Label)),
+
+            # Ordered list items
+            (r'^( *)([0-9]+[.)])( \[@[0-9]+\])?', bygroups(Whitespace, Keyword, Generic.Emph)),
+
+            # Dynamic blocks
+            (r'(?i)^( *#\+begin: *)((?:.|\n)*?)(^ *#\+end: *$)',
+             bygroups(Operator.Word, using(this), Operator.Word)),
+
+            # Comment blocks
+            (r'(?i)^( *#\+begin_comment *\n)((?:.|\n)*?)(^ *#\+end_comment *$)',
+             bygroups(Operator.Word, Comment.Multiline, Operator.Word)),
+
+            # Source code blocks
+            # TODO: language-dependent syntax highlighting (see Markdown lexer)
+            (r'(?i)^( *#\+begin_src .*)((?:.|\n)*?)(^ *#\+end_src *$)',
+             bygroups(Operator.Word, Text, Operator.Word)),
+
+            # Other blocks
+            (r'(?i)^( *#\+begin_\w+)( *\n)((?:.|\n)*?)(^ *#\+end_\w+)( *$)',
+             bygroups(Operator.Word, Whitespace, Text, Operator.Word, Whitespace)),
+
+            # Keywords
+            (r'^(#\+\w+:)(.*)$', bygroups(Name.Namespace, Text)),
+
+            # Properties and drawers
+            (r'(?i)^( *:\w+: *\n)((?:.|\n)*?)(^ *:end: *$)',
+             bygroups(Name.Decorator, Comment.Special, Name.Decorator)),
+
+            # Line break operator
+            (r'\\\\$', Operator),
+
+            # Deadline, Scheduled, CLOSED
+            (r'(?i)^( *(?:DEADLINE|SCHEDULED): )(<.+?> *)$',
+             bygroups(Generic.Error, Literal.Date)),
+            (r'(?i)^( *CLOSED: )(\[.+?\] *)$',
+             bygroups(Generic.Deleted, Literal.Date)),
+
+            # Bold
+            (_inline(r'\*', r'\*+'), Generic.Strong),
+            # Italic
+            (_inline(r'/', r'/'), Generic.Emph),
+            # Verbatim
+            (_inline(r'=', r'='), String), # TODO token
+            # Code
+            (_inline(r'~', r'~'), String),
+            # Strikethrough
+            (_inline(r'\+', r'\+'), Generic.Deleted),
+            # Underline
+            (_inline(r'_', r'_+'), Generic.EmphStrong),
+
+            # Dates
+            (r'<.+?>', Literal.Date),
+            # Macros
+            (r'\{\{\{.+?\}\}\}', Comment.Preproc),
+            # Footnotes
+            (r'(?<!\[)\[fn:.+?\]', Name.Tag),
+            # Links
+            (r'(?s)(\[\[)(.*?)(\]\[)(.*?)(\]\])',
+             bygroups(Punctuation, Name.Attribute, Punctuation, Name.Tag, Punctuation)),
+            (r'(?s)(\[\[)(.+?)(\]\])', bygroups(Punctuation, Name.Attribute, Punctuation)),
+            (r'(<<)(.+?)(>>)', bygroups(Punctuation, Name.Attribute, Punctuation)),
+
+            # Tables
+            (r'^( *)(\|[ -].*?[ -]\|)$', bygroups(Whitespace, String)),
+
+            # Any other text
+            (r'[^#*+\-0-9:\\/=~_<{\[|\n]+', Text),
+            (r'[#*+\-0-9:\\/=~_<{\[|\n]', Text),
+        ],
+    }
 
 class TiddlyWiki5Lexer(RegexLexer):
     """

tests/examplefiles/org/example.org

@@ -0,0 +1,149 @@
+# -*- org -*-
+#+title: Single Post with TOML front matter
+#+author:
+#+date: 2017-07-20
+#+options: creator:t toc:2
+
+#+hugo_base_dir: ../../
+#+hugo_categories: cat1 cat2
+#+hugo_menu: :menu "foo" :weight 10 :parent main :identifier single-toml
+#+description: Some description for this post.
+
+This is a single post. You do not need to set the =EXPORT_FILE_NAME=
+property in here. But then you also lose the tag and property
+inheritance Org awesomeness.
+
+line one \\
+line two
+
+*bold* /italics/ =verbatim= ~code~ +strikethrough+ _underline_
+* First heading in this post
+** Subheading :TAG:
+This is a under first heading. <2018-07-27 Fri>
+* COMMENT heading
+** COMMENT subheading
+* TODO heading
+** TODO subheading
+* DONE heading
+** DONE subheading
+CLOSED: [2018-08-01 Wed 16:17]
+
+{{{n}}} abc {{{n(,10)}}} def {{{n(x,5)}}}
+
+- unordered list item 1
+- unordered list item 2
+- Desc1 :: Description list item 1
+- Desc2 :: Description list item 2
+
+1. ordered list item 1
+2. ordered list item 2
+
+
+1) ordered list item 1
+2) ordered list item 2
+3) [@20] asdf
+
+
+- *bold*
+- /italics/
+- =verbatim=
+- ~code~
+- +strikethrough+
+- _underline_
+
+#+begin_comment
+this is a
+comment
+#+end_comment
+
+- 09 :: asdf
+- 09 :: asdf
+** Checkbox lists                                             :checkbox:list:
+- [ ] asdf
+- [X] def
+* Dates
+** deadline
+DEADLINE: <2018-07-31 Tue>
+something
+** scheduled
+SCHEDULED: <2018-07-31 Tue>
+something
+* Blocks
+#+BEGIN_QUOTE
+quote
+block
+#+END_QUOTE
+
+#+begin_example
+example
+block
+#+end_example
+
+#+begin_src emacs-lisp
+(message "hello")
+#+end_src
+
+#+begin_details
+#+begin_summary
+Why is this in *green*?
+#+end_summary
+You will learn that later below in [[#details-css][CSS]] section.
+#+end_details
+
+#+begin_export html
+<style>
+.my-table th,
+.my-table td {
+    padding: 20px;
+    text-align: left;
+}
+</style>
+#+end_export
+
+- {{{latex}}}
+- {{{xetex}}}
+#+caption: Declaring Constants
+#+name: code__decl_constants
+#+begin_src systemverilog
+const bit [7:0] R_CHARACTER = 8'b000_11100; // The /K28.0/ character
+const bit [7:0] A_CHARACTER = 8'b011_11100; // The /K28.3/ character
+const bit [7:0] Q_CHARACTER = 8'b100_11100; // The /K28.4/ character
+#+end_src
+Check out code snippet [[code__decl_constants]].
+* Links                                                            :some_tag:
+:properties:
+:CUSTOM_ID: links
+:end:
+- [[file:post-yaml.org]]
+- [[file:post-yaml.org][Post exported with YAML front-matter]].
+- <<target>>
+* Tables
+:PROPERTIES:
+:CUSTOM_ID: tables
+:END:
+| a | b |
+| c | d |
+
+|---+---|
+| a | b |
+|---+---|
+| c | d |
+|---+---|
+
+Some text [fn:2]
+
+#+BEGIN: aggregate :table "original" :cols "Color count()"
+| Color | count() |
+|-------+---------|
+| Red   |       7 |
+| Blue  |       7 |
+#+END:
+* Footnotes
+
+[fn:2] footnote 2
+[fn:1] For more detail, check out the Org manual [[http://orgmode.org/
+manual/Footnotes.html][page for footnotes]].
+* Local Variables :ARCHIVE:
+# Local Variables:
+# org-link-file-path-type: relative
+# End: