Merge branch 'master' into rtf-linenos

AUTHORS

@@ -198,6 +198,7 @@ Other contributors, listed alphabetically, are:
 * Kashif Rasul -- CUDA lexer
 * Nathan Reed -- HLSL lexer
 * Justin Reidy -- MXML lexer
+* Jonathon Reinhart, Google LLC -- Soong lexer
 * Norman Richards -- JSON lexer
 * Corey Richardson -- Rust lexer updates
 * Fabrizio Riguzzi -- cplint leder
@@ -277,6 +278,6 @@ Other contributors, listed alphabetically, are:
 * vanillajonathan -- PRQL lexer
 * Nikolay Antipov -- OpenSCAD lexer
 * Markus Meyer, Nextron Systems -- YARA lexer
-
+* Hannes Römer -- Mojo lexer
 
 Many thanks for all contributions!

CHANGES

@@ -12,11 +12,17 @@ Version 2.18.0
   * Janet (#2557)
   * Typst (#2596)
   * Lean 4 (#2618, #2626)
+  * Mojo (#2691, #2515)
+  * org-mode (#2636)
   * Promela (#2620)
+  * Soong / ``Android.bp`` (#2659)
 
 - Updated lexers:
 
+  * Awk: recognize ternary operator (#2687)
   * Bash: add ``openrc`` alias (#2599)
+  * Coq: add keywords, lex more vernacular command arguments, produce
+    fewer tokens on heading comments (#2678)
   * lean: Fix name handling (#2614)
   * Logtalk: add ``uninstantiation`` keyword and recognize
     escape sequences (#2619)

doc/conf.py

@@ -230,7 +230,7 @@
 # Example configuration for intersphinx: refer to the Python standard library.
 #intersphinx_mapping = {'http://docs.python.org/': None}
 
-rst_prolog = '.. |language_count| replace:: {}'.format(len(list(pygments.lexers.get_all_lexers())))
+rst_prolog = f'.. |language_count| replace:: {len(list(pygments.lexers.get_all_lexers()))}'
 
 def pg_context(app, pagename, templatename, ctx, event_arg):
     ctx['demo_active'] = bool(os.environ.get('WEBSITE_BUILD'))
@@ -284,7 +284,7 @@ def source_read(app, docname, source):
 
         def linkify(match):
             url = 'https://github.com/pygments/pygments/issues/' + match[1]
-            return '`{} <{}>`_'.format(match[0], url)
+            return f'`{match[0]} <{url}>`_'
 
         linkified = re.sub(r'(?:PR)?#([0-9]+)\b', linkify, changelog[:idx])
         source[0] = linkified + changelog[idx:]

doc/docs/cmdline.rst

@@ -183,7 +183,7 @@ will list only all installed filters.
 .. versionadded:: 2.11
 
 The ``--json`` option can be used in conjunction with the ``-L`` option to
-output it's contents as JSON. Thus, to print all the installed styles and their
+output its contents as JSON. Thus, to print all the installed styles and their
 description in JSON, use the command::
 
     $ pygmentize -L styles --json

doc/docs/lexerdevelopment.rst

@@ -39,7 +39,7 @@ To add a lexer, you have to perform the following steps:
 
     __all__ = ['AutohotkeyLexer', 'AutoItLexer']
 
-  Add the name of your lexer class to this list (or create the list if your 
+  Add the name of your lexer class to this list (or create the list if your
   lexer   is the only class in the module).
 
 * Finally the lexer can be made publicly known by rebuilding the lexer mapping.
@@ -96,7 +96,7 @@ state.
 .. note::
 
     This means you're always jumping back to the first entry, i.e. you cannot match states in a particular order. For example, a state with the following rules won't work as intended:
-    
+
     .. code:: python
 
         'state': [
@@ -251,9 +251,9 @@ sections, comments and ``key = value`` pairs::
         tokens = {
             'root': [
                 (r'\s+', Text),
-                (r';.*?$', Comment),
+                (r';.*', Comment),
                 (r'\[.*?\]$', Keyword),
-                (r'(.*?)(\s*)(=)(\s*)(.*?)$',
+                (r'(.*?)(\s*)(=)(\s*)(.*)',
                  bygroups(Name.Attribute, Text, Operator, Text, String))
             ]
         }

doc/examples/example.py

@@ -11,4 +11,4 @@ def fib(n: int) -> Iterator[int]:
             a, b = b, a + b
 
 result = sum(Math.fib(42))
-print("The answer is {}".format(result))
+print(f"The answer is {result}")

pygments/cmdline.py

@@ -97,7 +97,7 @@ def _print_list(what):
             info.append(tup)
         info.sort()
         for i in info:
-            print(('* %s\n    %s %s') % i)
+            print(('* {}\n    {} {}').format(*i))
 
     elif what == 'formatter':
         print()
@@ -112,7 +112,7 @@ def _print_list(what):
             info.append(tup)
         info.sort()
         for i in info:
-            print(('* %s\n    %s %s') % i)
+            print(('* {}\n    {} {}').format(*i))
 
     elif what == 'filter':
         print()

pygments/formatter.py

@@ -122,3 +122,8 @@ def format(self, tokensource, outfile):
             # wrap the outfile in a StreamWriter
             outfile = codecs.lookup(self.encoding)[3](outfile)
         return self.format_unencoded(tokensource, outfile)
+
+    # Allow writing Formatter[str] or Formatter[bytes]. That's equivalent to
+    # Formatter. This helps when using third-party type stubs from typeshed.
+    def __class_getitem__(cls, name):
+        return cls

pygments/formatters/__init__.py

@@ -103,13 +103,12 @@ def load_formatter_from_file(filename, formattername="CustomFormatter", **option
             exec(f.read(), custom_namespace)
         # Retrieve the class `formattername` from that namespace
         if formattername not in custom_namespace:
-            raise ClassNotFound('no valid %s class found in %s' %
-                                (formattername, filename))
+            raise ClassNotFound(f'no valid {formattername} class found in {filename}')
         formatter_class = custom_namespace[formattername]
         # And finally instantiate it with the options
         return formatter_class(**options)
     except OSError as err:
-        raise ClassNotFound('cannot read %s: %s' % (filename, err))
+        raise ClassNotFound(f'cannot read {filename}: {err}')
     except ClassNotFound:
         raise
     except Exception as err:

pygments/formatters/html.py

@@ -530,7 +530,7 @@ def get_token_style_defs(self, arg=None):
         styles.sort()
 
         lines = [
-            '%s { %s } /* %s */' % (prefix(cls), style, repr(ttype)[6:])
+            f'{prefix(cls)} {{ {style} }} /* {repr(ttype)[6:]} */'
             for (level, ttype, cls, style) in styles
         ]
 
@@ -548,13 +548,13 @@ def get_background_style_defs(self, arg=None):
             if Text in self.ttype2class:
                 text_style = ' ' + self.class2style[self.ttype2class[Text]][0]
             lines.insert(
-                0, '%s{ background: %s;%s }' % (
+                0, '{}{{ background: {};{} }}'.format(
                     prefix(''), bg_color, text_style
                 )
             )
         if hl_color is not None:
             lines.insert(
-                0, '%s { background-color: %s }' % (prefix('hll'), hl_color)
+                0, '{} {{ background-color: {} }}'.format(prefix('hll'), hl_color)
             )
 
         return lines
@@ -594,17 +594,15 @@ def _pre_style(self):
 
     @property
     def _linenos_style(self):
-        return 'color: %s; background-color: %s; padding-left: 5px; padding-right: 5px;' % (
-            self.style.line_number_color,
-            self.style.line_number_background_color
-        )
+        color = self.style.line_number_color
+        background_color = self.style.line_number_background_color
+        return f'color: {color}; background-color: {background_color}; padding-left: 5px; padding-right: 5px;'
 
     @property
     def _linenos_special_style(self):
-        return 'color: %s; background-color: %s; padding-left: 5px; padding-right: 5px;' % (
-            self.style.line_number_special_color,
-            self.style.line_number_special_background_color
-        )
+        color = self.style.line_number_special_color
+        background_color = self.style.line_number_special_background_color
+        return f'color: {color}; background-color: {background_color}; padding-left: 5px; padding-right: 5px;'
 
     def _decodeifneeded(self, value):
         if isinstance(value, bytes):
@@ -695,7 +693,7 @@ def _wrap_tablelinenos(self, inner):
                     style = ' class="normal"'
 
             if style:
-                line = '<span%s>%s</span>' % (style, line)
+                line = f'<span{style}>{line}</span>'
 
             lines.append(line)
 
@@ -754,7 +752,7 @@ def _wrap_inlinelinenos(self, inner):
                     style = ' class="linenos"'
 
             if style:
-                linenos = '<span%s>%s</span>' % (style, line)
+                linenos = f'<span{style}>{line}</span>'
             else:
                 linenos = line
 
@@ -791,7 +789,7 @@ def _wrap_div(self, inner):
         style = []
         if (self.noclasses and not self.nobackground and
                 self.style.background_color is not None):
-            style.append('background: %s' % (self.style.background_color,))
+            style.append(f'background: {self.style.background_color}')
         if self.cssstyles:
             style.append(self.cssstyles)
         style = '; '.join(style)
@@ -848,13 +846,13 @@ def _format_lines(self, tokensource):
                     css_style = self._get_css_inline_styles(ttype)
                     if css_style:
                         css_style = self.class2style[css_style][0]
-                        cspan = '<span style="%s"%s>' % (css_style, title)
+                        cspan = f'<span style="{css_style}"{title}>'
                     else:
                         cspan = ''
                 else:
                     css_class = self._get_css_classes(ttype)
                     if css_class:
-                        cspan = '<span class="%s"%s>' % (css_class, title)
+                        cspan = f'<span class="{css_class}"{title}>'
                     else:
                         cspan = ''
                 self.span_element_openers[ttype] = cspan
@@ -927,9 +925,8 @@ def _highlight_lines(self, tokensource):
                 if self.noclasses:
                     style = ''
                     if self.style.highlight_color is not None:
-                        style = (' style="background-color: %s"' %
-                                 (self.style.highlight_color,))
-                    yield 1, '<span%s>%s</span>' % (style, value)
+                        style = (f' style="background-color: {self.style.highlight_color}"')
+                    yield 1, f'<span{style}>{value}</span>'
                 else:
                     yield 1, '<span class="hll">%s</span>' % value
             else:

pygments/formatters/img.py

@@ -90,7 +90,7 @@ def __init__(self, font_name, font_size=14):
             self._create_nix()
 
     def _get_nix_font_path(self, name, style):
-        proc = subprocess.Popen(['fc-list', "%s:style=%s" % (name, style), 'file'],
+        proc = subprocess.Popen(['fc-list', f"{name}:style={style}", 'file'],
                                 stdout=subprocess.PIPE, stderr=None)
         stdout, _ = proc.communicate()
         if proc.returncode == 0:
@@ -160,15 +160,14 @@ def _lookup_win(self, key, basename, styles, fail=False):
         for suffix in ('', ' (TrueType)'):
             for style in styles:
                 try:
-                    valname = '%s%s%s' % (basename, style and ' '+style, suffix)
+                    valname = '{}{}{}'.format(basename, style and ' '+style, suffix)
                     val, _ = _winreg.QueryValueEx(key, valname)
                     return val
                 except OSError:
                     continue
         else:
             if fail:
-                raise FontNotFound('Font %s (%s) not found in registry' %
-                                   (basename, styles[0]))
+                raise FontNotFound(f'Font {basename} ({styles[0]}) not found in registry')
             return None
 
     def _create_win(self):

pygments/formatters/latex.py

@@ -307,9 +307,8 @@ def rgbcolor(col):
                 cmndef += (r'\def\$$@tc##1{\textcolor[rgb]{%s}{##1}}' %
                            rgbcolor(ndef['color']))
             if ndef['border']:
-                cmndef += (r'\def\$$@bc##1{{\setlength{\fboxsep}{\string -\fboxrule}'
-                           r'\fcolorbox[rgb]{%s}{%s}{\strut ##1}}}' %
-                           (rgbcolor(ndef['border']),
+                cmndef += (r'\def\$$@bc##1{{{{\setlength{{\fboxsep}}{{\string -\fboxrule}}'
+                           r'\fcolorbox[rgb]{{{}}}{{{}}}{{\strut ##1}}}}}}'.format(rgbcolor(ndef['border']),
                             rgbcolor(ndef['bgcolor'])))
             elif ndef['bgcolor']:
                 cmndef += (r'\def\$$@bc##1{{\setlength{\fboxsep}{0pt}'
@@ -329,7 +328,7 @@ def get_style_defs(self, arg=''):
         cp = self.commandprefix
         styles = []
         for name, definition in self.cmd2def.items():
-            styles.append(r'\@namedef{%s@tok@%s}{%s}' % (cp, name, definition))
+            styles.append(rf'\@namedef{{{cp}@tok@{name}}}{{{definition}}}')
         return STYLE_TEMPLATE % {'cp': self.commandprefix,
                                  'styles': '\n'.join(styles)}
 
@@ -410,10 +409,10 @@ def format_unencoded(self, tokensource, outfile):
                 spl = value.split('\n')
                 for line in spl[:-1]:
                     if line:
-                        outfile.write("\\%s{%s}{%s}" % (cp, styleval, line))
+                        outfile.write(f"\\{cp}{{{styleval}}}{{{line}}}")
                     outfile.write('\n')
                 if spl[-1]:
-                    outfile.write("\\%s{%s}{%s}" % (cp, styleval, spl[-1]))
+                    outfile.write(f"\\{cp}{{{styleval}}}{{{spl[-1]}}}")
             else:
                 outfile.write(value)
 

pygments/formatters/other.py

@@ -147,7 +147,7 @@ def format(self, tokensource, outfile):
         outbuf = []
         for ttype, value in tokensource:
             rawbuf.append(value)
-            outbuf.append('%s(%s, %r),\n' % (indentation, ttype, value))
+            outbuf.append(f'{indentation}({ttype}, {value!r}),\n')
 
         before = TESTCASE_BEFORE % (''.join(rawbuf),)
         during = ''.join(outbuf)

pygments/formatters/svg.py

@@ -131,8 +131,7 @@ def format_unencoded(self, tokensource, outfile):
                           '"http://www.w3.org/TR/2001/REC-SVG-20010904/DTD/'
                           'svg10.dtd">\n')
             outfile.write('<svg xmlns="http://www.w3.org/2000/svg">\n')
-            outfile.write('<g font-family="%s" font-size="%s">\n' %
-                          (self.fontfamily, self.fontsize))
+            outfile.write(f'<g font-family="{self.fontfamily}" font-size="{self.fontsize}">\n')
 
         counter = self.linenostart
         counter_step = self.linenostep
@@ -141,12 +140,11 @@ def format_unencoded(self, tokensource, outfile):
 
         if self.linenos:
             if counter % counter_step == 0:
-                outfile.write('<text x="%s" y="%s" %s text-anchor="end">%s</text>' %
-                    (x+self.linenowidth,y,counter_style,counter))
+                outfile.write(f'<text x="{x+self.linenowidth}" y="{y}" {counter_style} text-anchor="end">{counter}</text>')
             line_x += self.linenowidth + self.ystep
             counter += 1
 
-        outfile.write('<text x="%s" y="%s" xml:space="preserve">' % (line_x, y))
+        outfile.write(f'<text x="{line_x}" y="{y}" xml:space="preserve">')
         for ttype, value in tokensource:
             style = self._get_style(ttype)
             tspan = style and '<tspan' + style + '>' or ''
@@ -160,11 +158,10 @@ def format_unencoded(self, tokensource, outfile):
                 y += self.ystep
                 outfile.write('</text>\n')
                 if self.linenos and counter % counter_step == 0:
-                    outfile.write('<text x="%s" y="%s" text-anchor="end" %s>%s</text>' %
-                        (x+self.linenowidth,y,counter_style,counter))
+                    outfile.write(f'<text x="{x+self.linenowidth}" y="{y}" text-anchor="end" {counter_style}>{counter}</text>')
 
                 counter += 1
-                outfile.write('<text x="%s" y="%s" ' 'xml:space="preserve">' % (line_x,y))
+                outfile.write(f'<text x="{line_x}" y="{y}" ' 'xml:space="preserve">')
             outfile.write(tspan + parts[-1] + tspanend)
         outfile.write('</text>')
 

pygments/lexer.py

@@ -174,8 +174,7 @@ def __init__(self, **options):
 
     def __repr__(self):
         if self.options:
-            return '<pygments.lexers.%s with %r>' % (self.__class__.__name__,
-                                                     self.options)
+            return f'<pygments.lexers.{self.__class__.__name__} with {self.options!r}>'
         else:
             return '<pygments.lexers.%s>' % self.__class__.__name__
 
@@ -511,7 +510,7 @@ def _process_regex(cls, regex, rflags, state):
     def _process_token(cls, token):
         """Preprocess the token component of a token definition."""
         assert type(token) is _TokenType or callable(token), \
-            'token type must be simple type or callable, not %r' % (token,)
+            f'token type must be simple type or callable, not {token!r}'
         return token
 
     def _process_new_state(cls, new_state, unprocessed, processed):
@@ -579,8 +578,7 @@ def _process_state(cls, unprocessed, processed, state):
             try:
                 rex = cls._process_regex(tdef[0], rflags, state)
             except Exception as err:
-                raise ValueError("uncompilable regex %r in state %r of %r: %s" %
-                                 (tdef[0], state, cls, err)) from err
+                raise ValueError(f"uncompilable regex {tdef[0]!r} in state {state!r} of {cls!r}: {err}") from err
 
             token = cls._process_token(tdef[1])
 
@@ -773,8 +771,7 @@ def __init__(self, text, pos, stack=None, end=None):
         self.stack = stack or ['root']
 
     def __repr__(self):
-        return 'LexerContext(%r, %r, %r)' % (
-            self.text, self.pos, self.stack)
+        return f'LexerContext({self.text!r}, {self.pos!r}, {self.stack!r})'
 
 
 class ExtendedRegexLexer(RegexLexer):