Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add a Lexer for Numba IR #2433

Draft
wants to merge 5 commits into
base: master
Choose a base branch
from
Draft

Add a Lexer for Numba IR #2433

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
e5eabbd
Add a Lexer for Numba Intermediate Representation (IR)
Matt711 May 16, 2023
98416b7
Add docstring, mimetypes list, and url
Matt711 May 17, 2023
22a7849
Merge remote-tracking branch 'upstream/master' into numbair
Matt711 May 25, 2023
762b3f0
Add NVIDIA copyright
Matt711 May 25, 2023
713867c
revert
Matt711 May 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions pygments/lexers/_mapping.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,7 @@
'NotmuchLexer': ('pygments.lexers.textfmts', 'Notmuch', ('notmuch',), (), ()),
'NuSMVLexer': ('pygments.lexers.smv', 'NuSMV', ('nusmv',), ('*.smv',), ()),
'NumPyLexer': ('pygments.lexers.python', 'NumPy', ('numpy',), (), ()),
'NumbaIRLexer': ('pygments.lexers.numbair', 'Numba_IR', ('numba_ir', 'numbair'), ('*.numba_ir',), ('text/x-numba_ir', 'text/x-numbair')),
'ObjdumpLexer': ('pygments.lexers.asm', 'objdump', ('objdump',), ('*.objdump',), ('text/x-objdump',)),
'ObjectiveCLexer': ('pygments.lexers.objective', 'Objective-C', ('objective-c', 'objectivec', 'obj-c', 'objc'), ('*.m', '*.h'), ('text/x-objective-c',)),
'ObjectiveCppLexer': ('pygments.lexers.objective', 'Objective-C++', ('objective-c++', 'objectivec++', 'obj-c++', 'objc++'), ('*.mm', '*.hh'), ('text/x-objective-c++',)),
Expand Down
69 changes: 69 additions & 0 deletions pygments/lexers/numbair.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
"""
pygments.lexers.numbair
~~~~~~~~~~~~~~~~~~~~~~~

Lexer for other Numba Intermediate Representation.

:copyright: Copyright 2006-2023 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""

from pygments.styles.default import DefaultStyle
from pygments.styles.manni import ManniStyle
from pygments.styles.monokai import MonokaiStyle
from pygments.styles.native import NativeStyle

from pygments.lexer import RegexLexer, include, bygroups, words
from pygments.token import Text, Name, String, Punctuation, Keyword, \
Operator, Number

from pygments.style import Style

__all__ = ["NumbaIRLexer"]

class NumbaIRLexer(RegexLexer):
"""
Lexer for Numba IR
"""
name = 'Numba_IR'
url = "https://numba.readthedocs.io/en/stable/developer/architecture.html#stage-2-generate-the-numba-ir"
aliases = ['numba_ir', 'numbair']
filenames = ['*.numba_ir']
mimetypes = ['text/x-numba_ir', 'text/x-numbair']

identifier = r'\$[a-zA-Z0-9._]+'
fun_or_var = r'([a-zA-Z_]+[a-zA-Z0-9]*)'

tokens = {
'root' : [
(r'(label)(\ [0-9]+)(:)$',
bygroups(Keyword, Name.Label, Punctuation)),

(r' = ', Operator),
include('whitespace'),
include('keyword'),

(identifier, Name.Variable),
(fun_or_var + r'(\()',
bygroups(Name.Function, Punctuation)),
(fun_or_var + r'(\=)',
bygroups(Name.Attribute, Punctuation)),
(fun_or_var, Name.Constant),
(r'[0-9]+', Number),

# <built-in function some>
(r'<[^>\n]*>', String),

(r'[=<>{}\[\]()*.,!\':]|x\b', Punctuation)
],

'keyword':[
(words((
'del', 'jump', 'call', 'branch',
), suffix=' '), Keyword),
Comment on lines +61 to +63
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
(words((
'del', 'jump', 'call', 'branch',
), suffix=' '), Keyword),
(words(('del', 'jump', 'call', 'branch'), prefix=r'(', suffix=r')( )'), bygroups(Keyword, Whitespace)),

],

'whitespace': [
(r'(\n|\s)', Text),
],
Comment on lines +66 to +68
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
'whitespace': [
(r'(\n|\s)', Text),
],
'whitespace': [
(r'\s+', Whitespace),
],

Avoid “1 token per char”. See https://pygments.org/docs/lexerdevelopment/#common-pitfalls-and-best-practices

}
150 changes: 150 additions & 0 deletions tests/snippets/numbair/test_numba_ir_snippet.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,150 @@
---input---
label 0
x = arg(0, name=x) :: float64

$const2.0 = const(int, 2) :: Literal[int](2)
$6binary_multiply.2 = $const2.0 * x :: float64
del $const2.0
$10binary_multiply.4 = $6binary_multiply.2 * x :: float64
del x
del $6binary_multiply.2
$12return_value.5 = cast(value=$10binary_multiply.4) :: float64
del $10binary_multiply.4
return $12return_value.5

---tokens---
'label' Name.Constant
' ' Text
'0' Literal.Number
'\n' Text

' ' Text
' ' Text
'x' Name.Constant
' = ' Operator
'arg' Name.Function
'(' Punctuation
'0' Literal.Number
',' Punctuation
' ' Text
'name' Name.Attribute
'=' Punctuation
'x' Name.Constant
')' Punctuation
' ' Text
' ' Text
':' Punctuation
':' Punctuation
' ' Text
'float64' Name.Constant
'\n' Text

'\n' Text

' ' Text
' ' Text
'$const2.0' Name.Variable
' = ' Operator
'const' Name.Function
'(' Punctuation
'int' Name.Constant
',' Punctuation
' ' Text
'2' Literal.Number
')' Punctuation
' ' Text
' ' Text
':' Punctuation
':' Punctuation
' ' Text
'Literal' Name.Constant
'[' Punctuation
'int' Name.Constant
']' Punctuation
'(' Punctuation
'2' Literal.Number
')' Punctuation
'\n' Text

' ' Text
' ' Text
'$6binary_multiply.2' Name.Variable
' = ' Operator
'$const2.0' Name.Variable
' ' Text
'*' Punctuation
' ' Text
'x' Name.Constant
' ' Text
' ' Text
':' Punctuation
':' Punctuation
' ' Text
'float64' Name.Constant
'\n' Text

' ' Text
' ' Text
'del ' Keyword
'$const2.0' Name.Variable
'\n' Text

' ' Text
' ' Text
'$10binary_multiply.4' Name.Variable
' = ' Operator
'$6binary_multiply.2' Name.Variable
' ' Text
'*' Punctuation
' ' Text
'x' Name.Constant
' ' Text
' ' Text
':' Punctuation
':' Punctuation
' ' Text
'float64' Name.Constant
'\n' Text

' ' Text
' ' Text
'del ' Keyword
'x' Name.Constant
'\n' Text

' ' Text
' ' Text
'del ' Keyword
'$6binary_multiply.2' Name.Variable
'\n' Text

' ' Text
' ' Text
'$12return_value.5' Name.Variable
' = ' Operator
'cast' Name.Function
'(' Punctuation
'value' Name.Attribute
'=' Punctuation
'$10binary_multiply.4' Name.Variable
')' Punctuation
' ' Text
' ' Text
':' Punctuation
':' Punctuation
' ' Text
'float64' Name.Constant
'\n' Text

' ' Text
' ' Text
'del ' Keyword
'$10binary_multiply.4' Name.Variable
'\n' Text

' ' Text
' ' Text
'return' Name.Constant
' ' Text
'$12return_value.5' Name.Variable
'\n' Text