Skip to content

Commit

Permalink
Update Crystal lexer (#1650)
Browse files Browse the repository at this point in the history 
* crystal: drop all classes from builtins; these aren't normally highlighted

("normally" meaning all other highlighter tools)

* crystal: fix percent-strings, drop Ruby-specific arbitrary delimiters

It seems that Ruby supports strings such as `%*text*` where `*` can be anything. But Crystal never had anything like that. It does, however, keep `%|text|`, so add a case for that.

* crystal: update keywords and builtins

* crystal: fix string literals and escape sequences

Update list of escapes. Support Unicode escape sequences.

Also remove the Ruby-specific `:@foo` symbol syntax, Crystal doesn't have it.

* crystal: uppercase identifiers aren't always constants

Make `FOO::Bar` be highlighted like `Foo::Bar` would be, rather than like `FOO`

* crystal: annotations can be namespaced

Highlight the entire inside part of `@[Foo::Bar]`, not just the `Foo` part

(these used to be named 'attributes' but the official name is 'annotations' now, so I also change that)

* fixup! crystal: fix percent-strings, drop Ruby-specific arbitrary delimiters
  • Loading branch information
@oprypin
oprypin committed Jan 4, 2021
1 parent c01ccfe commit f8b097c
Show file tree
Hide file tree
Showing 3 changed files with 282 additions and 90 deletions.
119 changes: 45 additions & 74 deletions pygments/lexers/crystal.py
View file
Expand Up @@ -89,26 +89,10 @@ def heredoc_callback(self, match, ctx):
del heredocstack[:]

def gen_crystalstrings_rules():
def intp_regex_callback(self, match, ctx):
yield match.start(1), String.Regex, match.group(1) # begin
nctx = LexerContext(match.group(3), 0, ['interpolated-regex'])
for i, t, v in self.get_tokens_unprocessed(context=nctx):
yield match.start(3)+i, t, v
yield match.start(4), String.Regex, match.group(4) # end[imsx]*
ctx.pos = match.end()

def intp_string_callback(self, match, ctx):
yield match.start(1), String.Other, match.group(1)
nctx = LexerContext(match.group(3), 0, ['interpolated-string'])
for i, t, v in self.get_tokens_unprocessed(context=nctx):
yield match.start(3)+i, t, v
yield match.start(4), String.Other, match.group(4) # end
ctx.pos = match.end()

states = {}
states['strings'] = [
(r'\:@{0,2}[a-zA-Z_]\w*[!?]?', String.Symbol),
(words(CRYSTAL_OPERATORS, prefix=r'\:@{0,2}'), String.Symbol),
(r'\:\w+[!?]?', String.Symbol),
(words(CRYSTAL_OPERATORS, prefix=r'\:'), String.Symbol),
(r":'(\\\\|\\[^\\]|[^'\\])*'", String.Symbol),
# This allows arbitrary text after '\ for simplicity
(r"'(\\\\|\\'|[^']|\\[^'\\]+)'", String.Char),
Expand All @@ -130,35 +114,42 @@ def intp_string_callback(self, match, ctx):
(end, ttype, '#pop'),
]

# braced quoted strings
# https://crystal-lang.org/docs/syntax_and_semantics/literals/string.html#percent-string-literals
for lbrace, rbrace, bracecc, name in \
('\\{', '\\}', '{}', 'cb'), \
('\\[', '\\]', '\\[\\]', 'sb'), \
('\\(', '\\)', '()', 'pa'), \
('<', '>', '<>', 'ab'):
('<', '>', '<>', 'ab'), \
('\\|', '\\|', '\\|', 'pi'):
states[name+'-intp-string'] = [
(r'\\' + lbrace, String.Other),
] + (lbrace != rbrace) * [
(lbrace, String.Other, '#push'),
] + [
(rbrace, String.Other, '#pop'),
include('string-intp-escaped'),
(r'[\\#' + bracecc + ']', String.Other),
(r'[^\\#' + bracecc + ']+', String.Other),
]
states['strings'].append((r'%' + lbrace, String.Other,
states['strings'].append((r'%Q?' + lbrace, String.Other,
name+'-intp-string'))
states[name+'-string'] = [
(r'\\[\\' + bracecc + ']', String.Other),
] + (lbrace != rbrace) * [
(lbrace, String.Other, '#push'),
] + [
(rbrace, String.Other, '#pop'),
(r'[\\#' + bracecc + ']', String.Other),
(r'[^\\#' + bracecc + ']+', String.Other),
]
# http://crystal-lang.org/docs/syntax_and_semantics/literals/array.html
states['strings'].append((r'%[wi]' + lbrace, String.Other,
# https://crystal-lang.org/docs/syntax_and_semantics/literals/array.html#percent-array-literals
states['strings'].append((r'%[qwi]' + lbrace, String.Other,
name+'-string'))
states[name+'-regex'] = [
(r'\\[\\' + bracecc + ']', String.Regex),
] + (lbrace != rbrace) * [
(lbrace, String.Regex, '#push'),
] + [
(rbrace + '[imsx]*', String.Regex, '#pop'),
include('string-intp'),
(r'[\\#' + bracecc + ']', String.Regex),
Expand All @@ -167,68 +158,46 @@ def intp_string_callback(self, match, ctx):
states['strings'].append((r'%r' + lbrace, String.Regex,
name+'-regex'))

# these must come after %<brace>!
states['strings'] += [
# %r regex
(r'(%r([\W_]))((?:\\\2|(?!\2).)*)(\2[imsx]*)',
intp_regex_callback),
# regular fancy strings with qsw
(r'(%[wi]([\W_]))((?:\\\2|(?!\2).)*)(\2)',
intp_string_callback),
# special forms of fancy strings after operators or
# in method calls with braces
(r'(?<=[-+/*%=<>&!^|~,(])(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
bygroups(Text, String.Other, None)),
# and because of fixed width lookbehinds the whole thing a
# second time for line startings...
(r'^(\s*)(%([\t ])(?:(?:\\\3|(?!\3).)*)\3)',
bygroups(Text, String.Other, None)),
# all regular fancy strings without qsw
(r'(%([\[{(<]))((?:\\\2|(?!\2).)*)(\2)',
intp_string_callback),
]

return states

tokens = {
'root': [
(r'#.*?$', Comment.Single),
# keywords
(words('''
abstract asm as begin break case do else elsif end ensure extend ifdef if
include instance_sizeof next of pointerof private protected rescue return
require sizeof super then typeof unless until when while with yield
abstract asm begin break case do else elsif end ensure extend if in
include next of private protected require rescue return select self super
then unless until when while with yield
'''.split(), suffix=r'\b'), Keyword),
(words('''
previous_def forall out uninitialized __DIR__ __FILE__ __LINE__
'''.split(), prefix=r'(?<!\.)', suffix=r'\b'), Keyword.Pseudo),
# https://crystal-lang.org/docs/syntax_and_semantics/is_a.html
(r'\.(is_a\?|nil\?|responds_to\?|as\?|as\b)', Keyword.Pseudo),
(words(['true', 'false', 'nil'], suffix=r'\b'), Keyword.Constant),
# start of function, class and module names
(r'(module|lib)(\s+)([a-zA-Z_]\w*(?:::[a-zA-Z_]\w*)*)',
bygroups(Keyword, Text, Name.Namespace)),
(r'(def|fun|macro)(\s+)((?:[a-zA-Z_]\w*::)*)',
bygroups(Keyword, Text, Name.Namespace), 'funcname'),
(r'def(?=[*%&^`~+-/\[<>=])', Keyword, 'funcname'),
(r'(class|struct|union|type|alias|enum)(\s+)((?:[a-zA-Z_]\w*::)*)',
(r'(annotation|class|struct|union|type|alias|enum)(\s+)((?:[a-zA-Z_]\w*::)*)',
bygroups(Keyword, Text, Name.Namespace), 'classname'),
(r'(self|out|uninitialized)\b|(is_a|responds_to)\?', Keyword.Pseudo),
# macros
# https://crystal-lang.org/api/toplevel.html
(words('''
debugger record pp assert_responds_to spawn parallel
getter setter property delegate def_hash def_equals def_equals_and_hash
forward_missing_to
'''.split(), suffix=r'\b'), Name.Builtin.Pseudo),
(r'getter[!?]|property[!?]|__(DIR|FILE|LINE)__\b', Name.Builtin.Pseudo),
instance_sizeof offsetof pointerof sizeof typeof
'''.split(), prefix=r'(?<!\.)', suffix=r'\b'), Keyword.Pseudo),
# macros
(r'(?<!\.)(debugger\b|p!|pp!|record\b|spawn\b)', Name.Builtin.Pseudo),
# builtins
# http://crystal-lang.org/api/toplevel.html
(words('''
Object Value Struct Reference Proc Class Nil Symbol Enum Void
Bool Number Int Int8 Int16 Int32 Int64 UInt8 UInt16 UInt32 UInt64
Float Float32 Float64 Char String
Pointer Slice Range Exception Regex
Mutex StaticArray Array Hash Set Tuple Deque Box Process File
Dir Time Channel Concurrent Scheduler
abort at_exit caller delay exit fork future get_stack_top gets
lazy loop main p print printf puts
raise rand read_line sleep sprintf system with_color
abort at_exit caller exit gets loop main p pp print printf puts
raise rand read_line sleep spawn sprintf system
'''.split(), prefix=r'(?<!\.)', suffix=r'\b'), Name.Builtin),
# https://crystal-lang.org/api/Object.html#macro-summary
(r'(?<!\.)(((class_)?((getter|property)\b[!?]?|setter\b))|'
r'(def_(clone|equals|equals_and_hash|hash)|delegate|forward_missing_to)\b)',
Name.Builtin.Pseudo),
# normal heredocs
(r'(?<!\w)(<<-?)(["`\']?)([a-zA-Z_]\w*)(\2)(.*?\n)',
heredoc_callback),
Expand Down Expand Up @@ -300,18 +269,18 @@ def intp_string_callback(self, match, ctx):
(r'\$-[0adFiIlpvw]', Name.Variable.Global),
(r'::', Operator),
include('strings'),
# chars
# https://crystal-lang.org/reference/syntax_and_semantics/literals/char.html
(r'\?(\\[MC]-)*' # modifiers
r'(\\([\\befnrtv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})|\S)'
r'(\\([\\abefnrtv#"\']|[0-7]{1,3}|x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|u\{[a-fA-F0-9 ]+\})|\S)'
r'(?!\w)',
String.Char),
(r'[A-Z][A-Z_]+\b', Name.Constant),
(r'[A-Z][A-Z_]+\b(?!::|\.)', Name.Constant),
# macro expansion
(r'\{%', String.Interpol, 'in-macro-control'),
(r'\{\{', String.Interpol, 'in-macro-expr'),
# attributes
(r'(@\[)(\s*)([A-Z]\w*)',
bygroups(Operator, Text, Name.Decorator), 'in-attr'),
# annotations
(r'(@\[)(\s*)([A-Z]\w*(::[A-Z]\w*)*)',
bygroups(Operator, Text, Name.Decorator), 'in-annot'),
# this is needed because Crystal attributes can look
# like keywords (class) or like this: ` ?!?
(words(CRYSTAL_OPERATORS, prefix=r'(\.|::)'),
Expand Down Expand Up @@ -348,7 +317,9 @@ def intp_string_callback(self, match, ctx):
(r'#\{', String.Interpol, 'in-intp'),
],
'string-escaped': [
(r'\\([\\befnstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})', String.Escape)
# https://crystal-lang.org/reference/syntax_and_semantics/literals/string.html
(r'\\([\\abefnrtv#"\']|[0-7]{1,3}|x[a-fA-F0-9]{2}|u[a-fA-F0-9]{4}|u\{[a-fA-F0-9 ]+\})',
String.Escape)
],
'string-intp-escaped': [
include('string-intp'),
Expand Down Expand Up @@ -378,15 +349,15 @@ def intp_string_callback(self, match, ctx):
'in-macro-control': [
(r'\{%', String.Interpol, '#push'),
(r'%\}', String.Interpol, '#pop'),
(r'for\b|in\b', Keyword),
(r'(for|verbatim)\b', Keyword),
include('root'),
],
'in-macro-expr': [
(r'\{\{', String.Interpol, '#push'),
(r'\}\}', String.Interpol, '#pop'),
include('root'),
],
'in-attr': [
'in-annot': [
(r'\[', Operator, '#push'),
(r'\]', Operator, '#pop'),
include('root'),
Expand Down
16 changes: 5 additions & 11 deletions tests/examplefiles/test.cr
View file
Expand Up @@ -1954,17 +1954,11 @@ r = %r(regex with slash: /)
world, \
no newlines" # same as "hello world, no newlines"

# Supports double quotes and nested parenthesis
%(hello ("world")) # same as "hello (\"world\")"

# Supports double quotes and nested brackets
%[hello ["world"]] # same as "hello [\"world\"]"

# Supports double quotes and nested curlies
%{hello {"world"}} # same as "hello {\"world\"}"

# Supports double quotes and nested angles
%<hello <"world">> # same as "hello <\"world\">"
%(hello ("world")) # => "hello (\"world\")"
%[hello ["world"]] # => "hello [\"world\"]"
%{hello {"world"}} # => "hello {\"world\"}"
%<hello <"world">> # => "hello <\"world\">"
%|hello "world"| # => "hello \"world\""

<<-XML
<parent>
Expand Down

0 comments on commit f8b097c

Please sign in to comment.