Fix: Issues with .properties format using whitespace delimited key (#…

…2241)

Added:
- support for space delimitor in every case, included multiline value
- check for odd number of backslash escapes
- "!" as comment start
- support for escape of spaces and separators
Dropped:
- undocumented ";" and "//" comment start

pygments/lexers/configs.py

@@ -128,14 +128,26 @@ class PropertiesLexer(RegexLexer):
 
     tokens = {
         'root': [
-            (r'^(\w+)([ \t])(\w+\s*)$', bygroups(Name.Attribute, Whitespace, String)),
-            (r'^\w+(\\[ \t]\w*)*$', Name.Attribute),
-            (r'(^ *)([#!].*)', bygroups(Whitespace, Comment)),
-            # More controversial comments
-            (r'(^ *)((?:;|//).*)', bygroups(Whitespace, Comment)),
-            (r'(.*?)([ \t]*)([=:])([ \t]*)(.*(?:(?<=\\)\n.*)*)',
-             bygroups(Name.Attribute, Whitespace, Operator, Whitespace, String)),
-            (r'\s', Whitespace),
+            (r'\s+', Whitespace),
+            (r'[!#].*|/{2}.*', Comment.Single),
+            # search for first separator
+            (r'([^\\\n]|\\.)*?(?=[ \f\t=:])', Name.Attribute, "separator"),
+            # empty key
+            (r'.+?$', Name.Attribute),
+        ],
+        'separator': [
+            # search for line continuation escape
+            (r'([ \f\t]*)([=:]*)([ \f\t]*)(.*(?<!\\)(?:\\{2})*)(\\)(?!\\)$',
+             bygroups(Whitespace, Operator, Whitespace, String, Text), "value", "#pop"),
+            (r'([ \f\t]*)([=:]*)([ \f\t]*)(.*)',
+             bygroups(Whitespace, Operator, Whitespace, String), "#pop"),
+        ],
+        'value': [     # line continuation
+            (r'\s+', Whitespace),
+            # search for line continuation escape
+            (r'(\s*)(.*(?<!\\)(?:\\{2})*)(\\)(?!\\)([ \t]*)',
+             bygroups(Whitespace, String, Text, Whitespace)),
+            (r'.*$', String, "#pop"),
         ],
     }
 

tests/examplefiles/properties/java.properties

@@ -1,16 +1,24 @@
-foo = bar
-foo: bar
-foo.oof: \
-    bar=baz; \
-    asdf
+#https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/util/Properties.htm
+# mixing spaces
+	Truth = Beauty
+  Truth:Beauty
+Truth	Beauty
+Truth                    :Beauty
+
+! line continuations and escapes
+ fruits                           apple, banana, pear, \
+                                  cantaloupe, watermelon, \
+                                  kiwi, mango
+key = \
+    value1 \\\
+    and value2\\
+key\ 2 = value
+key\\ 3 = value3
 
-// comment
-# comment
-; comment
-
-x:a\
-b
-x: a \
-  b
-
-x = \
+! empty keys and edge cases
+key1 =
+key2
+key3 the value3
+key4 the:value4
+key5 the=value5
+key6=the value6

tests/snippets/properties/test_comments.txt

@@ -5,8 +5,8 @@
 # also a comment
 
 ---tokens---
-'! a comment' Comment
+'! a comment' Comment.Single
 '\n'          Text.Whitespace
 
-'# also a comment' Comment
+'# also a comment' Comment.Single
 '\n'          Text.Whitespace

tests/snippets/properties/test_leading_whitespace_comments.txt

@@ -2,5 +2,5 @@
 # comment
 
 ---tokens---
-'# comment'   Comment
+'# comment'   Comment.Single
 '\n'          Text.Whitespace

tests/snippets/properties/test_space_delimited_kv_pair.txt

@@ -4,4 +4,5 @@ key value
 ---tokens---
 'key'         Name.Attribute
 ' '           Text.Whitespace
-'value\n'     Literal.String
+'value'       Literal.String
+'\n'          Text.Whitespace