From a38cb38e93c9635240b3ae89d78d38cf182745da Mon Sep 17 00:00:00 2001
From: jmzambon <jeanmarczambon@gmail.com>
Date: Sun, 25 Sep 2022 22:56:48 +0200
Subject: [PATCH] Fix: Issues with .properties format using whitespace
 delimited key (#2241)

Added:
- support for space delimitor in every case, included multiline value
- check for odd number of backslash escapes
- "!" as comment start
- support for escape of spaces and separators
Dropped:
- undocumented ";" and "//" comment start
---
 pygments/lexers/configs.py                    | 28 ++++--
 tests/examplefiles/properties/java.properties | 38 ++++----
 .../properties/java.properties.output         | 88 ++++++++++++++-----
 tests/snippets/properties/test_comments.txt   |  4 +-
 .../test_leading_whitespace_comments.txt      |  2 +-
 .../test_space_delimited_kv_pair.txt          |  3 +-
 6 files changed, 114 insertions(+), 49 deletions(-)

diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py
index 4e0e7f1282..6f61d98e41 100644
--- a/pygments/lexers/configs.py
+++ b/pygments/lexers/configs.py
@@ -128,14 +128,26 @@ class PropertiesLexer(RegexLexer):
 
     tokens = {
         'root': [
-            (r'^(\w+)([ \t])(\w+\s*)$', bygroups(Name.Attribute, Whitespace, String)),
-            (r'^\w+(\\[ \t]\w*)*$', Name.Attribute),
-            (r'(^ *)([#!].*)', bygroups(Whitespace, Comment)),
-            # More controversial comments
-            (r'(^ *)((?:;|//).*)', bygroups(Whitespace, Comment)),
-            (r'(.*?)([ \t]*)([=:])([ \t]*)(.*(?:(?<=\\)\n.*)*)',
-             bygroups(Name.Attribute, Whitespace, Operator, Whitespace, String)),
-            (r'\s', Whitespace),
+            (r'\s+', Whitespace),
+            (r'[!#].*|/{2}.*', Comment.Single),
+            # search for first separator
+            (r'([^\\\n]|\\.)*?(?=[ \f\t=:])', Name.Attribute, "separator"),
+            # empty key
+            (r'.+?$', Name.Attribute),
+        ],
+        'separator': [
+            # search for line continuation escape
+            (r'([ \f\t]*)([=:]*)([ \f\t]*)(.*(?<!\\)(?:\\{2})*)(\\)(?!\\)$',
+             bygroups(Whitespace, Operator, Whitespace, String, Text), "value", "#pop"),
+            (r'([ \f\t]*)([=:]*)([ \f\t]*)(.*)',
+             bygroups(Whitespace, Operator, Whitespace, String), "#pop"),
+        ],
+        'value': [     # line continuation
+            (r'\s+', Whitespace),
+            # search for line continuation escape
+            (r'(\s*)(.*(?<!\\)(?:\\{2})*)(\\)(?!\\)([ \t]*)',
+             bygroups(Whitespace, String, Text, Whitespace)),
+            (r'.*$', String, "#pop"),
         ],
     }
 
diff --git a/tests/examplefiles/properties/java.properties b/tests/examplefiles/properties/java.properties
index 72ad0f964c..d5b594e3d8 100644
--- a/tests/examplefiles/properties/java.properties
+++ b/tests/examplefiles/properties/java.properties
@@ -1,16 +1,24 @@
-foo = bar
-foo: bar
-foo.oof: \
-    bar=baz; \
-    asdf
+#https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/util/Properties.htm
+# mixing spaces
+	Truth = Beauty
+  Truth:Beauty
+Truth	Beauty
+Truth                    :Beauty
+ 
+! line continuations and escapes
+ fruits                           apple, banana, pear, \
+                                  cantaloupe, watermelon, \
+                                  kiwi, mango
+key = \
+    value1 \\\
+    and value2\\
+key\ 2 = value
+key\\ 3 = value3
 
-// comment
-# comment
-; comment
-
-x:a\
-b
-x: a \
-  b
-
-x = \
+! empty keys and edge cases
+key1 =
+key2
+key3 the value3
+key4 the:value4
+key5 the=value5
+key6=the value6
diff --git a/tests/examplefiles/properties/java.properties.output b/tests/examplefiles/properties/java.properties.output
index 3efd7e2a4f..0c1fdeebf7 100644
--- a/tests/examplefiles/properties/java.properties.output
+++ b/tests/examplefiles/properties/java.properties.output
@@ -1,50 +1,94 @@
-'foo'         Name.Attribute
+'#https://docs.oracle.com/en/java/javase/17/docs/api/java.base/java/util/Properties.htm' Comment.Single
+'\n'          Text.Whitespace
+
+'# mixing spaces' Comment.Single
+'\n\t'        Text.Whitespace
+'Truth'       Name.Attribute
 ' '           Text.Whitespace
 '='           Operator
 ' '           Text.Whitespace
-'bar'         Literal.String
+'Beauty'      Literal.String
+'\n  '        Text.Whitespace
+'Truth'       Name.Attribute
+':'           Operator
+'Beauty'      Literal.String
 '\n'          Text.Whitespace
 
-'foo'         Name.Attribute
-':'           Operator
-' '           Text.Whitespace
-'bar'         Literal.String
+'Truth'       Name.Attribute
+'\t'          Text.Whitespace
+'Beauty'      Literal.String
 '\n'          Text.Whitespace
 
-'foo.oof'     Name.Attribute
+'Truth'       Name.Attribute
+'                    ' Text.Whitespace
 ':'           Operator
-' '           Text.Whitespace
-'\\\n    bar=baz; \\\n    asdf' Literal.String
-'\n'          Text.Whitespace
+'Beauty'      Literal.String
+'\n \n'       Text.Whitespace
 
+'! line continuations and escapes' Comment.Single
+'\n '         Text.Whitespace
+'fruits'      Name.Attribute
+'                           ' Text.Whitespace
+'apple, banana, pear, ' Literal.String
+'\\'          Text
+'\n                                  ' Text.Whitespace
+'cantaloupe, watermelon, ' Literal.String
+'\\'          Text
+'\n                                  ' Text.Whitespace
+'kiwi, mango' Literal.String
 '\n'          Text.Whitespace
 
-'// comment'  Comment
+'key'         Name.Attribute
+' '           Text.Whitespace
+'='           Operator
+' '           Text.Whitespace
+'\\'          Text
+'\n    '      Text.Whitespace
+'value1 \\\\' Literal.String
+'\\'          Text
+'\n    '      Text.Whitespace
+'and value2\\\\' Literal.String
 '\n'          Text.Whitespace
 
-'# comment'   Comment
+'key\\ 2'     Name.Attribute
+' '           Text.Whitespace
+'='           Operator
+' '           Text.Whitespace
+'value'       Literal.String
 '\n'          Text.Whitespace
 
-'; comment'   Comment
+'key\\\\'     Name.Attribute
+' '           Text.Whitespace
+'3 = value3'  Literal.String
+'\n\n'        Text.Whitespace
+
+'! empty keys and edge cases' Comment.Single
 '\n'          Text.Whitespace
 
+'key1'        Name.Attribute
+' '           Text.Whitespace
+'='           Operator
 '\n'          Text.Whitespace
 
-'x'           Name.Attribute
-':'           Operator
-'a\\\nb'      Literal.String
+'key2'        Name.Attribute
 '\n'          Text.Whitespace
 
-'x'           Name.Attribute
-':'           Operator
+'key3'        Name.Attribute
 ' '           Text.Whitespace
-'a \\\n  b'   Literal.String
+'the value3'  Literal.String
 '\n'          Text.Whitespace
 
+'key4'        Name.Attribute
+' '           Text.Whitespace
+'the:value4'  Literal.String
 '\n'          Text.Whitespace
 
-'x'           Name.Attribute
+'key5'        Name.Attribute
 ' '           Text.Whitespace
+'the=value5'  Literal.String
+'\n'          Text.Whitespace
+
+'key6'        Name.Attribute
 '='           Operator
-' '           Text.Whitespace
-'\\\n'        Literal.String
+'the value6'  Literal.String
+'\n'          Text.Whitespace
diff --git a/tests/snippets/properties/test_comments.txt b/tests/snippets/properties/test_comments.txt
index b6bc8fb9e6..9bc6586078 100644
--- a/tests/snippets/properties/test_comments.txt
+++ b/tests/snippets/properties/test_comments.txt
@@ -5,8 +5,8 @@
 # also a comment
 
 ---tokens---
-'! a comment' Comment
+'! a comment' Comment.Single
 '\n'          Text.Whitespace
 
-'# also a comment' Comment
+'# also a comment' Comment.Single
 '\n'          Text.Whitespace
diff --git a/tests/snippets/properties/test_leading_whitespace_comments.txt b/tests/snippets/properties/test_leading_whitespace_comments.txt
index be77792b33..3a36afc9df 100644
--- a/tests/snippets/properties/test_leading_whitespace_comments.txt
+++ b/tests/snippets/properties/test_leading_whitespace_comments.txt
@@ -2,5 +2,5 @@
 # comment
 
 ---tokens---
-'# comment'   Comment
+'# comment'   Comment.Single
 '\n'          Text.Whitespace
diff --git a/tests/snippets/properties/test_space_delimited_kv_pair.txt b/tests/snippets/properties/test_space_delimited_kv_pair.txt
index fe12d48efa..98961e4256 100644
--- a/tests/snippets/properties/test_space_delimited_kv_pair.txt
+++ b/tests/snippets/properties/test_space_delimited_kv_pair.txt
@@ -4,4 +4,5 @@ key value
 ---tokens---
 'key'         Name.Attribute
 ' '           Text.Whitespace
-'value\n'     Literal.String
+'value'       Literal.String
+'\n'          Text.Whitespace