From 4402e3ee1fb7ed9b470c25b6c23fe49d587acbf4 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sat, 25 Apr 2020 08:12:07 +0900 Subject: [PATCH 1/5] Add example with invalid escape sequences --- spec/visual/samples/python | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/visual/samples/python b/spec/visual/samples/python index 148e113939..5d667c9b51 100644 --- a/spec/visual/samples/python +++ b/spec/visual/samples/python @@ -47,6 +47,7 @@ def baz(): '\UaaaaAF09' '\xaf\xAF\x09' '\007' + '.*\[p00t_(d\d{4})\].*' # This should include errors # escaped characters in raw strings def baz(): @@ -56,6 +57,7 @@ def baz(): r'\UaaaaAF09' r'\xaf\xAF\x09' r'\007' + r'.*\[p00t_(d\d{4})\].*' # This should not include errors # line continuations apple.filter(x, y) From c0c1391c8a4b5e8d7f8458d547cd343cf56114ed Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sat, 25 Apr 2020 08:34:31 +0900 Subject: [PATCH 2/5] Add new escape state for raw strings --- lib/rouge/lexers/python.rb | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/rouge/lexers/python.rb b/lib/rouge/lexers/python.rb index 9a718299c4..22f0c825e9 100644 --- a/lib/rouge/lexers/python.rb +++ b/lib/rouge/lexers/python.rb @@ -189,10 +189,11 @@ def current_string rule %r/\\/ do |m| if current_string.type? "r" token Str + push :raw_escape else token Str::Interpol + push :generic_escape end - push :generic_escape end rule %r/{/ do |m| @@ -215,14 +216,9 @@ def current_string | x[a-fA-F0-9]{2} | [0-7]{1,3} ) - )x do - if current_string.type? "r" - token Str - else - token Str::Escape - end - pop! - end + )x, Str::Escape, :pop! + + rule %r/./, Error, :pop! end state :generic_interpol do @@ -233,6 +229,10 @@ def current_string rule %r/}/, Str::Interpol, :pop! end + state :raw_escape do + rule %r/./, Str, :pop! + end + class StringRegister < Array def delim?(delim) self.last[1] == delim From 5fa37f1eba804e9be0fae6281e0d1842688ad0c6 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sat, 25 Apr 2020 09:08:32 +0900 Subject: [PATCH 3/5] Permit all characters after \ --- lib/rouge/lexers/python.rb | 23 ++++++++--------------- spec/visual/samples/python | 4 ++-- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/lib/rouge/lexers/python.rb b/lib/rouge/lexers/python.rb index 22f0c825e9..3b428e6387 100644 --- a/lib/rouge/lexers/python.rb +++ b/lib/rouge/lexers/python.rb @@ -186,14 +186,8 @@ def current_string end end - rule %r/\\/ do |m| - if current_string.type? "r" - token Str - push :raw_escape - else - token Str::Interpol - push :generic_escape - end + rule %r/(?=\\)/ do |m| + push :generic_escape end rule %r/{/ do |m| @@ -207,7 +201,7 @@ def current_string end state :generic_escape do - rule %r( + rule %r(\\ ( [\\abfnrtv"'] | \n | N{[a-zA-Z][a-zA-Z ]+[a-zA-Z]} @@ -216,9 +210,12 @@ def current_string | x[a-fA-F0-9]{2} | [0-7]{1,3} ) - )x, Str::Escape, :pop! + )x do + token (current_string.type?("r") ? Str : Str::Escape) + pop! + end - rule %r/./, Error, :pop! + rule %r/\\./, Str, :pop! end state :generic_interpol do @@ -229,10 +226,6 @@ def current_string rule %r/}/, Str::Interpol, :pop! end - state :raw_escape do - rule %r/./, Str, :pop! - end - class StringRegister < Array def delim?(delim) self.last[1] == delim diff --git a/spec/visual/samples/python b/spec/visual/samples/python index 5d667c9b51..8331581809 100644 --- a/spec/visual/samples/python +++ b/spec/visual/samples/python @@ -47,7 +47,7 @@ def baz(): '\UaaaaAF09' '\xaf\xAF\x09' '\007' - '.*\[p00t_(d\d{4})\].*' # This should include errors + '.*\[p00t_(d\d{4})\].*' # There are no escape sequences in this string # escaped characters in raw strings def baz(): @@ -57,7 +57,7 @@ def baz(): r'\UaaaaAF09' r'\xaf\xAF\x09' r'\007' - r'.*\[p00t_(d\d{4})\].*' # This should not include errors + r'.*\[p00t_(d\d{4})\].*' # line continuations apple.filter(x, y) From e8f73d41f5bacdb6bf7155fa650717af7d83c5a2 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sat, 25 Apr 2020 21:37:35 +0900 Subject: [PATCH 4/5] Simplify rule --- lib/rouge/lexers/python.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/rouge/lexers/python.rb b/lib/rouge/lexers/python.rb index 3b428e6387..b507d710e3 100644 --- a/lib/rouge/lexers/python.rb +++ b/lib/rouge/lexers/python.rb @@ -186,9 +186,7 @@ def current_string end end - rule %r/(?=\\)/ do |m| - push :generic_escape - end + rule %r/(?=\\)/, Str, :generic_escape rule %r/{/ do |m| if current_string.type? "f" From a1907b4e692adb435510023614edcdf92ca15316 Mon Sep 17 00:00:00 2001 From: Michael Camilleri Date: Sat, 25 Apr 2020 21:38:39 +0900 Subject: [PATCH 5/5] Add \newline pattern --- lib/rouge/lexers/python.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/rouge/lexers/python.rb b/lib/rouge/lexers/python.rb index b507d710e3..6eeea88655 100644 --- a/lib/rouge/lexers/python.rb +++ b/lib/rouge/lexers/python.rb @@ -202,6 +202,7 @@ def current_string rule %r(\\ ( [\\abfnrtv"'] | \n + | newline | N{[a-zA-Z][a-zA-Z ]+[a-zA-Z]} | u[a-fA-F0-9]{4} | U[a-fA-F0-9]{8}