From 5c727422f30872689abc92281b2b6d733e5d6b28 Mon Sep 17 00:00:00 2001 From: TAHRI Ahmed R Date: Sat, 23 Oct 2021 22:15:39 +0200 Subject: [PATCH] :sparkle: Improvement over Vietnamese detection (#126) Latin character with combining diacritical mark were miss detected/seen as mess (MD). --- charset_normalizer/md.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/charset_normalizer/md.py b/charset_normalizer/md.py index bb09100b..79e1efb8 100644 --- a/charset_normalizer/md.py +++ b/charset_normalizer/md.py @@ -453,6 +453,12 @@ def is_suspiciously_successive_range( if "Emoticons" in unicode_range_a or "Emoticons" in unicode_range_b: return False + # Latin characters can be accompanied with a combining diacritical mark + # eg. Vietnamese. + if "Latin" in unicode_range_a or "Latin" in unicode_range_b: + if "Combining" in unicode_range_a or "Combining" in unicode_range_b: + return False + keywords_range_a, keywords_range_b = unicode_range_a.split( " " ), unicode_range_b.split(" ")