❇️ Adjust the MD to lower the sensitivity around certain CSVs (#69)

* ❇️ Adjust the MD to lower the sensitivity around certain CSVs ..having many columns. * 🔧 ✔️ 0.01 MD ratio diff on single subtest is OK * 🔧 Remove tab duplicate, using vertical tab instead
Ousret · Jul 23, 2021 · 9fc6985 · 9fc6985
1 parent 87a5a98
commit 9fc6985
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 3 deletions.
diff --git a/charset_normalizer/md.py b/charset_normalizer/md.py
@@ -56,7 +56,7 @@ def eligible(self, character: str) -> bool:
     def feed(self, character: str) -> None:
         self._character_count += 1
 
-        if character != self._last_printable_char and character not in ["<", ">", "=", ":", "/", "&", ";", "{", "}", "[", "]"]:
+        if character != self._last_printable_char and character not in ["<", ">", "=", ":", "/", "&", ";", "{", "}", "[", "]", ",", "|", '"']:
             if is_punctuation(character):
                 self._punctuation_count += 1
             elif character.isdigit() is False and is_symbol(character):
@@ -116,7 +116,7 @@ def eligible(self, character: str) -> bool:
         return True
 
     def feed(self, character: str) -> None:
-        if character not in {'\n', '\t', '\r'} and character.isprintable() is False:
+        if character not in {'\n', '\t', '\r', '\v'} and character.isprintable() is False:
             self._unprintable_count += 1
         self._character_count += 1
 

diff --git a/tests/test_probe_chaos.py b/tests/test_probe_chaos.py
@@ -40,7 +40,7 @@ def test_subtle_gibberish(self):
 
         self.assertGreater(
             mess_ratio("´Á¥½³ø§i --  ±i®Ìºû, ³¯·Ø©v"),
-            0.8
+            0.7
         )
 
         self.assertGreater(