Merge pull request crate-ci#135 from epage/type

perf: Faster binary-file detection
epage · Aug 22, 2020 · 8d267ed · 8d267ed
2 parents 443aa5c + a63dfa0
commit 8d267ed
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 17 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/typos/Cargo.toml b/crates/typos/Cargo.toml
@@ -27,3 +27,4 @@ log = "0.4"
 unicode-segmentation = "1.6.0"
 derive_more = "0.99.9"
 derive_setters = "0.1"
+content_inspector = "0.2.4"
diff --git a/crates/typos/src/checks.rs b/crates/typos/src/checks.rs
@@ -113,10 +113,17 @@ impl ParseIdentifiers {
 
         let buffer = std::fs::read(path)
             .map_err(|e| crate::ErrorKind::IoError.into_error().with_source(e))?;
-        if !explicit && !self.binary && is_binary(&buffer) {
-            let msg = report::BinaryFile { path };
-            reporter.report(msg.into());
-            return Ok(typos_found);
+        if !explicit && !self.binary {
+            let content_type = content_inspector::inspect(&buffer);
+            if content_type.is_binary()
+                // HACK: We only support UTF-8 at the moment
+                || (content_type != content_inspector::ContentType::UTF_8_BOM
+                    && content_type != content_inspector::ContentType::UTF_8)
+            {
+                let msg = report::BinaryFile { path };
+                reporter.report(msg.into());
+                return Ok(typos_found);
+            }
         }
 
         for line in buffer.lines() {
@@ -182,10 +189,17 @@ impl ParseWords {
 
         let buffer = std::fs::read(path)
             .map_err(|e| crate::ErrorKind::IoError.into_error().with_source(e))?;
-        if !explicit && !self.binary && is_binary(&buffer) {
-            let msg = report::BinaryFile { path };
-            reporter.report(msg.into());
-            return Ok(typos_found);
+        if !explicit && !self.binary {
+            let content_type = content_inspector::inspect(&buffer);
+            // HACK: We only support UTF-8 at the moment
+            if content_type.is_binary()
+                || (content_type != content_inspector::ContentType::UTF_8_BOM
+                    && content_type != content_inspector::ContentType::UTF_8)
+            {
+                let msg = report::BinaryFile { path };
+                reporter.report(msg.into());
+                return Ok(typos_found);
+            }
         }
 
         for line in buffer.lines() {
@@ -274,10 +288,18 @@ impl Checks {
 
         let buffer = std::fs::read(path)
             .map_err(|e| crate::ErrorKind::IoError.into_error().with_source(e))?;
-        if !explicit && !self.binary && is_binary(&buffer) {
-            let msg = report::BinaryFile { path };
-            reporter.report(msg.into());
-            return Ok(typos_found);
+        if !explicit && !self.binary {
+            let content_type = content_inspector::inspect(&buffer);
+            // HACK: We only support UTF-8 at the moment
+            if content_type.is_binary()
+                || (content_type != content_inspector::ContentType::UTF_8_BOM
+                    && content_type != content_inspector::ContentType::UTF_8)
+            {
+                // HACK: we don't support alternative encodings atm
+                let msg = report::BinaryFile { path };
+                reporter.report(msg.into());
+                return Ok(typos_found);
+            }
         }
 
         for (line_idx, line) in buffer.lines().enumerate() {
@@ -318,8 +340,3 @@ impl Checks {
         Ok(typos_found)
     }
 }
-
-fn is_binary(buffer: &[u8]) -> bool {
-    let null_max = std::cmp::min(buffer.len(), 1024);
-    buffer[0..null_max].find_byte(b'\0').is_some()
-}