undici and very long strings (#178)

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## [3.4.6](https://github.com/omrilotan/isbot/compare/v3.4.5...v3.4.6)
+-   Add nodejs native agent (undici)
+-   Add random long string
+
 ## [3.4.5](https://github.com/omrilotan/isbot/compare/v3.4.4...v3.4.5)
 -   Add CF-UC web crawler
 -   Add TagInspector

fixtures/browsers.yml

@@ -659,6 +659,8 @@ Yandex Browser:
 ZZZ Glitches and Misidentified Browsers - These browsers are legit user agent even though they are included in some bot listings (not all bots identify themselves):
   - ; Android 11) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/96.0.4664.45 Mobile DuckDuckGo/5 Safari/537.36
   - asafaweb.com
+  - Hello, world
+  - iexplore.exe
   - LWPrCa2l') OR 941=(SELECT 941 FROM PG_SLEEP(15))--
   - Mozilla / 5.0 (Linux; Android 7.0; GHIA_ZEUS_3G) AppleWebKit / 537.36 (KHTML, como Gecko) Chrome / 87.0.4280.101 Mobile Safari / 537.36
   - Mozilla/5.0 (en-us) AppleWebKit/525.13 (KHTML, like Gecko) Version/3.1 Safari/525.13

fixtures/crawlers.yml

@@ -351,6 +351,8 @@ IIS Site Analysis:
   - iisbot/1.0 (+http://www.iis.net/iisbot.html)
 inoreader:
   - Mozilla/5.0 (compatible; inoreader.com; 2 subscribers)
+Interasco:
+  - InterascoAgent/1.0
 Invision:
   - Invision Community 4
 IP-Guide Crawler:
@@ -472,6 +474,8 @@ NLCrawler:
   - nlcrawler/1.0 (+http://northernlight.com/)
 Nmap:
   - Mozilla/5.0 (compatible; Nmap Scripting Engine; https://nmap.org/book/nse.html)
+Nodejs:
+  - undici
 Nuclei:
   - Nuclei - Open-source project (github.com/projectdiscovery/nuclei)
 Nutch-based Bot:
@@ -820,3 +824,4 @@ ZZZ Glitches and Errornous User Agent Strings:
   - "123"
   - Chrome
   - iphone 6 plus;afengineurl=https://intoli.com:443;traceId=63028f8e-c5fc-4846-993f-59a96268a85d
+  - U2FsdGVkX1+uKxeH2946/bMTDvtm/Fr0nWjvFR/oPtc64LSh1Gg0qkbJUIhLpSw5h/mjF86TFOXrl4U2SG1KBi4BC0EfphyIzeOxVXkpBWHDMfJnkrFGrubrRGjmJNIN49DKkOcjVgq2/iVDBMSAQe30k9wNIDtflfnlrOrmDPkXiYNjLbohSHLaNWS/GK5hu62gkOH25c9i1B+jMq5kc590HoQqJ0o4es9QrEnwluMsYPbQy14LxgPjeCQveiPHPXtkSM7TmfTY53HEJdbFHylstSOJNTQclbL67BKx33M=

package.json

@@ -1,6 +1,6 @@
 {
   "name": "isbot",
-  "version": "3.4.5",
+  "version": "3.4.6",
   "description": "🤖 detect bots/crawlers/spiders via the user agent.",
   "keywords": [
     "bot",
@@ -67,29 +67,29 @@
     "prepublishOnly": "npm run authors && npm run build"
   },
   "devDependencies": {
-    "@babel/eslint-parser": "^7.14.7",
-    "@babel/preset-env": "^7.14.8",
+    "@babel/eslint-parser": "^7.17.0",
+    "@babel/preset-env": "^7.16.11",
     "@lets/wait": "^2.0.2",
-    "@rollup/plugin-babel": "^5.3.0",
+    "@rollup/plugin-babel": "^5.3.1",
     "@rollup/plugin-json": "^4.1.0",
-    "@rollup/plugin-node-resolve": "^13.0.4",
+    "@rollup/plugin-node-resolve": "^13.2.1",
     "eslint-plugin-log": "^1.2.6",
     "form-data": "^4.0.0",
-    "karma": "^6.3.4",
-    "karma-chrome-launcher": "^3.1.0",
+    "karma": "^6.3.18",
+    "karma-chrome-launcher": "^3.1.1",
     "karma-cli": "^2.0.0",
     "karma-firefox-launcher": "^2.1.1",
     "karma-mocha": "^2.0.1",
     "karma-mocha-reporter": "^2.2.5",
-    "mocha": "^9.0.3",
+    "mocha": "^9.2.2",
     "pug": "^3.0.0",
     "remark-cli": "^10.0.0",
     "remark-preset-lint-recommended": "^6.0.0",
-    "rollup": "^2.55.0",
+    "rollup": "^2.70.2",
     "standard": "^16.0.3",
     "stdline": "^1.0.0",
-    "typescript": "^4.3.5",
-    "user-agents": "^1.0.727",
-    "yaml": "^1.10.0"
+    "typescript": "^4.6.3",
+    "user-agents": "^1.0.990",
+    "yaml": "^2.0.1"
   }
 }

page/script.js

@@ -2,15 +2,14 @@ import isbot from '../src/index.js'
 import { amend } from '../src/amend/index.js'
 import list from '../src/list.json'
 
-(function () {
+{
   const textarea = document.querySelector('textarea')
   const output = document.querySelector('output')
   let timer
 
   const query = window.location.search.replace(/\?ua=(.*)$/, '$1')
 
   amend(list)
-  const pattern = new RegExp(list.join('|'), 'i')
 
   textarea.childNodes.forEach(child => child.parentNode?.removeChild(child))
   textarea.appendChild(document.createTextNode(
@@ -29,9 +28,31 @@ import list from '../src/list.json'
     timer = setTimeout(check, 200, value)
   }
 
+  function showMatch (output, ua) {
+    const pattern = document.createElement('kbd')
+    pattern.appendChild(
+      document.createTextNode(isbot.matches(ua)?.pop())
+    )
+    output.appendChild(
+      document.createTextNode(
+        'I think so, yes\nThe pattern that was matched is: '
+      )
+    )
+    output.appendChild(pattern)
+  }
+  function noMatch (output) {
+    output.appendChild(
+      document.createTextNode(
+        'I don\'t think so, no\nI could not find a pattern I recognise'
+      )
+    )
+  }
+
   function check (value = textarea.innerHTML) {
     value = value.trim()
-    output.childNodes.forEach(child => child.parentNode?.removeChild(child))
+    while (output.firstChild) {
+      output.removeChild(output.firstChild)
+    }
     if (value === '') {
       output.appendChild(
         document.createTextNode(
@@ -41,21 +62,11 @@ import list from '../src/list.json'
       return
     }
 
-    const result = isbot(value)
-    output.appendChild(
-      document.createTextNode(
-        result
-          ? `I think so, yes\nThe pattern that was matched is ‟${find(value)}”`
-          : 'I don\'t think so, no\nI could not find a pattern I recognise'
-      )
-    )
+    isbot(value)
+      ? showMatch(output, value)
+      : noMatch(output)
 
     output.className = ''
     setTimeout(() => { output.className = 'highlight' }, 100)
   }
-
-  function find (ua) {
-    const match = ua.match(pattern)
-    return match && match[0]
-  }
-})()
+}

page/styles.css

@@ -46,3 +46,10 @@ output {
   white-space: pre-line;
   font-size: 2em;
 }
+
+kbd {
+  display: inline-block;
+  background: #eee;
+  font-size: 0.8em;
+  padding: 0 0.2em;
+}

scripts/sort/sortYamlFile/index.js

@@ -31,10 +31,8 @@ module.exports = async function sortYamlFile (filepath) {
     )
   )
 
-  YAML.scalarOptions.str.fold.lineWidth = Infinity
-
   await writeFile(
     filepath,
-    YAML.stringify(sorted)
+    YAML.stringify(sorted, undefined, { lineWidth: Infinity })
   )
 }

src/list.json

@@ -10,6 +10,7 @@
   "^<",
   "^[\\w\\s\\.]+/v?\\d+(\\.\\d+)?(\\.\\d{1,10})?$",
   "^[\\w]+$",
+  "^[^\\s]{50,}$",
   "^ace explorer",
   "^acoon",
   "^active",