Merge pull request #696 from microlinkhq/favicon/content-type

fix(logo-favicon): favicon.ico has valid content-type

packages/metascraper-audio/package.json

@@ -40,5 +40,11 @@
   "scripts": {
     "test": "NODE_PATH=.. TZ=UTC ava --timeout 15s"
   },
-  "license": "MIT"
+  "license": "MIT",
+  "ava": {
+    "files": [
+      "test/**/*.js",
+      "!test/helpers.js"
+    ]
+  }
 }

packages/metascraper-audio/test/helpers.js

@@ -0,0 +1,24 @@
+'use strict'
+
+const { default: listen } = require('async-listen')
+const { createServer } = require('http')
+
+const closeServer = server =>
+  require('util').promisify(server.close.bind(server))()
+
+const runServer = async (t, handler, opts) => {
+  const server = createServer(async (req, res) => {
+    try {
+      await handler({ req, res })
+    } catch (error) {
+      console.error(error)
+      res.statusCode = 500
+      res.end()
+    }
+  })
+  const url = await listen(server, { port: 0, host: '0.0.0.0', ...opts })
+  t.teardown(() => closeServer(server))
+  return url.toString()
+}
+
+module.exports = { runServer }

packages/metascraper-audio/test/iframe.js

@@ -1,55 +1,43 @@
 'use strict'
 
-const { default: listen } = require('async-listen')
-const { createServer } = require('http')
 const test = require('ava')
 
-const closeServer = server =>
-  require('util').promisify(server.close.bind(server))()
+const { runServer } = require('./helpers')
 
 const createMetascraper = (...args) =>
   require('metascraper')([require('../src')(...args)])
 
 test('absolute http', async t => {
-  const server = createServer((_, res) => {
+  const url = await runServer(t, ({ res }) => {
     res.setHeader('Content-Type', 'text/html')
     res.end(
       '<meta property="og:audio" content="https://cdn.microlink.io/file-examples/sample.mp3">'
     )
   })
-
-  t.teardown(() => closeServer(server))
-  const url = (await listen(server, { port: 0, host: '0.0.0.0' })).toString()
   const html = `<iframe src="${url}">`
   const metascraper = createMetascraper()
   const metadata = await metascraper({ html, url })
   t.is(metadata.audio, 'https://cdn.microlink.io/file-examples/sample.mp3')
 })
 
 test('relative http', async t => {
-  const server = createServer((_, res) => {
+  const url = await runServer(t, ({ res }) => {
     res.setHeader('Content-Type', 'text/html')
     res.end('<meta property="og:audio" content="/file-examples/sample.mp3">')
   })
-
-  t.teardown(() => closeServer(server))
-  const url = (await listen(server, { port: 0, host: '0.0.0.0' })).toString()
   const html = '<iframe src="/">'
   const metascraper = createMetascraper()
   const metadata = await metascraper({ html, url })
   t.is(metadata.audio, url + 'file-examples/sample.mp3')
 })
 
 test('ignore non http urls', async t => {
-  const server = createServer((_, res) => {
+  const url = await runServer(t, ({ res }) => {
     res.setHeader('Content-Type', 'text/html')
     res.end(
       '<meta property="og:audio" content="tg://join?invite=n3gS0R7pjFJhMWM0">'
     )
   })
-
-  t.teardown(() => closeServer(server))
-  const url = (await listen(server, { port: 0, host: '0.0.0.0' })).toString()
   const html = `<iframe src="${url}">`
   const metascraper = createMetascraper()
   const metadata = await metascraper({ html, url })

packages/metascraper-logo-favicon/package.json

@@ -30,6 +30,7 @@
     "reachable-url": "~1.8.0"
   },
   "devDependencies": {
+    "async-listen": "latest",
     "ava": "5"
   },
   "engines": {
@@ -41,5 +42,11 @@
   "scripts": {
     "test": "NODE_PATH=.. TZ=UTC ava --timeout 15s"
   },
-  "license": "MIT"
+  "license": "MIT",
+  "ava": {
+    "files": [
+      "test/**/*.js",
+      "!test/helpers.js"
+    ]
+  }
 }

packages/metascraper-logo-favicon/src/index.js

@@ -110,10 +110,18 @@ pickBiggerSize.sortBySize = collection =>
 const favicon = async (url, { gotOpts } = {}) => {
   const faviconUrl = logo('/favicon.ico', { url })
   if (!faviconUrl) return undefined
+
   const response = await reachableUrl(faviconUrl, gotOpts)
-  return reachableUrl.isReachable(response) &&
-    response.headers['content-type']?.startsWith('image')
-    ? faviconUrl
+  const contentType = response.headers['content-type']
+
+  const isValidContenType =
+    contentType &&
+    ['image/vnd.microsoft.icon', 'image/x-icon'].some(ct =>
+      contentType.includes(ct)
+    )
+
+  return isValidContenType && reachableUrl.isReachable(response)
+    ? response.url
     : undefined
 }
 

packages/metascraper-logo-favicon/test/favicon.js

@@ -4,12 +4,42 @@ const test = require('ava')
 
 const { favicon } = require('..')
 
+const { runServer } = require('./helpers')
+
 test('return undefined if favicon is not reachable', async t => {
   const url = 'https://idontexist.lol'
   t.is(await favicon(url), undefined)
 })
 
-test("with { contentType: 'image/vnd.microsoft.icon' }", async t => {
+test("don't resolve favicon.ico with no content-type", async t => {
+  const url = await runServer(t, async ({ res }) => {
+    res.end('<svg></svg>')
+  })
+  t.is(await favicon(url), undefined)
+})
+
+test("don't resolve favicon.ico with no valid content-type", async t => {
+  const url = await runServer(t, async ({ res }) => {
+    res.setHeader('content-type', 'image/svg+xml; charset=utf-8')
+    res.end('<svg></svg>')
+  })
+  t.is(await favicon(url), undefined)
+})
+
+test("favicon.ico with 'image/vnd.microsoft.icon' content-type", async t => {
   const url = 'https://microlink.io/'
   t.is(await favicon(url), 'https://microlink.io/favicon.ico')
 })
+
+test("favicon.ico with 'image/x-icon' content-type", async t => {
+  const url = 'https://2miners.com/'
+  t.is(await favicon(url), 'https://2miners.com/favicon.ico')
+})
+
+test('handle redirects', async t => {
+  const url = await runServer(t, async ({ res }) => {
+    res.writeHead(301, { Location: 'https://microlink.io/favicon.ico' })
+    res.end()
+  })
+  t.is(await favicon(url), 'https://microlink.io/favicon.ico')
+})

packages/metascraper-logo-favicon/test/helpers.js

@@ -0,0 +1,24 @@
+'use strict'
+
+const { default: listen } = require('async-listen')
+const { createServer } = require('http')
+
+const closeServer = server =>
+  require('util').promisify(server.close.bind(server))()
+
+const runServer = async (t, handler, opts) => {
+  const server = createServer(async (req, res) => {
+    try {
+      await handler({ req, res })
+    } catch (error) {
+      console.error(error)
+      res.statusCode = 500
+      res.end()
+    }
+  })
+  const url = await listen(server, { port: 0, host: '0.0.0.0', ...opts })
+  t.teardown(() => closeServer(server))
+  return url.toString()
+}
+
+module.exports = { runServer }

packages/metascraper-media-provider/src/get-media/util.js

@@ -11,13 +11,7 @@ const isTweetUrl = url =>
   isTweet(url) && parseUrl(url).domainWithoutSuffix === 'twitter'
 
 const getTweetId = url =>
-  chain(url)
-    .split('/')
-    .reverse()
-    .first()
-    .split('?')
-    .first()
-    .value()
+  chain(url).split('/').reverse().first().split('?').first().value()
 
 const expirableCounter = (value = 0, ttl = TEN_MIN_MS) => {
   let timestamp = Date.now()

packages/metascraper-media-provider/test/helpers.js

@@ -8,7 +8,7 @@ const { PROXY_PASSWORD, PROXY_USERNAME, PROXY_HOST } = process.env
 const proxy =
   PROXY_PASSWORD && PROXY_USERNAME && PROXY_HOST
     ? parseProxyUri(
-        `socks5://${PROXY_USERNAME}:${PROXY_PASSWORD}@${PROXY_HOST}`
+      `socks5://${PROXY_USERNAME}:${PROXY_PASSWORD}@${PROXY_HOST}`
     )
     : undefined
 

packages/metascraper-video/package.json

@@ -39,5 +39,11 @@
   "scripts": {
     "test": "NODE_PATH=.. TZ=UTC ava --timeout 15s"
   },
-  "license": "MIT"
+  "license": "MIT",
+  "ava": {
+    "files": [
+      "test/**/*.js",
+      "!test/helpers.js"
+    ]
+  }
 }

packages/metascraper-video/test/helpers.js

@@ -0,0 +1,24 @@
+'use strict'
+
+const { default: listen } = require('async-listen')
+const { createServer } = require('http')
+
+const closeServer = server =>
+  require('util').promisify(server.close.bind(server))()
+
+const runServer = async (t, handler, opts) => {
+  const server = createServer(async (req, res) => {
+    try {
+      await handler({ req, res })
+    } catch (error) {
+      console.error(error)
+      res.statusCode = 500
+      res.end()
+    }
+  })
+  const url = await listen(server, { port: 0, host: '0.0.0.0', ...opts })
+  t.teardown(() => closeServer(server))
+  return url.toString()
+}
+
+module.exports = { runServer }

packages/metascraper-video/test/iframe.js

@@ -1,55 +1,43 @@
 'use strict'
 
-const { default: listen } = require('async-listen')
-const { createServer } = require('http')
-const { promisify } = require('util')
 const test = require('ava')
 
-const closeServer = server => promisify(server.close)
+const { runServer } = require('./helpers')
 
 const createMetascraper = (...args) =>
   require('metascraper')([require('../src')(...args)])
 
 test('absolute http', async t => {
-  const server = createServer((_, res) => {
+  const url = await runServer(t, ({ res }) => {
     res.setHeader('Content-Type', 'text/html')
     res.end(
       '<meta property="og:video" content="https://cdn.microlink.io/file-examples/sample.mp4">'
     )
   })
-
-  t.teardown(() => closeServer(server))
-  const url = (await listen(server, { port: 0, host: '0.0.0.0' })).toString()
   const html = `<iframe src="${url}">`
   const metascraper = createMetascraper()
   const metadata = await metascraper({ html, url })
   t.is(metadata.video, 'https://cdn.microlink.io/file-examples/sample.mp4')
 })
 
 test('relative http', async t => {
-  const server = createServer((_, res) => {
+  const url = await runServer(t, ({ res }) => {
     res.setHeader('Content-Type', 'text/html')
     res.end('<meta property="og:video" content="/file-examples/sample.mp4">')
   })
-
-  t.teardown(() => closeServer(server))
-  const url = (await listen(server, { port: 0, host: '0.0.0.0' })).toString()
   const html = '<iframe src="/">'
   const metascraper = createMetascraper()
   const metadata = await metascraper({ html, url })
   t.is(metadata.video, url + 'file-examples/sample.mp4')
 })
 
 test('ignore non http urls', async t => {
-  const server = createServer((_, res) => {
+  const url = await runServer(t, ({ res }) => {
     res.setHeader('Content-Type', 'text/html')
     res.end(
       '<meta property="og:video" content="tg://join?invite=n3gS0R7pjFJhMWM0">'
     )
   })
-
-  t.teardown(() => closeServer(server))
-  const url = (await listen(server, { port: 0, host: '0.0.0.0' })).toString()
   const html = `<iframe src="${url}">`
   const metascraper = createMetascraper()
   const metadata = await metascraper({ html, url })