wpt: add remaining fetch/api/basic tests

lib/fetch/dataURL.js

@@ -135,7 +135,7 @@ function URLSerializer (url, excludeFragment = false) {
     }
 
     // 3. Append url’s host, serialized, to output.
-    output += decodeURIComponent(url.host)
+    output += decodeURIComponent(url.hostname)
 
     // 4. If url’s port is non-null, append U+003A (:) followed by url’s port,
     // serialized, to output.

lib/fetch/index.js

@@ -753,31 +753,17 @@ async function schemeFetch (fetchParams) {
   // let request be fetchParams’s request
   const { request } = fetchParams
 
-  const {
-    protocol: scheme,
-    pathname: path
-  } = requestCurrentURL(request)
+  const { protocol: scheme } = requestCurrentURL(request)
 
   // switch on request’s current URL’s scheme, and run the associated steps:
   switch (scheme) {
     case 'about:': {
       // If request’s current URL’s path is the string "blank", then return a new response
       // whose status message is `OK`, header list is « (`Content-Type`, `text/html;charset=utf-8`) »,
       // and body is the empty byte sequence.
-      if (path === 'blank') {
-        const resp = makeResponse({
-          statusText: 'OK',
-          headersList: [
-            ['content-type', 'text/html;charset=utf-8']
-          ]
-        })
-
-        resp.urlList = [new URL('about:blank')]
-        return resp
-      }
 
       // Otherwise, return a network error.
-      return makeNetworkError('invalid path called')
+      return makeNetworkError('about scheme is not supported')
     }
     case 'blob:': {
       resolveObjectURL = resolveObjectURL || require('buffer').resolveObjectURL

lib/fetch/request.js

@@ -24,6 +24,7 @@ const { kEnumerableProperty } = util
 const { kHeaders, kSignal, kState, kGuard, kRealm } = require('./symbols')
 const { webidl } = require('./webidl')
 const { getGlobalOrigin } = require('./global')
+const { URLSerializer } = require('./dataURL')
 const { kHeadersList } = require('../core/symbols')
 const assert = require('assert')
 
@@ -542,7 +543,7 @@ class Request {
     }
 
     // The url getter steps are to return this’s request’s URL, serialized.
-    return this[kState].url.toString()
+    return URLSerializer(this[kState].url)
   }
 
   // Returns a Headers object consisting of the headers associated with request.

lib/fetch/response.js

@@ -5,7 +5,6 @@ const { extractBody, cloneBody, mixinBody } = require('./body')
 const util = require('../core/util')
 const { kEnumerableProperty } = util
 const {
-  responseURL,
   isValidReasonPhrase,
   isCancelled,
   isAborted,
@@ -22,6 +21,7 @@ const { kState, kHeaders, kGuard, kRealm } = require('./symbols')
 const { webidl } = require('./webidl')
 const { FormData } = require('./formdata')
 const { getGlobalOrigin } = require('./global')
+const { URLSerializer } = require('./dataURL')
 const { kHeadersList } = require('../core/symbols')
 const assert = require('assert')
 const { types } = require('util')
@@ -189,21 +189,18 @@ class Response {
       throw new TypeError('Illegal invocation')
     }
 
+    const urlList = this[kState].urlList
+
     // The url getter steps are to return the empty string if this’s
     // response’s URL is null; otherwise this’s response’s URL,
     // serialized with exclude fragment set to true.
-    let url = responseURL(this[kState])
+    const url = urlList[urlList.length - 1] ?? null
 
-    if (url == null) {
+    if (url === null) {
       return ''
     }
 
-    if (url.hash) {
-      url = new URL(url)
-      url.hash = ''
-    }
-
-    return url.toString()
+    return URLSerializer(url, true)
   }
 
   // Returns whether response was obtained through a redirect.

lib/fetch/util.js

@@ -532,8 +532,11 @@ function bytesMatch (bytes, metadataList) {
 
   // 4. Let metadata be the result of getting the strongest
   //    metadata from parsedMetadata.
-  // Note: this will only work for SHA- algorithms and it's lazy *at best*.
-  const metadata = parsedMetadata.sort((c, d) => d.algo.localeCompare(c.algo))
+  const list = parsedMetadata.sort((c, d) => d.algo.localeCompare(c.algo))
+  // get the strongest algorithm
+  const strongest = list[0].algo
+  // get all entries that use the strongest algorithm; ignore weaker
+  const metadata = list.filter((item) => item.algo === strongest)
 
   // 5. For each item in metadata:
   for (const item of metadata) {
@@ -544,7 +547,6 @@ function bytesMatch (bytes, metadataList) {
     const expectedValue = item.hash
 
     // 3. Let actualValue be the result of applying algorithm to bytes.
-    // Note: "applying algorithm to bytes" converts the result to base64
     const actualValue = crypto.createHash(algorithm).update(bytes).digest('base64')
 
     // 4. If actualValue is a case-sensitive match for expectedValue,
@@ -559,10 +561,9 @@ function bytesMatch (bytes, metadataList) {
 }
 
 // https://w3c.github.io/webappsec-subresource-integrity/#grammardef-hash-with-options
-// hash-algo is defined in Content Security Policy 2 Section 4.2
-// base64-value is similary defined there
-// VCHAR is defined https://www.rfc-editor.org/rfc/rfc5234#appendix-B.1
-const parseHashWithOptions = /((?<algo>sha256|sha384|sha512)-(?<hash>[A-z0-9+/]{1}.*={1,2}))( +[\x21-\x7e]?)?/i
+// https://www.w3.org/TR/CSP2/#source-list-syntax
+// https://www.rfc-editor.org/rfc/rfc5234#appendix-B.1
+const parseHashWithOptions = /((?<algo>sha256|sha384|sha512)-(?<hash>[A-z0-9+/]{1}.*={0,2}))( +[\x21-\x7e]?)?/i
 
 /**
  * @see https://w3c.github.io/webappsec-subresource-integrity/#parse-metadata

test/fetch/about-uri.js

@@ -5,12 +5,7 @@ const { fetch } = require('../..')
 
 test('fetching about: uris', async (t) => {
   t.test('about:blank', async (t) => {
-    const res = await fetch('about:blank')
-
-    t.equal(res.url, 'about:blank')
-    t.equal(res.statusText, 'OK')
-    t.equal(res.headers.get('Content-Type'), 'text/html;charset=utf-8')
-    t.end()
+    await t.rejects(fetch('about:blank'))
   })
 
   t.test('All other about: urls should return an error', async (t) => {

test/fetch/integrity.js

@@ -16,7 +16,7 @@ setGlobalDispatcher(new Agent({
 
 test('request with correct integrity checksum', (t) => {
   const body = 'Hello world!'
-  const hash = createHash('sha256').update(body).digest('hex')
+  const hash = createHash('sha256').update(body).digest('base64')
 
   const server = createServer((req, res) => {
     res.end(body)
@@ -58,7 +58,7 @@ test('request with wrong integrity checksum', (t) => {
 
 test('request with integrity checksum on encoded body', (t) => {
   const body = 'Hello world!'
-  const hash = createHash('sha256').update(body).digest('hex')
+  const hash = createHash('sha256').update(body).digest('base64')
 
   const server = createServer((req, res) => {
     res.setHeader('content-encoding', 'gzip')

test/node-fetch/main.js

@@ -1532,12 +1532,12 @@ describe('node-fetch', () => {
     })
   })
 
-  it('should keep `?` sign in URL when no params are given', () => {
+  it('should NOT keep `?` sign in URL when no params are given', () => {
     const url = `${base}question?`
     const urlObject = new URL(url)
     const request = new Request(urlObject)
     return fetch(request).then(res => {
-      expect(res.url).to.equal(url)
+      expect(res.url).to.equal(url.slice(0, -1))
       expect(res.ok).to.be.true
       expect(res.status).to.equal(200)
     })
@@ -1554,12 +1554,12 @@ describe('node-fetch', () => {
     })
   })
 
-  it('should preserve the hash (#) symbol', () => {
+  it('should NOT preserve the hash (#) symbol', () => {
     const url = `${base}question?#`
     const urlObject = new URL(url)
     const request = new Request(urlObject)
     return fetch(request).then(res => {
-      expect(res.url).to.equal(url)
+      expect(res.url).to.equal(url.slice(0, -2))
       expect(res.ok).to.be.true
       expect(res.status).to.equal(200)
     })

test/node-fetch/response.js

@@ -121,7 +121,7 @@ describe('Response', () => {
       status: 346,
       statusText: 'production'
     })
-    res[kState].urlList = [base]
+    res[kState].urlList = [new URL(base)]
     const cl = res.clone()
     expect(cl.headers.get('a')).to.equal('1')
     expect(cl.type).to.equal('default')

test/wpt/runner/runner/runner.mjs

@@ -3,7 +3,7 @@ import { readdirSync, readFileSync, statSync } from 'node:fs'
 import { basename, isAbsolute, join, resolve } from 'node:path'
 import { fileURLToPath } from 'node:url'
 import { Worker } from 'node:worker_threads'
-import { parseMeta, handlePipes } from './util.mjs'
+import { parseMeta, handlePipes, normalizeName } from './util.mjs'
 
 const basePath = fileURLToPath(join(import.meta.url, '../../..'))
 const testPath = join(basePath, 'tests')
@@ -81,7 +81,9 @@ export class WPTRunner extends EventEmitter {
     const workerPath = fileURLToPath(join(import.meta.url, '../worker.mjs'))
 
     for (const test of this.#files) {
-      const code = readFileSync(test, 'utf-8')
+      const code = test.includes('.sub.')
+        ? handlePipes(readFileSync(test, 'utf-8'), this.#url)
+        : readFileSync(test, 'utf-8')
       const meta = this.resolveMeta(code, test)
 
       const worker = new Worker(workerPath, {
@@ -122,15 +124,19 @@ export class WPTRunner extends EventEmitter {
    * Called after a test has succeeded or failed.
    */
   handleIndividualTestCompletion (message, fileName) {
-    const { fail, allowUnexpectedFailures } = this.#status[fileName] ?? {}
+    const { fail, allowUnexpectedFailures, flaky } = this.#status[fileName] ?? {}
 
     if (message.type === 'result') {
       this.#stats.completed += 1
 
       if (message.result.status === 1) {
         this.#stats.failed += 1
 
-        if (allowUnexpectedFailures || (fail && fail.includes(message.result.name))) {
+        const name = normalizeName(message.result.name)
+
+        if (flaky?.includes(name)) {
+          this.#stats.expectedFailures += 1
+        } else if (allowUnexpectedFailures || fail?.includes(name)) {
           this.#stats.expectedFailures += 1
         } else {
           process.exitCode = 1

test/wpt/runner/runner/util.mjs

@@ -1,4 +1,5 @@
 import { exit } from 'node:process'
+import { inspect } from 'node:util'
 
 /**
  * Parse the `Meta:` tags sometimes included in tests.
@@ -117,3 +118,16 @@ export function handlePipes (code, url) {
     }
   })
 }
+
+/**
+ * Some test names may contain characters that JSON cannot handle.
+ * @param {string} name
+ */
+export function normalizeName (name) {
+  return name.replace(/(\v)/g, (_, match) => {
+    switch (inspect(match)) {
+      case '\'\\x0B\'': return '\\x0B'
+      default: return match
+    }
+  })
+}

test/wpt/server/routes/network-partition-key.mjs

@@ -0,0 +1,111 @@
+const stash = new Map()
+
+/**
+ * @see https://github.com/web-platform-tests/wpt/blob/master/fetch/connection-pool/resources/network-partition-key.py
+ * @param {Parameters<import('http').RequestListener>[0]} req
+ * @param {Parameters<import('http').RequestListener>[1]} res
+ * @param {URL} url
+ */
+export function route (req, res, { searchParams, port }) {
+  res.setHeader('Cache-Control', 'no-store')
+
+  const dispatch = searchParams.get('dispatch')
+  const uuid = searchParams.get('uuid')
+  const partitionId = searchParams.get('partition_id')
+
+  if (!uuid || !dispatch || !partitionId) {
+    res.statusCode = 404
+    res.end('Invalid query parameters')
+    return
+  }
+
+  let testFailed = false
+  let requestCount = 0
+  let connectionCount = 0
+
+  if (searchParams.get('nocheck_partition') !== 'True') {
+    const addressKey = `${req.socket.localAddress}|${port}`
+    const serverState = stash.get(uuid) ?? {
+      testFailed: false,
+      requestCount: 0,
+      connectionCount: 0
+    }
+
+    stash.delete(uuid)
+    requestCount = serverState.requestCount + 1
+    serverState.requestCount = requestCount
+
+    if (Object.hasOwn(serverState, addressKey)) {
+      if (serverState[addressKey] !== partitionId) {
+        serverState.testFailed = true
+      }
+    } else {
+      connectionCount = serverState.connectionCount + 1
+      serverState.connectionCount = connectionCount
+    }
+
+    serverState[addressKey] = partitionId
+    testFailed = serverState.testFailed
+    stash.set(uuid, serverState)
+  }
+
+  const origin = req.headers.origin
+  if (origin) {
+    res.setHeader('Access-Control-Allow-Origin', origin)
+    res.setHeader('Access-Control-Allow-Credentials', 'true')
+  }
+
+  if (req.method === 'OPTIONS') {
+    return handlePreflight(req, res)
+  }
+
+  if (dispatch === 'fetch_file') {
+    res.end()
+    return
+  }
+
+  if (dispatch === 'check_partition') {
+    const status = searchParams.get('status') ?? 200
+
+    if (testFailed) {
+      res.statusCode = status
+      res.end('Multiple partition IDs used on a socket')
+      return
+    }
+
+    let body = 'ok'
+    if (searchParams.get('addcounter')) {
+      body += `. Request was sent ${requestCount} times. ${connectionCount} connections were created.`
+      res.statusCode = status
+      res.end(body)
+      return
+    }
+  }
+
+  if (dispatch === 'clean_up') {
+    stash.delete(uuid)
+    res.statusCode = 200
+    if (testFailed) {
+      res.end('Test failed, but cleanup completed.')
+    } else {
+      res.end('cleanup complete')
+    }
+
+    return
+  }
+
+  res.statusCode = 404
+  res.end('Unrecognized dispatch parameter: ' + dispatch)
+}
+
+/**
+ * @param {Parameters<import('http').RequestListener>[0]} req
+ * @param {Parameters<import('http').RequestListener>[1]} res
+ */
+function handlePreflight (req, res) {
+  res.statusCode = 200
+  res.setHeader('Access-Control-Allow-Methods', 'GET')
+  res.setHeader('Access-Control-Allow-Headers', 'header-to-force-cors')
+  res.setHeader('Access-Control-Max-Age', '86400')
+  res.end('Preflight request')
+}