From 4937381db425297f789bbe1f507dd3b062df8cfa Mon Sep 17 00:00:00 2001 From: Khafra <42794878+KhafraDev@users.noreply.github.com> Date: Wed, 12 Oct 2022 20:30:47 -0400 Subject: [PATCH] feat: implement spec-compliant body mixins (#1694) * feat: implement spec-compliant body mixins * fix: skip tests on v16.8 --- lib/fetch/body.js | 289 +++++++++++++++-------- lib/fetch/dataURL.js | 6 +- test/fetch/client-fetch.js | 4 +- test/fetch/data-uri.js | 9 +- test/fetch/response.js | 4 +- test/wpt/runner/runner/runner.mjs | 4 +- test/wpt/runner/runner/util.mjs | 6 - test/wpt/status/fetch.status.json | 3 +- test/wpt/tests/resources/WebIDLParser.js | 1 - 9 files changed, 203 insertions(+), 123 deletions(-) delete mode 120000 test/wpt/tests/resources/WebIDLParser.js diff --git a/lib/fetch/body.js b/lib/fetch/body.js index 0891fa87282..4356a0371cb 100644 --- a/lib/fetch/body.js +++ b/lib/fetch/body.js @@ -13,6 +13,8 @@ const assert = require('assert') const { isErrored } = require('../core/util') const { isUint8Array, isArrayBuffer } = require('util/types') const { File } = require('./file') +const { StringDecoder } = require('string_decoder') +const { parseMIMEType } = require('./dataURL') let ReadableStream @@ -301,117 +303,28 @@ function throwIfAborted (state) { function bodyMixinMethods (instance) { const methods = { - async blob () { - if (!(this instanceof instance)) { - throw new TypeError('Illegal invocation') - } - - throwIfAborted(this[kState]) - - const chunks = [] - - for await (const chunk of consumeBody(this[kState].body)) { - if (!isUint8Array(chunk)) { - throw new TypeError('Expected Uint8Array chunk') - } - - // Assemble one final large blob with Uint8Array's can exhaust memory. - // That's why we create create multiple blob's and using references - chunks.push(new Blob([chunk])) - } - - return new Blob(chunks, { type: this.headers.get('Content-Type') || '' }) + blob () { + // The blob() method steps are to return the result of + // running consume body with this and Blob. + return specConsumeBody(this, 'Blob', instance) }, - async arrayBuffer () { - if (!(this instanceof instance)) { - throw new TypeError('Illegal invocation') - } - - throwIfAborted(this[kState]) - - const contentLength = this.headers.get('content-length') - const encoded = this.headers.has('content-encoding') - - // if we have content length and no encoding, then we can - // pre allocate the buffer and just read the data into it - if (!encoded && contentLength) { - const buffer = new Uint8Array(contentLength) - let offset = 0 - - for await (const chunk of consumeBody(this[kState].body)) { - if (!isUint8Array(chunk)) { - throw new TypeError('Expected Uint8Array chunk') - } - - buffer.set(chunk, offset) - offset += chunk.length - } - - return buffer.buffer - } - - // if we don't have content length, then we have to allocate 2x the - // size of the body, once for consumed data, and once for the final buffer - - // This could be optimized by using growable ArrayBuffer, but it's not - // implemented yet. https://github.com/tc39/proposal-resizablearraybuffer - - const chunks = [] - let size = 0 - - for await (const chunk of consumeBody(this[kState].body)) { - if (!isUint8Array(chunk)) { - throw new TypeError('Expected Uint8Array chunk') - } - - chunks.push(chunk) - size += chunk.byteLength - } - - const buffer = new Uint8Array(size) - let offset = 0 - - for (const chunk of chunks) { - buffer.set(chunk, offset) - offset += chunk.byteLength - } - - return buffer.buffer + arrayBuffer () { + // The arrayBuffer() method steps are to return the + // result of running consume body with this and ArrayBuffer. + return specConsumeBody(this, 'ArrayBuffer', instance) }, - async text () { - if (!(this instanceof instance)) { - throw new TypeError('Illegal invocation') - } - - throwIfAborted(this[kState]) - - let result = '' - const textDecoder = new TextDecoder() - - for await (const chunk of consumeBody(this[kState].body)) { - if (!isUint8Array(chunk)) { - throw new TypeError('Expected Uint8Array chunk') - } - - result += textDecoder.decode(chunk, { stream: true }) - } - - // flush - result += textDecoder.decode() - - return result + text () { + // The text() method steps are to return the result of + // running consume body with this and text. + return specConsumeBody(this, 'text', instance) }, - async json () { - if (!(this instanceof instance)) { - throw new TypeError('Illegal invocation') - } - - throwIfAborted(this[kState]) - - return JSON.parse(await this.text()) + json () { + // The json() method steps are to return the result of + // running consume body with this and JSON. + return specConsumeBody(this, 'JSON', instance) }, async formData () { @@ -534,6 +447,172 @@ function mixinBody (prototype) { Object.assign(prototype.prototype, bodyMixinMethods(prototype)) } +// https://fetch.spec.whatwg.org/#concept-body-consume-body +async function specConsumeBody (object, type, instance) { + if (!(object instanceof instance)) { + throw new TypeError('Illegal invocation') + } + + // TODO: why is this needed? + throwIfAborted(object[kState]) + + // 1. If object is unusable, then return a promise rejected + // with a TypeError. + if (bodyUnusable(object[kState].body)) { + throw new TypeError('Body is unusable') + } + + // 2. Let promise be a promise resolved with an empty byte + // sequence. + let promise + + // 3. If object’s body is non-null, then set promise to the + // result of fully reading body as promise given object’s + // body. + if (object[kState].body != null) { + promise = await fullyReadBodyAsPromise(object[kState].body) + } else { + // step #2 + promise = { size: 0, bytes: [new Uint8Array()] } + } + + // 4. Let steps be to return the result of package data with + // the first argument given, type, and object’s MIME type. + const mimeType = type === 'Blob' || type === 'FormData' + ? bodyMimeType(object) + : undefined + + // 5. Return the result of upon fulfillment of promise given + // steps. + return packageData(promise, type, mimeType) +} + +/** + * @see https://fetch.spec.whatwg.org/#concept-body-package-data + * @param {{ size: number, bytes: Uint8Array[] }} bytes + * @param {string} type + * @param {ReturnType|undefined} mimeType + */ +function packageData ({ bytes, size }, type, mimeType) { + switch (type) { + case 'ArrayBuffer': { + // Return a new ArrayBuffer whose contents are bytes. + const uint8 = new Uint8Array(size) + let offset = 0 + + for (const chunk of bytes) { + uint8.set(chunk, offset) + offset += chunk.byteLength + } + + return uint8.buffer + } + case 'Blob': { + // Return a Blob whose contents are bytes and type attribute + // is mimeType. + return new Blob(bytes, { type: mimeType?.essence }) + } + case 'JSON': { + // Return the result of running parse JSON from bytes on bytes. + return JSON.parse(utf8DecodeBytes(bytes)) + } + case 'text': { + // 1. Return the result of running UTF-8 decode on bytes. + return utf8DecodeBytes(bytes) + } + } +} + +// https://fetch.spec.whatwg.org/#body-unusable +function bodyUnusable (body) { + // An object including the Body interface mixin is + // said to be unusable if its body is non-null and + // its body’s stream is disturbed or locked. + return body != null && (body.stream.locked || util.isDisturbed(body.stream)) +} + +// https://fetch.spec.whatwg.org/#fully-reading-body-as-promise +async function fullyReadBodyAsPromise (body) { + // 1. Let reader be the result of getting a reader for body’s + // stream. If that threw an exception, then return a promise + // rejected with that exception. + const reader = body.stream.getReader() + + // 2. Return the result of reading all bytes from reader. + /** @type {Uint8Array[]} */ + const bytes = [] + let size = 0 + + while (true) { + const { done, value } = await reader.read() + + if (done) { + break + } + + // https://streams.spec.whatwg.org/#read-loop + // If chunk is not a Uint8Array object, reject promise with + // a TypeError and abort these steps. + if (!isUint8Array(value)) { + throw new TypeError('Value is not a Uint8Array.') + } + + bytes.push(value) + size += value.byteLength + } + + return { size, bytes } +} + +/** + * @see https://encoding.spec.whatwg.org/#utf-8-decode + * @param {Uint8Array[]} ioQueue + */ +function utf8DecodeBytes (ioQueue) { + if (ioQueue.length === 0) { + return '' + } + + // 1. Let buffer be the result of peeking three bytes + // from ioQueue, converted to a byte sequence. + const buffer = ioQueue[0] + + // 2. If buffer is 0xEF 0xBB 0xBF, then read three + // bytes from ioQueue. (Do nothing with those bytes.) + if (buffer[0] === 0xEF && buffer[1] === 0xBB && buffer[2] === 0xBF) { + ioQueue[0] = ioQueue[0].subarray(3) + } + + // 3. Process a queue with an instance of UTF-8’s + // decoder, ioQueue, output, and "replacement". + const decoder = new StringDecoder('utf-8') + let output = '' + + for (const chunk of ioQueue) { + output += decoder.write(chunk) + } + + output += decoder.end() + + // 4. Return output. + return output +} + +/** + * @see https://fetch.spec.whatwg.org/#concept-body-mime-type + * @param {import('./response').Response|import('./request').Request} object + */ +function bodyMimeType (object) { + const { headersList } = object[kState] + const contentType = headersList.get('content-type') + + if (contentType === null) { + return 'failure' + } + + return parseMIMEType(contentType) +} + module.exports = { extractBody, safelyExtractBody, diff --git a/lib/fetch/dataURL.js b/lib/fetch/dataURL.js index 71e5b35ba3a..a1c81a54a1d 100644 --- a/lib/fetch/dataURL.js +++ b/lib/fetch/dataURL.js @@ -321,7 +321,11 @@ function parseMIMEType (input) { type: type.toLowerCase(), subtype: subtype.toLowerCase(), /** @type {Map} */ - parameters: new Map() + parameters: new Map(), + // https://mimesniff.spec.whatwg.org/#mime-type-essence + get essence () { + return `${this.type}/${this.subtype}` + } } // 11. While position is not past the end of input: diff --git a/test/fetch/client-fetch.js b/test/fetch/client-fetch.js index 58151f7c66e..a378dc7d7dd 100644 --- a/test/fetch/client-fetch.js +++ b/test/fetch/client-fetch.js @@ -316,7 +316,7 @@ test('locked blob body', (t) => { const res = await fetch(`http://localhost:${server.address().port}`) const reader = res.body.getReader() res.blob().catch(err => { - t.equal(err.message, 'The stream is locked.') + t.equal(err.message, 'Body is unusable') reader.cancel() }) }) @@ -336,7 +336,7 @@ test('disturbed blob body', (t) => { t.pass(2) }) res.blob().catch(err => { - t.equal(err.message, 'The body has already been consumed.') + t.equal(err.message, 'Body is unusable') }) }) }) diff --git a/test/fetch/data-uri.js b/test/fetch/data-uri.js index c05fb8b09d6..291b7576f8b 100644 --- a/test/fetch/data-uri.js +++ b/test/fetch/data-uri.js @@ -113,19 +113,22 @@ test('https://mimesniff.spec.whatwg.org/#parse-a-mime-type', (t) => { t.same(parseMIMEType('text/plain'), { type: 'text', subtype: 'plain', - parameters: new Map() + parameters: new Map(), + essence: 'text/plain' }) t.same(parseMIMEType('text/html;charset="shift_jis"iso-2022-jp'), { type: 'text', subtype: 'html', - parameters: new Map([['charset', 'shift_jis']]) + parameters: new Map([['charset', 'shift_jis']]), + essence: 'text/html' }) t.same(parseMIMEType('application/javascript'), { type: 'application', subtype: 'javascript', - parameters: new Map() + parameters: new Map(), + essence: 'application/javascript' }) t.end() diff --git a/test/fetch/response.js b/test/fetch/response.js index c031d6d90d1..2342f0927ff 100644 --- a/test/fetch/response.js +++ b/test/fetch/response.js @@ -171,7 +171,7 @@ test('Modifying headers using Headers.prototype.set', (t) => { }) // https://github.com/nodejs/node/issues/43838 -test('constructing a Response with a ReadableStream body', async (t) => { +test('constructing a Response with a ReadableStream body', { skip: process.version.startsWith('v16.') }, async (t) => { const text = '{"foo":"bar"}' const uint8 = new TextEncoder().encode(text) @@ -209,7 +209,7 @@ test('constructing a Response with a ReadableStream body', async (t) => { t.end() }) - t.test('Readable with ArrayBuffer chunk still throws', async (t) => { + t.test('Readable with ArrayBuffer chunk still throws', { skip: process.version.startsWith('v16.') }, async (t) => { const readable = new ReadableStream({ start (controller) { controller.enqueue(uint8.buffer) diff --git a/test/wpt/runner/runner/runner.mjs b/test/wpt/runner/runner/runner.mjs index 7bf77d6e019..f9a37c24cb7 100644 --- a/test/wpt/runner/runner/runner.mjs +++ b/test/wpt/runner/runner/runner.mjs @@ -3,7 +3,7 @@ import { readdirSync, readFileSync, statSync } from 'node:fs' import { basename, isAbsolute, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { Worker } from 'node:worker_threads' -import { parseMeta, resolveSymLink } from './util.mjs' +import { parseMeta } from './util.mjs' const basePath = fileURLToPath(join(import.meta.url, '../../..')) const testPath = join(basePath, 'tests') @@ -162,7 +162,7 @@ export class WPTRunner extends EventEmitter { const scripts = meta.scripts.map((script) => { if (script === '/resources/WebIDLParser.js') { // See https://github.com/web-platform-tests/wpt/pull/731 - return resolveSymLink(join(testPath, script)) + return readFileSync(join(testPath, '/resources/webidl2/lib/webidl2.js'), 'utf-8') } else if (isAbsolute(script)) { return readFileSync(join(testPath, script), 'utf-8') } diff --git a/test/wpt/runner/runner/util.mjs b/test/wpt/runner/runner/util.mjs index 9f75a5f0b60..a816e700a50 100644 --- a/test/wpt/runner/runner/util.mjs +++ b/test/wpt/runner/runner/util.mjs @@ -1,5 +1,4 @@ import { exit } from 'node:process' -import { readFileSync, readlinkSync } from 'node:fs' /** * Parse the `Meta:` tags sometimes included in tests. @@ -64,8 +63,3 @@ export function parseMeta (fileContents) { return meta } - -export function resolveSymLink (path) { - const symlink = readlinkSync(path) - return readFileSync(symlink, 'utf-8') -} diff --git a/test/wpt/status/fetch.status.json b/test/wpt/status/fetch.status.json index 1a034fa16a1..8588dc07dea 100644 --- a/test/wpt/status/fetch.status.json +++ b/test/wpt/status/fetch.status.json @@ -26,7 +26,8 @@ }, "idlharness.any.js": { "fail": [ - "Response interface: operation json(any, optional ResponseInit)" + "Response interface: operation json(any, optional ResponseInit)", + "Window interface: operation fetch(RequestInfo, optional RequestInit)" ] } } \ No newline at end of file diff --git a/test/wpt/tests/resources/WebIDLParser.js b/test/wpt/tests/resources/WebIDLParser.js deleted file mode 120000 index d4d590faf9d..00000000000 --- a/test/wpt/tests/resources/WebIDLParser.js +++ /dev/null @@ -1 +0,0 @@ -./test/wpt/tests/resources/webidl2/lib/webidl2.js \ No newline at end of file