diff --git a/README.md b/README.md index 046030666..a8189f3f5 100644 --- a/README.md +++ b/README.md @@ -381,6 +381,20 @@ const options = { })(); ``` +node-fetch also supports spec-compliant FormData implementations such as [formdata-node](https://github.com/octet-stream/form-data): + +```js +const fetch = require('node-fetch'); +const FormData = require('formdata-node'); + +const form = new FormData(); +form.set('greeting', 'Hello, world!'); + +fetch('https://httpbin.org/post', {method: 'POST', body: form}) + .then(res => res.json()) + .then(json => console.log(json)); +``` + ### Request cancellation with AbortSignal You may cancel requests with `AbortController`. A suggested implementation is [`abort-controller`](https://www.npmjs.com/package/abort-controller). diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 0763dca86..87ec8d36e 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -33,7 +33,7 @@ Changelog ## v3.0.0-beta.5 -> NOTE: Since the previous beta version included serious issues, such as [#749](https://github.com/node-fetch/node-fetch/issues/749), they will now be deprecated. +> NOTE: Since the previous beta version included serious issues, such as [#749](https://github.com/node-fetch/node-fetch/issues/749), they will now be deprecated. - Enhance: use built-in AbortSignal for typings. - Enhance: compile CJS modules as a seperate set of files. diff --git a/package.json b/package.json index ac59b8361..818313bdb 100644 --- a/package.json +++ b/package.json @@ -49,6 +49,7 @@ "devDependencies": { "abort-controller": "^3.0.0", "abortcontroller-polyfill": "^1.4.0", + "busboy": "^0.3.1", "c8": "^7.1.2", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", @@ -57,6 +58,7 @@ "coveralls": "^3.1.0", "delay": "^4.3.0", "form-data": "^3.0.0", + "formdata-node": "^2.0.0", "mocha": "^7.2.0", "p-timeout": "^3.2.0", "parted": "^0.1.1", @@ -94,7 +96,15 @@ "import/extensions": 0, "import/no-useless-path-segments": 0, "unicorn/import-index": 0, - "capitalized-comments": 0 + "capitalized-comments": 0, + "node/no-unsupported-features/node-builtins": [ + "error", + { + "ignores": [ + "stream.Readable.from" + ] + } + ] }, "ignores": [ "dist", diff --git a/src/body.js b/src/body.js index b6cf43dfd..f1233034d 100644 --- a/src/body.js +++ b/src/body.js @@ -9,9 +9,11 @@ import Stream, {PassThrough} from 'stream'; import {types} from 'util'; import Blob from 'fetch-blob'; + import {FetchError} from './errors/fetch-error.js'; import {FetchBaseError} from './errors/base.js'; -import {isBlob, isURLSearchParameters} from './utils/is.js'; +import {formDataIterator, getBoundary, getFormDataLength} from './utils/form-data.js'; +import {isBlob, isURLSearchParameters, isFormData} from './utils/is.js'; const INTERNALS = Symbol('Body internals'); @@ -28,6 +30,8 @@ export default class Body { constructor(body, { size = 0 } = {}) { + let boundary = null; + if (body === null) { // Body is undefined or null body = null; @@ -46,6 +50,10 @@ export default class Body { body = Buffer.from(body.buffer, body.byteOffset, body.byteLength); } else if (body instanceof Stream) { // Body is stream + } else if (isFormData(body)) { + // Body is an instance of formdata-node + boundary = `NodeFetchFormDataBoundary${getBoundary()}`; + body = Stream.Readable.from(formDataIterator(body, boundary)); } else { // None of the above // coerce to string then buffer @@ -54,6 +62,7 @@ export default class Body { this[INTERNALS] = { body, + boundary, disturbed: false, error: null }; @@ -146,7 +155,7 @@ Object.defineProperties(Body.prototype, { * * Ref: https://fetch.spec.whatwg.org/#concept-body-consume-body * - * @return Promise + * @return Promise */ async function consumeBody(data) { if (data[INTERNALS].disturbed) { @@ -264,7 +273,7 @@ export const clone = (instance, highWaterMark) => { * @param {any} body Any options.body input * @returns {string | null} */ -export const extractContentType = body => { +export const extractContentType = (body, request) => { // Body is null or undefined if (body === null) { return null; @@ -295,6 +304,10 @@ export const extractContentType = body => { return `multipart/form-data;boundary=${body.getBoundary()}`; } + if (isFormData(body)) { + return `multipart/form-data; boundary=${request[INTERNALS].boundary}`; + } + // Body is stream - can't really do much about this if (body instanceof Stream) { return null; @@ -313,7 +326,9 @@ export const extractContentType = body => { * @param {any} obj.body Body object from the Body instance. * @returns {number | null} */ -export const getTotalBytes = ({body}) => { +export const getTotalBytes = request => { + const {body} = request; + // Body is null or undefined if (body === null) { return 0; @@ -334,6 +349,11 @@ export const getTotalBytes = ({body}) => { return body.hasKnownLength && body.hasKnownLength() ? body.getLengthSync() : null; } + // Body is a spec-compliant form-data + if (isFormData(body)) { + return getFormDataLength(request[INTERNALS].boundary); + } + // Body is stream return null; }; diff --git a/src/request.js b/src/request.js index bb0df344e..83e5d0a84 100644 --- a/src/request.js +++ b/src/request.js @@ -69,7 +69,7 @@ export default class Request extends Body { const headers = new Headers(init.headers || input.headers || {}); if (inputBody !== null && !headers.has('Content-Type')) { - const contentType = extractContentType(inputBody); + const contentType = extractContentType(inputBody, this); if (contentType) { headers.append('Content-Type', contentType); } @@ -169,7 +169,8 @@ export const getNodeRequestOptions = request => { if (request.body !== null) { const totalBytes = getTotalBytes(request); - if (typeof totalBytes === 'number') { + // Set Content-Length if totalBytes is a number (that is not NaN) + if (typeof totalBytes === 'number' && !Number.isNaN(totalBytes)) { contentLengthValue = String(totalBytes); } } diff --git a/src/utils/form-data.js b/src/utils/form-data.js new file mode 100644 index 000000000..1fd23b0ad --- /dev/null +++ b/src/utils/form-data.js @@ -0,0 +1,82 @@ +import {randomBytes} from 'crypto'; + +import {isBlob} from './is.js'; + +const carriage = '\r\n'; +const dashes = '-'.repeat(2); +const carriageLength = Buffer.byteLength(carriage); + +/** + * @param {string} boundary + */ +const getFooter = boundary => `${dashes}${boundary}${dashes}${carriage.repeat(2)}`; + +/** + * @param {string} boundary + * @param {string} name + * @param {*} field + * + * @return {string} + */ +function getHeader(boundary, name, field) { + let header = ''; + + header += `${dashes}${boundary}${carriage}`; + header += `Content-Disposition: form-data; name="${name}"`; + + if (isBlob(field)) { + header += `; filename="${field.name}"${carriage}`; + header += `Content-Type: ${field.type || 'application/octet-stream'}`; + } + + return `${header}${carriage.repeat(2)}`; +} + +/** + * @return {string} + */ +export const getBoundary = () => randomBytes(8).toString('hex'); + +/** + * @param {FormData} form + * @param {string} boundary + */ +export async function * formDataIterator(form, boundary) { + for (const [name, value] of form) { + yield getHeader(boundary, name, value); + + if (isBlob(value)) { + yield * value.stream(); + } else { + yield value; + } + + yield carriage; + } + + yield getFooter(boundary); +} + +/** + * @param {FormData} form + * @param {string} boundary + */ +export function getFormDataLength(form, boundary) { + let length = 0; + + for (const [name, value] of form) { + length += Buffer.byteLength(getHeader(boundary, name, value)); + + if (isBlob(value)) { + length += value.size; + } else { + length += Buffer.byteLength(String(value)); + } + + length += carriageLength; + } + + length += Buffer.byteLength(getFooter(boundary)); + + return length; +} diff --git a/src/utils/is.js b/src/utils/is.js index ea74a0021..e48165d9f 100644 --- a/src/utils/is.js +++ b/src/utils/is.js @@ -28,7 +28,7 @@ export const isURLSearchParameters = object => { }; /** - * Check if `obj` is a W3C `Blob` object (which `File` inherits from) + * Check if `object` is a W3C `Blob` object (which `File` inherits from) * * @param {*} obj * @return {boolean} @@ -44,6 +44,28 @@ export const isBlob = object => { ); }; +/** + * Check if `obj` is a spec-compliant `FormData` object + * + * @param {*} object + * @return {boolean} + */ +export function isFormData(object) { + return ( + typeof object === 'object' && + typeof object.append === 'function' && + typeof object.set === 'function' && + typeof object.get === 'function' && + typeof object.getAll === 'function' && + typeof object.delete === 'function' && + typeof object.keys === 'function' && + typeof object.values === 'function' && + typeof object.entries === 'function' && + typeof object.constructor === 'function' && + object[NAME] === 'FormData' + ); +} + /** * Check if `obj` is an instance of AbortSignal. * diff --git a/test/form-data.js b/test/form-data.js new file mode 100644 index 000000000..fe08fe4c6 --- /dev/null +++ b/test/form-data.js @@ -0,0 +1,104 @@ +import FormData from 'formdata-node'; +import Blob from 'fetch-blob'; + +import chai from 'chai'; + +import read from './utils/read-stream.js'; + +import {getFormDataLength, getBoundary, formDataIterator} from '../src/utils/form-data.js'; + +const {expect} = chai; + +const carriage = '\r\n'; + +const getFooter = boundary => `--${boundary}--${carriage.repeat(2)}`; + +describe('FormData', () => { + it('should return a length for empty form-data', () => { + const form = new FormData(); + const boundary = getBoundary(); + + expect(getFormDataLength(form, boundary)).to.be.equal(Buffer.byteLength(getFooter(boundary))); + }); + + it('should add a Blob field\'s size to the FormData length', () => { + const form = new FormData(); + const boundary = getBoundary(); + + const string = 'Hello, world!'; + const expected = Buffer.byteLength( + `--${boundary}${carriage}` + + `Content-Disposition: form-data; name="field"${carriage.repeat(2)}` + + string + + `${carriage}${getFooter(boundary)}` + ); + + form.set('field', string); + + expect(getFormDataLength(form, boundary)).to.be.equal(expected); + }); + + it('should return a length for a Blob field', () => { + const form = new FormData(); + const boundary = getBoundary(); + + const blob = new Blob(['Hello, world!'], {type: 'text/plain'}); + + form.set('blob', blob); + + const expected = blob.size + Buffer.byteLength( + `--${boundary}${carriage}` + + 'Content-Disposition: form-data; name="blob"; ' + + `filename="blob"${carriage}Content-Type: text/plain` + + `${carriage.repeat(3)}${getFooter(boundary)}` + ); + + expect(getFormDataLength(form, boundary)).to.be.equal(expected); + }); + + it('should create a body from empty form-data', async () => { + const form = new FormData(); + const boundary = getBoundary(); + + expect(String(await read(formDataIterator(form, boundary)))).to.be.equal(getFooter(boundary)); + }); + + it('should set default content-type', async () => { + const form = new FormData(); + const boundary = getBoundary(); + + form.set('blob', new Blob([])); + + expect(String(await read(formDataIterator(form, boundary)))).to.contain('Content-Type: application/octet-stream'); + }); + + it('should create a body with a FormData field', async () => { + const form = new FormData(); + const boundary = getBoundary(); + const string = 'Hello, World!'; + + form.set('field', string); + + const expected = `--${boundary}${carriage}` + + `Content-Disposition: form-data; name="field"${carriage.repeat(2)}` + + string + + `${carriage}${getFooter(boundary)}`; + + expect(String(await read(formDataIterator(form, boundary)))).to.be.equal(expected); + }); + + it('should create a body with a FormData Blob field', async () => { + const form = new FormData(); + const boundary = getBoundary(); + + const expected = `--${boundary}${carriage}` + + 'Content-Disposition: form-data; name="blob"; ' + + `filename="blob"${carriage}Content-Type: text/plain${carriage.repeat(2)}` + + 'Hello, World!' + + `${carriage}${getFooter(boundary)}`; + + form.set('blob', new Blob(['Hello, World!'], {type: 'text/plain'})); + + expect(String(await read(formDataIterator(form, boundary)))).to.be.equal(expected); + }); +}); diff --git a/test/main.js b/test/main.js index e013c5236..fd159a675 100644 --- a/test/main.js +++ b/test/main.js @@ -1,10 +1,10 @@ // Test tools -/* eslint-disable node/no-unsupported-features/node-builtins */ import zlib from 'zlib'; import crypto from 'crypto'; import http from 'http'; import fs from 'fs'; import stream from 'stream'; +import path from 'path'; import {lookup} from 'dns'; import vm from 'vm'; import chai from 'chai'; @@ -12,6 +12,7 @@ import chaiPromised from 'chai-as-promised'; import chaiIterator from 'chai-iterator'; import chaiString from 'chai-string'; import FormData from 'form-data'; +import FormDataNode from 'formdata-node'; import stringToArrayBuffer from 'string-to-arraybuffer'; import delay from 'delay'; import AbortControllerPolyfill from 'abortcontroller-polyfill/dist/abortcontroller.js'; @@ -58,8 +59,6 @@ after(done => { local.stop(done); }); -const itIf = value => value ? it : it.skip; - function streamToPromise(stream, dataHandler) { return new Promise((resolve, reject) => { stream.on('data', (...args) => { @@ -108,7 +107,8 @@ describe('node-fetch', () => { return expect(fetch(url)).to.eventually.be.rejectedWith(TypeError, /URL scheme "ftp" is not supported/); }); - itIf(process.platform !== 'win32')('should reject with error on network failure', () => { + it('should reject with error on network failure', function () { + this.timeout(5000); const url = 'http://localhost:50000/'; return expect(fetch(url)).to.eventually.be.rejected .and.be.an.instanceOf(FetchError) @@ -125,7 +125,8 @@ describe('node-fetch', () => { return expect(err).to.not.have.property('erroredSysCall'); }); - itIf(process.platform !== 'win32')('system error is extracted from failed requests', () => { + it('system error is extracted from failed requests', function () { + this.timeout(5000); const url = 'http://localhost:50000/'; return expect(fetch(url)).to.eventually.be.rejected .and.be.an.instanceOf(FetchError) @@ -1285,7 +1286,7 @@ describe('node-fetch', () => { }); }); - itIf(process.platform !== 'win32')('should allow POST request with form-data using stream as body', () => { + it('should allow POST request with form-data using stream as body', () => { const form = new FormData(); form.append('my_field', fs.createReadStream('test/utils/dummy.txt')); @@ -1329,6 +1330,30 @@ describe('node-fetch', () => { }); }); + it('should support spec-compliant form-data as POST body', () => { + const form = new FormDataNode(); + + const filename = path.join('test', 'utils', 'dummy.txt'); + + form.set('field', 'some text'); + form.set('file', fs.createReadStream(filename), { + size: fs.statSync(filename).size + }); + + const url = `${base}multipart`; + const options = { + method: 'POST', + body: form + }; + + return fetch(url, options).then(res => res.json()).then(res => { + expect(res.method).to.equal('POST'); + expect(res.headers['content-type']).to.startWith('multipart/form-data'); + expect(res.body).to.contain('field='); + expect(res.body).to.contain('file='); + }); + }); + it('should allow POST request with object body', () => { const url = `${base}inspect`; // Note that fetch simply calls tostring on an object diff --git a/test/request.js b/test/request.js index 502b86a9b..5a7acc0f9 100644 --- a/test/request.js +++ b/test/request.js @@ -1,4 +1,3 @@ -/* eslint-disable node/no-unsupported-features/node-builtins */ import stream from 'stream'; import http from 'http'; diff --git a/test/response.js b/test/response.js index 7126eb95c..7ccef7102 100644 --- a/test/response.js +++ b/test/response.js @@ -1,4 +1,3 @@ -/* eslint-disable node/no-unsupported-features/node-builtins */ import * as stream from 'stream'; import chai from 'chai'; diff --git a/test/utils/read-stream.js b/test/utils/read-stream.js new file mode 100644 index 000000000..90dcf6e59 --- /dev/null +++ b/test/utils/read-stream.js @@ -0,0 +1,9 @@ +export default async function readStream(stream) { + const chunks = []; + + for await (const chunk of stream) { + chunks.push(chunk instanceof Buffer ? chunk : Buffer.from(chunk)); + } + + return Buffer.concat(chunks); +} diff --git a/test/utils/server.js b/test/utils/server.js index fbcea48d6..cc9a9ab24 100644 --- a/test/utils/server.js +++ b/test/utils/server.js @@ -1,7 +1,6 @@ import http from 'http'; import zlib from 'zlib'; -import parted from 'parted'; -const {multipart: Multipart} = parted; +import Busboy from 'busboy'; export default class TestServer { constructor() { @@ -364,12 +363,19 @@ export default class TestServer { if (p === '/multipart') { res.statusCode = 200; res.setHeader('Content-Type', 'application/json'); - const parser = new Multipart(request.headers['content-type']); + const busboy = new Busboy({headers: request.headers}); let body = ''; - parser.on('part', (field, part) => { - body += field + '=' + part; + busboy.on('file', async (fieldName, file, fileName) => { + body += `${fieldName}=${fileName}`; + // consume file data + // eslint-disable-next-line no-empty, no-unused-vars + for await (const c of file) { } }); - parser.on('end', () => { + + busboy.on('field', (fieldName, value) => { + body += `${fieldName}=${value}`; + }); + busboy.on('finish', () => { res.end(JSON.stringify({ method: request.method, url: request.url, @@ -377,7 +383,7 @@ export default class TestServer { body })); }); - request.pipe(parser); + request.pipe(busboy); } if (p === '/m%C3%B6bius') { @@ -387,4 +393,3 @@ export default class TestServer { } } } -