v2.6.0 (#638)

* Update readme and changelog for `options.agent`
- Fix content-length issue introduced in v2.5.0
* More test coverage for `extractContentType`
* Slightly improve test performance
* `Response.url` should not return null
* Document `Headers.raw()` usage better
* 2.6.0

CHANGELOG.md

@@ -5,6 +5,12 @@ Changelog
 
 # 2.x release
 
+## v2.6.0
+
+- Enhance: `options.agent`, it now accepts a function that returns custom http(s).Agent instance based on current URL, see readme for more information.
+- Fix: incorrect `Content-Length` was returned for stream body in 2.5.0 release; note that `node-fetch` doesn't calculate content length for stream body.
+- Fix: `Response.url` should return empty string instead of `null` by default.
+
 ## v2.5.0
 
 - Enhance: `Response` object now includes `redirected` property.

README.md

@@ -29,6 +29,7 @@ A light-weight module that brings `window.fetch` to Node.js
     - [Streams](#streams)
     - [Buffer](#buffer)
     - [Accessing Headers and other Meta data](#accessing-headers-and-other-meta-data)
+    - [Extract Set-Cookie Header](#extract-set-cookie-header)
     - [Post data using a file stream](#post-data-using-a-file-stream)
     - [Post with form-data (detect multipart)](#post-with-form-data-detect-multipart)
     - [Request cancellation with AbortSignal](#request-cancellation-with-abortsignal)
@@ -208,6 +209,17 @@ fetch('https://github.com/')
     });
 ```
 
+#### Extract Set-Cookie Header
+
+Unlike browsers, you can access raw `Set-Cookie` headers manually using `Headers.raw()`, this is a `node-fetch` only API.
+
+```js
+fetch(url).then(res => {
+    // returns an array of values, instead of a string of comma-separated values
+    console.log(res.headers.raw()['set-cookie']);
+});
+```
+
 #### Post data using a file stream
 
 ```js
@@ -317,7 +329,7 @@ The default values are shown after each option key.
     timeout: 0,         // req/res timeout in ms, it resets on redirect. 0 to disable (OS limit applies). Signal is recommended instead.
     compress: true,     // support gzip/deflate content encoding. false to disable
     size: 0,            // maximum response body size in bytes. 0 to disable
-    agent: null         // http(s).Agent instance (or function providing one), allows custom proxy, certificate, dns lookup etc.
+    agent: null         // http(s).Agent instance or function that returns an instance (see below)
 }
 ```
 
@@ -334,6 +346,39 @@ Header              | Value
 `Transfer-Encoding` | `chunked` _(when `req.body` is a stream)_
 `User-Agent`        | `node-fetch/1.0 (+https://github.com/bitinn/node-fetch)`
 
+Note: when `body` is a `Stream`, `Content-Length` is not set automatically.
+
+##### Custom Agent
+
+The `agent` option allows you to specify networking related options that's out of the scope of Fetch. Including and not limit to:
+
+- Support self-signed certificate
+- Use only IPv4 or IPv6
+- Custom DNS Lookup
+
+See [`http.Agent`](https://nodejs.org/api/http.html#http_new_agent_options) for more information.
+
+In addition, `agent` option accepts a function that returns http(s).Agent instance given current [URL](https://nodejs.org/api/url.html), this is useful during a redirection chain across HTTP and HTTPS protocol.
+
+```js
+const httpAgent = new http.Agent({
+    keepAlive: true
+});
+const httpsAgent = new https.Agent({
+    keepAlive: true
+});
+
+const options = {
+    agent: function (_parsedURL) {
+        if (_parsedURL.protocol == 'http:') {
+            return httpAgent;
+        } else {
+            return httpsAgent;
+        }
+    }
+}
+```
+
 <a id="class-request"></a>
 ### Class: Request
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "node-fetch",
-  "version": "2.5.0",
+  "version": "2.6.0",
   "description": "A light-weight module that brings window.fetch to node.js",
   "main": "lib/index",
   "browser": "./browser.js",

src/body.js

@@ -415,11 +415,9 @@ export function clone(instance) {
  *
  * This function assumes that instance.body is present.
  *
- * @param   Mixed  instance  Response or Request instance
+ * @param   Mixed  instance  Any options.body input
  */
 export function extractContentType(body) {
-	// istanbul ignore if: Currently, because of a guard in Request, body
-	// can never be null. Included here for completeness.
 	if (body === null) {
 		// body is null
 		return null;
@@ -466,7 +464,6 @@ export function extractContentType(body) {
 export function getTotalBytes(instance) {
 	const {body} = instance;
 
-	// istanbul ignore if: included for completion
 	if (body === null) {
 		// body is null
 		return 0;
@@ -484,7 +481,7 @@ export function getTotalBytes(instance) {
 		return null;
 	} else {
 		// body is stream
-		return instance.size || null;
+		return null;
 	}
 }
 

src/response.js

@@ -46,7 +46,7 @@ export default class Response {
 	}
 
 	get url() {
-		return this[INTERNALS].url;
+		return this[INTERNALS].url || '';
 	}
 
 	get status() {

test/server.js

@@ -157,10 +157,10 @@ export default class TestServer {
 			res.setHeader('Content-Type', 'text/plain');
 			setTimeout(function() {
 				res.write('test');
-			}, 50);
+			}, 10);
 			setTimeout(function() {
 				res.end('test');
-			}, 100);
+			}, 20);
 		}
 
 		if (p === '/size/long') {
@@ -280,7 +280,7 @@ export default class TestServer {
 			res.setHeader('Location', '/redirect/slow');
 			setTimeout(function() {
 				res.end();
-			}, 100);
+			}, 10);
 		}
 
 		if (p === '/redirect/slow-stream') {

test/test.js

@@ -48,7 +48,7 @@ import FetchErrorOrig from '../src/fetch-error.js';
 import HeadersOrig, { createHeadersLenient } from '../src/headers.js';
 import RequestOrig from '../src/request.js';
 import ResponseOrig from '../src/response.js';
-import Body from '../src/body.js';
+import Body, { getTotalBytes, extractContentType } from '../src/body.js';
 import Blob from '../src/blob.js';
 import zlib from "zlib";
 
@@ -738,7 +738,7 @@ describe('node-fetch', () => {
 				// Wait a few ms to see if a uncaught error occurs
 				setTimeout(() => {
 					done();
-				}, 50);
+				}, 20);
 			});
 	});
 
@@ -748,7 +748,7 @@ describe('node-fetch', () => {
 			return new Promise((resolve) => {
 				setTimeout(() => {
 					resolve(value)
-				}, 100);
+				}, 20);
 			});
 		}
 
@@ -789,21 +789,19 @@ describe('node-fetch', () => {
 	});
 
 	it('should allow custom timeout', function() {
-		this.timeout(500);
 		const url = `${base}timeout`;
 		const opts = {
-			timeout: 100
+			timeout: 20
 		};
 		return expect(fetch(url, opts)).to.eventually.be.rejected
 			.and.be.an.instanceOf(FetchError)
 			.and.have.property('type', 'request-timeout');
 	});
 
 	it('should allow custom timeout on response body', function() {
-		this.timeout(500);
 		const url = `${base}slow`;
 		const opts = {
-			timeout: 100
+			timeout: 20
 		};
 		return fetch(url, opts).then(res => {
 			expect(res.ok).to.be.true;
@@ -814,10 +812,9 @@ describe('node-fetch', () => {
 	});
 
 	it('should allow custom timeout on redirected requests', function() {
-		this.timeout(2000);
 		const url = `${base}redirect/slow-chain`;
 		const opts = {
-			timeout: 200
+			timeout: 20
 		};
 		return expect(fetch(url, opts)).to.eventually.be.rejected
 			.and.be.an.instanceOf(FetchError)
@@ -908,7 +905,7 @@ describe('node-fetch', () => {
 				'${base}timeout',
 				{ signal: controller.signal, timeout: 10000 }
 			);
-			setTimeout(function () { controller.abort(); }, 100);
+			setTimeout(function () { controller.abort(); }, 20);
 		`
 		spawn('node', ['-e', script])
 			.on('exit', () => {
@@ -940,7 +937,7 @@ describe('node-fetch', () => {
 		});
 		setTimeout(() => {
 			abortController.abort();
-		}, 50);
+		}, 20);
 		return expect(fetch(request)).to.be.eventually.rejected
 			.and.be.an.instanceOf(Error)
 			.and.have.property('name', 'AbortError');
@@ -1914,8 +1911,8 @@ describe('node-fetch', () => {
 		expect(err.type).to.equal('test-error');
 		expect(err.code).to.equal('ESOMEERROR');
 		expect(err.errno).to.equal('ESOMEERROR');
-		expect(err.stack).to.include('funcName')
-			.and.to.startWith(`${err.name}: ${err.message}`);
+		// reading the stack is quite slow (~30-50ms)
+		expect(err.stack).to.include('funcName').and.to.startWith(`${err.name}: ${err.message}`);
 	});
 
 	it('should support https request', function() {
@@ -1982,7 +1979,7 @@ describe('node-fetch', () => {
 	it('should allow a function supplying the agent', function() {
 		const url = `${base}inspect`;
 
-		const agent = http.Agent({
+		const agent = new http.Agent({
 			keepAlive: true
 		});
 
@@ -2002,6 +1999,67 @@ describe('node-fetch', () => {
 			expect(res.headers['connection']).to.equal('keep-alive');
 		});
 	});
+
+	it('should calculate content length and extract content type for each body type', function () {
+		const url = `${base}hello`;
+		const bodyContent = 'a=1';
+
+		let streamBody = resumer().queue(bodyContent).end();
+		streamBody = streamBody.pipe(new stream.PassThrough());
+		const streamRequest = new Request(url, {
+			method: 'POST',
+			body: streamBody,
+			size: 1024
+		});
+
+		let blobBody = new Blob([bodyContent], { type: 'text/plain' });
+		const blobRequest = new Request(url, {
+			method: 'POST',
+			body: blobBody,
+			size: 1024
+		});
+
+		let formBody = new FormData();
+		formBody.append('a', '1');
+		const formRequest = new Request(url, {
+			method: 'POST',
+			body: formBody,
+			size: 1024
+		});
+
+		let bufferBody = Buffer.from(bodyContent);
+		const bufferRequest = new Request(url, {
+			method: 'POST',
+			body: bufferBody,
+			size: 1024
+		});
+
+		const stringRequest = new Request(url, {
+			method: 'POST',
+			body: bodyContent,
+			size: 1024
+		});
+
+		const nullRequest = new Request(url, {
+			method: 'GET',
+			body: null,
+			size: 1024
+		});
+
+		expect(getTotalBytes(streamRequest)).to.be.null;
+		expect(getTotalBytes(blobRequest)).to.equal(blobBody.size);
+		expect(getTotalBytes(formRequest)).to.not.be.null;
+		expect(getTotalBytes(bufferRequest)).to.equal(bufferBody.length);
+		expect(getTotalBytes(stringRequest)).to.equal(bodyContent.length);
+		expect(getTotalBytes(nullRequest)).to.equal(0);
+
+		expect(extractContentType(streamBody)).to.be.null;
+		expect(extractContentType(blobBody)).to.equal('text/plain');
+		expect(extractContentType(formBody)).to.startWith('multipart/form-data');
+		expect(extractContentType(bufferBody)).to.be.null;
+		expect(extractContentType(bodyContent)).to.equal('text/plain;charset=UTF-8');
+		expect(extractContentType(null)).to.be.null;
+	});
 });
 
 describe('Headers', function () {
@@ -2387,6 +2445,11 @@ describe('Response', function () {
 		const res = new Response(null);
 		expect(res.status).to.equal(200);
 	});
+
+	it('should default to empty string as url', function() {
+		const res = new Response();
+		expect(res.url).to.equal('');
+	});
 });
 
 describe('Request', function () {