General improvements to HTTPS API (sindresorhus#1255)

Co-authored-by: Sindre Sorhus <sindresorhus@gmail.com>
Co-authored-by: Szymon Marczak <36894700+szmarczak@users.noreply.github.com>

benchmark/index.ts

@@ -24,7 +24,9 @@ const gotOptions = {
 	agent: {
 		https: httpsAgent
 	},
-	rejectUnauthorized: false,
+	https: {
+		rejectUnauthorized: false
+	},
 	retry: 0
 };
 
@@ -43,7 +45,9 @@ const fetchOptions = {
 const axiosOptions = {
 	url: urlString,
 	httpsAgent,
-	rejectUnauthorized: false
+	https: {
+		rejectUnauthorized: false
+	}
 };
 
 const axiosStreamOptions: typeof axiosOptions & {responseType: 'stream'} = {
@@ -52,7 +56,9 @@ const axiosStreamOptions: typeof axiosOptions & {responseType: 'stream'} = {
 };
 
 const httpsOptions = {
-	rejectUnauthorized: false,
+	https: {
+		rejectUnauthorized: false
+	},
 	agent: httpsAgent
 };
 

package.json

@@ -80,7 +80,7 @@
 		"node-fetch": "^2.6.0",
 		"np": "^6.0.0",
 		"nyc": "^15.0.1",
-		"p-event": "^4.0.0",
+		"p-event": "^4.1.0",
 		"sinon": "^9.0.2",
 		"slow-stream": "0.0.4",
 		"tempy": "^0.5.0",

readme.md

@@ -40,6 +40,7 @@ For browser usage, we recommend [Ky](https://github.com/sindresorhus/ky) by the
 - [Errors with metadata](#errors)
 - [JSON mode](#json-mode)
 - [WHATWG URL support](#url)
+- [HTTPS API](#https)
 - [Hooks](#hooks)
 - [Instances with custom defaults](#instances)
 - [Types](#types)
@@ -826,7 +827,98 @@ Type: `string`
 
 The IP address used to send the request from.
 
-##### rejectUnauthorized
+### Advanced HTTPS API
+
+Note: If the request is not HTTPS, these options will be ignored.
+
+##### https.certificateAuthority
+
+Type: `string | Buffer | Array<string | Buffer>`
+
+Override the default Certificate Authorities ([from Mozilla](https://ccadb-public.secure.force.com/mozilla/IncludedCACertificateReport))
+
+```js
+// Single Certificate Authority
+got('https://example.com', {
+	https: {
+		certificateAuthority: fs.readFileSync('./my_ca.pem')
+	}
+});
+```
+
+##### https.key
+
+Type: `string | Buffer | Array<string | Buffer> | object[]`
+
+Private keys in [PEM](https://en.wikipedia.org/wiki/Privacy-Enhanced_Mail) format.\
+[PEM](https://en.wikipedia.org/wiki/Privacy-Enhanced_Mail) allows the option of private keys being encrypted. Encrypted keys will be decrypted with `options.https.passphrase`.\
+Multiple keys with different passphrases can be provided as an array of `{pem: <string | Buffer>, passphrase: <string>}`
+
+##### https.certificate
+
+Type: `string | Buffer | (string | Buffer)[]`
+
+[Certificate chains](https://en.wikipedia.org/wiki/X.509#Certificate_chains_and_cross-certification) in [PEM](https://en.wikipedia.org/wiki/Privacy-Enhanced_Mail) format.\
+One cert chain should be provided per private key (`options.https.key`).\
+When providing multiple cert chains, they do not have to be in the same order as their private keys in `options.https.key`.\
+If the intermediate certificates are not provided, the peer will not be able to validate the certificate, and the handshake will fail.
+
+##### https.passphrase
+
+Type: `string`
+
+The passphrase to decrypt the `options.https.key` (if different keys have different passphrases refer to `options.https.key` documentation).
+
+##### Examples for `https.key`, `https.certificate` and `https.passphrase`
+
+```js
+// Single key with certificate
+got('https://example.com', {
+	https: {
+		key: fs.readFileSync('./client_key.pem'),
+		certificate: fs.readFileSync('./client_cert.pem')
+	}
+});
+
+// Multiple keys with certificates (out of order)
+got('https://example.com', {
+	https: {
+		key: [
+			fs.readFileSync('./client_key1.pem'),
+			fs.readFileSync('./client_key2.pem')
+		],
+		certificate: [
+			fs.readFileSync('./client_cert2.pem'),
+			fs.readFileSync('./client_cert1.pem')
+		]
+	}
+});
+
+// Single key with passphrase
+got('https://example.com', {
+	https: {
+		key: fs.readFileSync('./client_key.pem'),
+		certificate: fs.readFileSync('./client_cert.pem'),
+		passphrase: 'client_key_passphrase'
+	}
+});
+
+// Multiple keys with different passphrases
+got('https://example.com', {
+	https: {
+		key: [
+			{pem: fs.readFileSync('./client_key1.pem'), passphrase: 'passphrase1'},
+			{pem: fs.readFileSync('./client_key2.pem'), passphrase: 'passphrase2'},
+		],
+		certificate: [
+			fs.readFileSync('./client_cert1.pem'),
+			fs.readFileSync('./client_cert2.pem')
+		]
+	}
+});
+```
+
+##### https.rejectUnauthorized
 
 Type: `boolean`\
 Default: `true`
@@ -841,14 +933,53 @@ const got = require('got');
 
 (async () => {
 	// Correct:
-	await got('https://example.com', {rejectUnauthorized: true});
+	await got('https://example.com', {
+		https: {
+			rejectUnauthorized: true
+		}
+	});
 
 	// You can disable it when developing an HTTPS app:
-	await got('https://localhost', {rejectUnauthorized: false});
+	await got('https://localhost', {
+		https: {
+			rejectUnauthorized: false
+		}
+	});
 
 	// Never do this:
-	await got('https://example.com', {rejectUnauthorized: false});
-})();
+	await got('https://example.com', {
+		https: {
+			rejectUnauthorized: false
+		}
+	});
+```
+
+##### https.checkServerIdentity
+
+Type: `Function`\
+Signature: `(hostname: string, certificate: DetailedPeerCertificate) => Error | undefined`\
+Default: `tls.checkServerIdentity` (from the `tls` module)
+
+This function enable a custom check of the certificate.\
+Note: In order to have the function called the certificate must not be `expired`, `self-signed` or with an `untrusted-root`.\
+The function parameters are:
+- `hostname`: The server hostname (used when connecting)
+- `certificate`: The server certificate
+
+The function must return `undefined` if the check succeeded or an `Error` if it failed.
+
+```js
+await got('https://example.com', {
+	https: {
+		checkServerIdentity: (hostname, certificate) => {
+			if (hostname === 'example.com') {
+				return; // Certificate OK
+			}
+
+			return new Error('Invalid Hostname'); // Certificate NOT OK
+		}
+	}
+});
 ```
 
 #### Response

source/core/index.ts

@@ -3,7 +3,7 @@ import {Duplex, Writable, Readable} from 'stream';
 import {ReadStream} from 'fs';
 import {URL, URLSearchParams} from 'url';
 import {Socket} from 'net';
-import {SecureContextOptions} from 'tls';
+import {SecureContextOptions, DetailedPeerCertificate} from 'tls';
 import http = require('http');
 import {ClientRequest, RequestOptions, IncomingMessage, ServerResponse, request as httpRequest} from 'http';
 import https = require('https');
@@ -24,6 +24,7 @@ import timedOut, {Delays, TimeoutError as TimedOutTimeoutError} from './utils/ti
 import urlToOptions from './utils/url-to-options';
 import optionsToUrl, {URLOptions} from './utils/options-to-url';
 import WeakableMap from './utils/weakable-map';
+import deprecationWarning from '../utils/deprecation-warning';
 
 type HttpRequestFunction = typeof httpRequest;
 type Error = NodeJS.ErrnoException;
@@ -116,7 +117,13 @@ type CacheableRequestFn = (
 	cb?: (response: ServerResponse | ResponseLike) => void
 ) => CacheableRequest.Emitter;
 
-export interface Options extends URLOptions, SecureContextOptions {
+type CheckServerIdentityFn = (hostname: string, certificate: DetailedPeerCertificate) => Error | void;
+
+interface RealRequestOptions extends https.RequestOptions {
+	checkServerIdentity: CheckServerIdentityFn;
+}
+
+export interface Options extends URLOptions {
 	request?: RequestFunction;
 	agent?: Agents | false;
 	decompress?: boolean;
@@ -141,15 +148,33 @@ export interface Options extends URLOptions, SecureContextOptions {
 	http2?: boolean;
 	allowGetBody?: boolean;
 	lookup?: CacheableLookup['lookup'];
-	rejectUnauthorized?: boolean;
 	headers?: Headers;
 	methodRewriting?: boolean;
 
-	// From http.RequestOptions
+	// From `http.RequestOptions`
 	localAddress?: string;
 	socketPath?: string;
 	method?: Method;
 	createConnection?: (options: http.RequestOptions, oncreate: (error: Error, socket: Socket) => void) => Socket;
+
+	// TODO: remove when Got 12 gets released
+	rejectUnauthorized?: boolean; // Here for backwards compatibility
+
+	https?: HTTPSOptions;
+}
+
+export interface HTTPSOptions {
+	// From `http.RequestOptions` and `tls.CommonConnectionOptions`
+	rejectUnauthorized?: https.RequestOptions['rejectUnauthorized'];
+
+	// From `tls.ConnectionOptions`
+	checkServerIdentity?: CheckServerIdentityFn;
+
+	// From `tls.SecureContextOptions`
+	certificateAuthority?: SecureContextOptions['ca'];
+	key?: SecureContextOptions['key'];
+	certificate?: SecureContextOptions['cert'];
+	passphrase?: SecureContextOptions['passphrase'];
 }
 
 export interface NormalizedOptions extends Options {
@@ -197,7 +222,7 @@ export interface Defaults {
 	throwHttpErrors: boolean;
 	http2: boolean;
 	allowGetBody: boolean;
-	rejectUnauthorized: boolean;
+	https?: HTTPSOptions;
 	methodRewriting: boolean;
 
 	// Optional
@@ -623,8 +648,17 @@ export default class Request extends Duplex implements RequestEvents<Request> {
 		assert.any([is.boolean, is.undefined], options.throwHttpErrors);
 		assert.any([is.boolean, is.undefined], options.http2);
 		assert.any([is.boolean, is.undefined], options.allowGetBody);
-		assert.any([is.boolean, is.undefined], options.rejectUnauthorized);
 		assert.any([is.string, is.undefined], options.localAddress);
+		assert.any([is.object, is.undefined], options.https);
+		assert.any([is.boolean, is.undefined], options.rejectUnauthorized);
+		if (options.https) {
+			assert.any([is.boolean, is.undefined], options.https.rejectUnauthorized);
+			assert.any([is.function_, is.undefined], options.https.checkServerIdentity);
+			assert.any([is.string, is.object, is.array, is.undefined], options.https.certificateAuthority);
+			assert.any([is.string, is.object, is.array, is.undefined], options.https.key);
+			assert.any([is.string, is.object, is.array, is.undefined], options.https.certificate);
+			assert.any([is.string, is.undefined], options.https.passphrase);
+		}
 
 		// `options.method`
 		if (is.string(options.method)) {
@@ -833,6 +867,31 @@ export default class Request extends Duplex implements RequestEvents<Request> {
 			}
 		}
 
+		// HTTPS options
+		if ('rejectUnauthorized' in options) {
+			deprecationWarning('"options.rejectUnauthorized" is now deprecated, please use "options.https.rejectUnauthorized"');
+		}
+
+		if ('checkServerIdentity' in options) {
+			deprecationWarning('"options.checkServerIdentity" was never documented, please use "options.https.checkServerIdentity"');
+		}
+
+		if ('ca' in options) {
+			deprecationWarning('"options.ca" was never documented, please use "options.https.certificateAuthority"');
+		}
+
+		if ('key' in options) {
+			deprecationWarning('"options.key" was never documented, please use "options.https.key"');
+		}
+
+		if ('cert' in options) {
+			deprecationWarning('"options.cert" was never documented, please use "options.https.certificate"');
+		}
+
+		if ('passphrase' in options) {
+			deprecationWarning('"options.passphrase" was never documented, please use "options.https.passphrase"');
+		}
+
 		// Other options
 		if ('followRedirects' in options) {
 			throw new TypeError('The `followRedirects` option does not exist. Use `followRedirect` instead.');
@@ -1333,11 +1392,40 @@ export default class Request extends Duplex implements RequestEvents<Request> {
 		delete options.request;
 		delete options.timeout;
 
+		const requestOptions = options as unknown as RealRequestOptions;
+
+		// HTTPS options remapping
+		if (options.https) {
+			if ('rejectUnauthorized' in options.https) {
+				requestOptions.rejectUnauthorized = options.https.rejectUnauthorized;
+			}
+
+			if (options.https.checkServerIdentity) {
+				requestOptions.checkServerIdentity = options.https.checkServerIdentity;
+			}
+
+			if (options.https.certificateAuthority) {
+				requestOptions.ca = options.https.certificateAuthority;
+			}
+
+			if (options.https.certificate) {
+				requestOptions.cert = options.https.certificate;
+			}
+
+			if (options.https.key) {
+				requestOptions.key = options.https.key;
+			}
+
+			if (options.https.passphrase) {
+				requestOptions.passphrase = options.https.passphrase;
+			}
+		}
+
 		try {
-			let requestOrResponse = await fn(url, options as unknown as RequestOptions);
+			let requestOrResponse = await fn(url, requestOptions);
 
 			if (is.undefined(requestOrResponse)) {
-				requestOrResponse = fallbackFn(url, options as unknown as RequestOptions);
+				requestOrResponse = fallbackFn(url, requestOptions);
 			}
 
 			// Restore options

source/index.ts

@@ -68,7 +68,7 @@ const defaults: InstanceDefaults = {
 		// TODO: Set this to `true` when Got 12 gets released
 		http2: false,
 		allowGetBody: false,
-		rejectUnauthorized: true,
+		https: undefined,
 		pagination: {
 			transform: (response: Response) => {
 				if (response.request.options.responseType === 'json') {