From c0d0764396f1660107aaa919f15278a4e1004ae4 Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Thu, 16 Jul 2020 18:11:40 -0400 Subject: [PATCH 01/23] add support for https.pfx option --- source/core/index.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/source/core/index.ts b/source/core/index.ts index f49889235..67aa20332 100644 --- a/source/core/index.ts +++ b/source/core/index.ts @@ -187,6 +187,7 @@ export interface HTTPSOptions { key?: SecureContextOptions['key']; certificate?: SecureContextOptions['cert']; passphrase?: SecureContextOptions['passphrase']; + pfx?: SecureContextOptions['pfx']; } interface NormalizedPlainOptions extends PlainOptions { @@ -975,6 +976,10 @@ export default class Request extends Duplex implements RequestEvents { deprecationWarning('"options.passphrase" was never documented, please use "options.https.passphrase"'); } + if ('pfx' in options) { + deprecationWarning('"options.pfx" was never documented, please use "options.https.pfx"'); + } + // Other options if ('followRedirects' in options) { throw new TypeError('The `followRedirects` option does not exist. Use `followRedirect` instead.'); @@ -1527,6 +1532,10 @@ export default class Request extends Duplex implements RequestEvents { if (options.https.passphrase) { requestOptions.passphrase = options.https.passphrase; } + + if (options.https.pfx) { + requestOptions.pfx = options.https.pfx; + } } try { From 8c2a9a36a5491facbdcecdd3c17d38729bce1b02 Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Fri, 17 Jul 2020 12:44:46 -0400 Subject: [PATCH 02/23] add more handling for pfx option --- source/core/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/source/core/index.ts b/source/core/index.ts index 67aa20332..707c059c9 100644 --- a/source/core/index.ts +++ b/source/core/index.ts @@ -685,6 +685,7 @@ export default class Request extends Duplex implements RequestEvents { assert.any([is.string, is.object, is.array, is.undefined], options.https.key); assert.any([is.string, is.object, is.array, is.undefined], options.https.certificate); assert.any([is.string, is.undefined], options.https.passphrase); + assert.any([is.string, is.undefined], options.https.pfx); } // `options.method` From ac2fc1d74497737ee7f4f909839e19c7a93672de Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Fri, 17 Jul 2020 12:44:52 -0400 Subject: [PATCH 03/23] update documentation --- readme.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/readme.md b/readme.md index 9f40ed17a..09dc25f16 100644 --- a/readme.md +++ b/readme.md @@ -985,6 +985,18 @@ Type: `string` The passphrase to decrypt the `options.https.key` (if different keys have different passphrases refer to `options.https.key` documentation). +##### https.pfx + +Type: `string | Buffer | (string | Buffer | object)[]` + +PFX or PKCS12 encoded private key and certificate chain. `pfx` is an alternative to providing `options.https.key` and `options.https.certificate` individually. PFX is usually encrypted, if it is, `options.https.passphrase` will be used to decrypt it. + +Multiple PFX can be be provided as an array of unencrypted buffers or an array of objects like: + +`{buf: [, passphrase: ]}` + +If the provided buffers are encrypted, `object.passphrase` can be used to decrypt them. If `object.passphrase` is not provided, `options.https.passphrase` will be used for decryption. + ##### Examples for `https.key`, `https.certificate` and `https.passphrase` ```js From 5ff96b03554f9c3a26eab0b7abc4fdffde6a707d Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Fri, 17 Jul 2020 12:57:22 -0400 Subject: [PATCH 04/23] add test for pfx deprecation --- test/https.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/https.ts b/test/https.ts index 9ead3ff92..442a59328 100644 --- a/test/https.ts +++ b/test/https.ts @@ -159,6 +159,25 @@ test.serial('non-deprecated `rejectUnauthorized` option', withServer, async (t, t.pass(); }); +test.serial('non-deprecated `pfx` option', withServer, async (t, server, got) => { + server.get('/', (_request, response) => { + response.end('ok'); + }); + + (async () => { + const warning = await pEvent(process, 'warning'); + t.not(warning.name, 'DeprecationWarning'); + })(); + + await got.secure({ + https: { + pfx: 'fake-pfx' + } + }); + + t.pass(); +}); + test.serial('no double deprecated warning', withServer, async (t, server, got) => { server.get('/', (_request, response) => { response.end('ok'); From dba39ee91f8acafe6c0bb3b5a17e59cce43d7955 Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Fri, 17 Jul 2020 13:09:32 -0400 Subject: [PATCH 05/23] update assertions for options.https.pfx --- source/core/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/core/index.ts b/source/core/index.ts index 707c059c9..47b8b30f8 100644 --- a/source/core/index.ts +++ b/source/core/index.ts @@ -685,7 +685,7 @@ export default class Request extends Duplex implements RequestEvents { assert.any([is.string, is.object, is.array, is.undefined], options.https.key); assert.any([is.string, is.object, is.array, is.undefined], options.https.certificate); assert.any([is.string, is.undefined], options.https.passphrase); - assert.any([is.string, is.undefined], options.https.pfx); + assert.any([is.string, is.array, is.undefined], options.https.pfx); } // `options.method` From 05221183318d30479fda07ee297b886642d73130 Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Fri, 17 Jul 2020 17:18:31 -0400 Subject: [PATCH 06/23] remove test --- test/https.ts | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/test/https.ts b/test/https.ts index 442a59328..9ead3ff92 100644 --- a/test/https.ts +++ b/test/https.ts @@ -159,25 +159,6 @@ test.serial('non-deprecated `rejectUnauthorized` option', withServer, async (t, t.pass(); }); -test.serial('non-deprecated `pfx` option', withServer, async (t, server, got) => { - server.get('/', (_request, response) => { - response.end('ok'); - }); - - (async () => { - const warning = await pEvent(process, 'warning'); - t.not(warning.name, 'DeprecationWarning'); - })(); - - await got.secure({ - https: { - pfx: 'fake-pfx' - } - }); - - t.pass(); -}); - test.serial('no double deprecated warning', withServer, async (t, server, got) => { server.get('/', (_request, response) => { response.end('ok'); From a3e171c4d43c8bbb236ef3a04feb25ba6fdf6c49 Mon Sep 17 00:00:00 2001 From: Tomas Della Vedova Date: Sat, 18 Jul 2020 15:04:02 +0200 Subject: [PATCH 07/23] Add hpagent to proxy section (#1363) --- readme.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/readme.md b/readme.md index 9f40ed17a..846f437bc 100644 --- a/readme.md +++ b/readme.md @@ -1750,6 +1750,26 @@ got('https://sindresorhus.com', { }); ``` +Otherwise, you can use the [`hpagent`](https://github.com/delvedor/hpagent) package, which keeps the internal sockets alive to be reused. + +```js +const got = require('got'); +const {HttpsProxyAgent} = require('hpagent'); + +got('https://sindresorhus.com', { + agent: { + https: new HttpsProxyAgent({ + keepAlive: true, + keepAliveMsecs: 1000, + maxSockets: 256, + maxFreeSockets: 256, + scheduling: 'lifo', + proxy: 'https://localhost:8080' + }) + } +}); +``` + Alternatively, use [`global-agent`](https://github.com/gajus/global-agent) to configure a global proxy for all HTTP/HTTPS traffic in your program. Read the [`http2-wrapper`](https://github.com/szmarczak/http2-wrapper/#proxy-support) docs to learn about proxying for HTTP/2. From 7ac949a9cfc5bd8040866723ccde5ba14e8a7b72 Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Tue, 21 Jul 2020 09:43:15 -0400 Subject: [PATCH 08/23] Update readme.md Co-authored-by: Sindre Sorhus --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 09dc25f16..332224d17 100644 --- a/readme.md +++ b/readme.md @@ -991,7 +991,7 @@ Type: `string | Buffer | (string | Buffer | object)[]` PFX or PKCS12 encoded private key and certificate chain. `pfx` is an alternative to providing `options.https.key` and `options.https.certificate` individually. PFX is usually encrypted, if it is, `options.https.passphrase` will be used to decrypt it. -Multiple PFX can be be provided as an array of unencrypted buffers or an array of objects like: +Multiple PFX's can be be provided as an array of unencrypted buffers or an array of objects like: `{buf: [, passphrase: ]}` From e7b7832db19e2c80348f84998a1c0cd2cc598d1d Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Tue, 21 Jul 2020 09:43:25 -0400 Subject: [PATCH 09/23] Update readme.md Co-authored-by: Sindre Sorhus --- readme.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 332224d17..1e6c8502d 100644 --- a/readme.md +++ b/readme.md @@ -993,7 +993,12 @@ PFX or PKCS12 encoded private key and certificate chain. `pfx` is an alternative Multiple PFX's can be be provided as an array of unencrypted buffers or an array of objects like: -`{buf: [, passphrase: ]}` +` +{ + buffer: string | Buffer, + passphrase?: string +} +` If the provided buffers are encrypted, `object.passphrase` can be used to decrypt them. If `object.passphrase` is not provided, `options.https.passphrase` will be used for decryption. From 5f29b6c33f0b8cad82aa5e35d4bd3a27ca059cc0 Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Tue, 21 Jul 2020 09:43:36 -0400 Subject: [PATCH 10/23] Update readme.md Co-authored-by: Sindre Sorhus --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 1e6c8502d..ab0718ac9 100644 --- a/readme.md +++ b/readme.md @@ -989,7 +989,7 @@ The passphrase to decrypt the `options.https.key` (if different keys have differ Type: `string | Buffer | (string | Buffer | object)[]` -PFX or PKCS12 encoded private key and certificate chain. `pfx` is an alternative to providing `options.https.key` and `options.https.certificate` individually. PFX is usually encrypted, if it is, `options.https.passphrase` will be used to decrypt it. +PFX or PKCS12 encoded private key and certificate chain. `pfx` is an alternative to providing `options.https.key` and `options.https.certificate` individually. PFX is usually encrypted, and if it is, `options.https.passphrase` will be used to decrypt it. Multiple PFX's can be be provided as an array of unencrypted buffers or an array of objects like: From 335641bb4cde6813e2beb874ede850c0f933866d Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Tue, 21 Jul 2020 09:47:49 -0400 Subject: [PATCH 11/23] update README --- readme.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/readme.md b/readme.md index ab0718ac9..3a78898e2 100644 --- a/readme.md +++ b/readme.md @@ -989,16 +989,16 @@ The passphrase to decrypt the `options.https.key` (if different keys have differ Type: `string | Buffer | (string | Buffer | object)[]` -PFX or PKCS12 encoded private key and certificate chain. `pfx` is an alternative to providing `options.https.key` and `options.https.certificate` individually. PFX is usually encrypted, and if it is, `options.https.passphrase` will be used to decrypt it. +[PFX or PKCS12](https://en.wikipedia.org/wiki/PKCS_12) encoded private key and certificate chain. `pfx` is an alternative to providing `options.https.key` and `options.https.certificate` individually. PFX is usually encrypted, and if it is, `options.https.passphrase` will be used to decrypt it. Multiple PFX's can be be provided as an array of unencrypted buffers or an array of objects like: -` +```typescript { - buffer: string | Buffer, - passphrase?: string + buffer: string | Buffer, + passphrase?: string } -` +``` If the provided buffers are encrypted, `object.passphrase` can be used to decrypt them. If `object.passphrase` is not provided, `options.https.passphrase` will be used for decryption. From b039a37707d1d2774282205c290997493234b5b1 Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Tue, 21 Jul 2020 09:50:03 -0400 Subject: [PATCH 12/23] add is.buffer type check for pfx option --- source/core/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/core/index.ts b/source/core/index.ts index 47b8b30f8..3bf02825b 100644 --- a/source/core/index.ts +++ b/source/core/index.ts @@ -685,7 +685,7 @@ export default class Request extends Duplex implements RequestEvents { assert.any([is.string, is.object, is.array, is.undefined], options.https.key); assert.any([is.string, is.object, is.array, is.undefined], options.https.certificate); assert.any([is.string, is.undefined], options.https.passphrase); - assert.any([is.string, is.array, is.undefined], options.https.pfx); + assert.any([is.string, is.buffer, is.array, is.undefined], options.https.pfx); } // `options.method` From 73994f37b963ececce737ef014d950833f9fb735 Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Tue, 21 Jul 2020 10:08:27 -0400 Subject: [PATCH 13/23] add examples to README --- readme.md | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/readme.md b/readme.md index 3a78898e2..e8b094fe0 100644 --- a/readme.md +++ b/readme.md @@ -989,7 +989,7 @@ The passphrase to decrypt the `options.https.key` (if different keys have differ Type: `string | Buffer | (string | Buffer | object)[]` -[PFX or PKCS12](https://en.wikipedia.org/wiki/PKCS_12) encoded private key and certificate chain. `pfx` is an alternative to providing `options.https.key` and `options.https.certificate` individually. PFX is usually encrypted, and if it is, `options.https.passphrase` will be used to decrypt it. +[PFX or PKCS12](https://en.wikipedia.org/wiki/PKCS_12) encoded private key and certificate chain. Using `options.https.pfx` is an alternative to providing `options.https.key` and `options.https.certificate` individually. A PFX is usually encrypted, and if it is, `options.https.passphrase` will be used to decrypt it. Multiple PFX's can be be provided as an array of unencrypted buffers or an array of objects like: @@ -1002,7 +1002,7 @@ Multiple PFX's can be be provided as an array of unencrypted buffers or an array If the provided buffers are encrypted, `object.passphrase` can be used to decrypt them. If `object.passphrase` is not provided, `options.https.passphrase` will be used for decryption. -##### Examples for `https.key`, `https.certificate` and `https.passphrase` +##### Examples for `https.key`, `https.certificate`, `https.passphrase`, and `https.pfx` ```js // Single key with certificate @@ -1049,6 +1049,39 @@ got('https://example.com', { ] } }); + +// Single encrypted PFX with passphrase +got('https://example.com', { + https: { + pfx: fs.readFileSync('./fake.pfx'), + passphrase: 'passphrase' + } +}); + +// Multiple encrypted PFX's with different passphrases +got('https://example.com', { + https: { + pfx: [{ + buffer: fs.readFileSync('./key1.pfx'), + passphrase: 'passphrase1' + }, { + buffer: fs.readFileSync('./key2.pfx'), + passphrase: 'passphrase2' + }] + } +}); + +// Multiple encrypted PFX's with single passphrase +got('https://example.com', { + https: { + passphrase: 'passphrase', + pfx: [{ + buffer: fs.readFileSync('./key1.pfx') + }, { + buffer: fs.readFileSync('./key2.pfx') + }] + } +}); ``` ##### https.rejectUnauthorized From c143991e82d17722b8e82af34093fe5a8855cd0a Mon Sep 17 00:00:00 2001 From: Sindre Sorhus Date: Fri, 24 Jul 2020 17:45:23 +0800 Subject: [PATCH 14/23] Update readme.md --- readme.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/readme.md b/readme.md index e8b094fe0..e7c5818fb 100644 --- a/readme.md +++ b/readme.md @@ -993,10 +993,10 @@ Type: `string | Buffer | (string | Buffer | object)[]` Multiple PFX's can be be provided as an array of unencrypted buffers or an array of objects like: -```typescript +```ts { - buffer: string | Buffer, - passphrase?: string + buffer: string | Buffer, + passphrase?: string } ``` From 2722d3c23f00a1094244fbf7d50adc5efca51100 Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Fri, 24 Jul 2020 10:16:26 -0400 Subject: [PATCH 15/23] Update readme.md Co-authored-by: Sindre Sorhus --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index e7c5818fb..a7562ad53 100644 --- a/readme.md +++ b/readme.md @@ -987,7 +987,7 @@ The passphrase to decrypt the `options.https.key` (if different keys have differ ##### https.pfx -Type: `string | Buffer | (string | Buffer | object)[]` +Type: `string | Buffer | Array` [PFX or PKCS12](https://en.wikipedia.org/wiki/PKCS_12) encoded private key and certificate chain. Using `options.https.pfx` is an alternative to providing `options.https.key` and `options.https.certificate` individually. A PFX is usually encrypted, and if it is, `options.https.passphrase` will be used to decrypt it. From 4977057a72dcccdbfcac653c115437eba798b307 Mon Sep 17 00:00:00 2001 From: Mark Boyd Date: Fri, 24 Jul 2020 10:21:23 -0400 Subject: [PATCH 16/23] update README --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index a7562ad53..0b72bed5e 100644 --- a/readme.md +++ b/readme.md @@ -1000,7 +1000,7 @@ Multiple PFX's can be be provided as an array of unencrypted buffers or an array } ``` -If the provided buffers are encrypted, `object.passphrase` can be used to decrypt them. If `object.passphrase` is not provided, `options.https.passphrase` will be used for decryption. +This object form can only occur in an array. If the provided buffers are encrypted, `object.passphrase` can be used to decrypt them. If `object.passphrase` is not provided, `options.https.passphrase` will be used for decryption. ##### Examples for `https.key`, `https.certificate`, `https.passphrase`, and `https.pfx` From 795da4086769269c1cc207eb9adc3b0196cdeec6 Mon Sep 17 00:00:00 2001 From: Sindre Sorhus Date: Sat, 25 Jul 2020 08:59:21 +0200 Subject: [PATCH 17/23] Update readme.md --- readme.md | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/readme.md b/readme.md index 0b72bed5e..f24766d0b 100644 --- a/readme.md +++ b/readme.md @@ -1061,13 +1061,16 @@ got('https://example.com', { // Multiple encrypted PFX's with different passphrases got('https://example.com', { https: { - pfx: [{ - buffer: fs.readFileSync('./key1.pfx'), - passphrase: 'passphrase1' - }, { - buffer: fs.readFileSync('./key2.pfx'), - passphrase: 'passphrase2' - }] + pfx: [ + { + buffer: fs.readFileSync('./key1.pfx'), + passphrase: 'passphrase1' + }, + { + buffer: fs.readFileSync('./key2.pfx'), + passphrase: 'passphrase2' + } + ] } }); @@ -1075,11 +1078,14 @@ got('https://example.com', { got('https://example.com', { https: { passphrase: 'passphrase', - pfx: [{ - buffer: fs.readFileSync('./key1.pfx') - }, { - buffer: fs.readFileSync('./key2.pfx') - }] + pfx: [ + { + buffer: fs.readFileSync('./key1.pfx') + }, + { + buffer: fs.readFileSync('./key2.pfx') + } + ] } }); ``` From e02845f1aa737d55dce23381d0f4f2a61b1eb5e1 Mon Sep 17 00:00:00 2001 From: David Wu Date: Tue, 28 Jul 2020 10:44:16 +0200 Subject: [PATCH 18/23] Fix duplicated hooks when paginating --- source/create.ts | 19 ++-- test/hooks.ts | 246 ++++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 254 insertions(+), 11 deletions(-) diff --git a/source/create.ts b/source/create.ts index a930dd230..09231a42d 100644 --- a/source/create.ts +++ b/source/create.ts @@ -38,7 +38,7 @@ import { StreamOptions } from './types'; import createRejection from './as-promise/create-rejection'; -import Request, {kIsNormalizedAlready, setNonEnumerableProperties} from './core'; +import Request, {kIsNormalizedAlready, setNonEnumerableProperties, Defaults} from './core'; import deepFreeze from './utils/deep-freeze'; const errors = { @@ -114,7 +114,7 @@ const create = (defaults: InstanceDefaults): Got => { })); // Got interface - const got: Got = ((url: string | URL, options?: Options): GotReturn => { + const got: Got = ((url: string | URL, options?: Options, _defaults?: Defaults): GotReturn => { let iteration = 0; const iterateHandlers = (newOptions: NormalizedOptions): GotReturn => { return defaults.handlers[iteration++]( @@ -147,7 +147,7 @@ const create = (defaults: InstanceDefaults): Got => { } // Normalize options & call handlers - const normalizedOptions = normalizeArguments(url, options, defaults.options); + const normalizedOptions = normalizeArguments(url, options, _defaults ?? defaults.options); normalizedOptions[kIsNormalizedAlready] = true; if (initHookError) { @@ -199,7 +199,7 @@ const create = (defaults: InstanceDefaults): Got => { }; // Pagination - const paginateEach = (async function * (url: string | URL, options?: OptionsWithPagination) { + const paginateEach = (async function * (url: string | URL, options?: OptionsWithPagination): AsyncIterableIterator { // TODO: Remove this `@ts-expect-error` when upgrading to TypeScript 4. // Error: Argument of type 'Merge> | undefined' is not assignable to parameter of type 'Options | undefined'. // @ts-expect-error @@ -222,9 +222,10 @@ const create = (defaults: InstanceDefaults): Got => { await delay(pagination.backoff); } + // @ts-expect-error FIXME! // TODO: Throw when result is not an instance of Response // eslint-disable-next-line no-await-in-loop - const result = (await got(normalizedOptions)) as Response; + const result = (await got(undefined, undefined, normalizedOptions)) as Response; // eslint-disable-next-line no-await-in-loop const parsed = await pagination.transform(result); @@ -236,7 +237,7 @@ const create = (defaults: InstanceDefaults): Got => { return; } - yield item; + yield item as T; if (pagination.stackAllItems) { all.push(item as T); @@ -266,14 +267,12 @@ const create = (defaults: InstanceDefaults): Got => { } }); - got.paginate = ((url: string | URL, options?: OptionsWithPagination) => { - return paginateEach(url, options); - }) as GotPaginate; + got.paginate = paginateEach as GotPaginate; got.paginate.all = (async (url: string | URL, options?: OptionsWithPagination) => { const results: T[] = []; - for await (const item of got.paginate(url, options)) { + for await (const item of paginateEach(url, options)) { results.push(item); } diff --git a/test/hooks.ts b/test/hooks.ts index a72d44247..130b8199e 100644 --- a/test/hooks.ts +++ b/test/hooks.ts @@ -7,7 +7,7 @@ import sinon = require('sinon'); import delay = require('delay'); import {Handler} from 'express'; import Responselike = require('responselike'); -import got, {RequestError, HTTPError} from '../source'; +import got, {RequestError, HTTPError, Response} from '../source'; import withServer from './helpers/with-server'; const errorString = 'oops'; @@ -950,3 +950,247 @@ test('beforeRequest hook respect `url` option', withServer, async (t, server, go } })).body, 'ok'); }); + +test('no duplicate hook calls in single-page paginated requests', withServer, async (t, server, got) => { + server.get('/get', (_request, response) => { + response.end('i <3 koalas'); + }); + + let beforeHookCount = 0; + let beforeHookCountAdditional = 0; + let afterHookCount = 0; + let afterHookCountAdditional = 0; + + const hooks = { + beforeRequest: [ + () => { + beforeHookCount++; + } + ], + afterResponse: [ + (response: any) => { + afterHookCount++; + return response; + } + ] + }; + + // Test only one request + const instance = got.extend({ + hooks, + pagination: { + paginate: () => false, + countLimit: 2009, + transform: response => [response] + } + }); + + await instance.paginate.all('get'); + t.is(beforeHookCount, 1); + t.is(afterHookCount, 1); + + await instance.paginate.all('get', { + hooks: { + beforeRequest: [ + () => { + beforeHookCountAdditional++; + } + ], + afterResponse: [ + (response: any) => { + afterHookCountAdditional++; + return response; + } + ] + } + }); + t.is(beforeHookCount, 2); + t.is(afterHookCount, 2); + t.is(beforeHookCountAdditional, 1); + t.is(afterHookCountAdditional, 1); + + await got.paginate.all('get', { + hooks, + pagination: { + paginate: () => false, + transform: response => [response] + } + }); + + t.is(beforeHookCount, 3); + t.is(afterHookCount, 3); +}); + +test('no duplicate hook calls in sequential paginated requests', withServer, async (t, server, got) => { + server.get('/get', (_request, response) => { + response.end('i <3 unicorns'); + }); + + let requestNumber = 0; + let beforeHookCount = 0; + let afterHookCount = 0; + + const hooks = { + beforeRequest: [ + () => { + beforeHookCount++; + } + ], + afterResponse: [ + (response: any) => { + afterHookCount++; + return response; + } + ] + }; + + // Test only two requests, one after another + const paginate = () => requestNumber++ === 0 ? {} : false; + + const instance = got.extend({ + hooks, + pagination: { + paginate, + countLimit: 2009, + transform: response => [response] + } + }); + + await instance.paginate.all('get'); + + t.is(beforeHookCount, 2); + t.is(afterHookCount, 2); + requestNumber = 0; + + await got.paginate.all('get', { + hooks, + pagination: { + paginate, + transform: response => [response] + } + }); + + t.is(beforeHookCount, 4); + t.is(afterHookCount, 4); +}); + +test('intentional duplicate hooks in pagination with extended instance', withServer, async (t, server, got) => { + server.get('/get', (_request, response) => { + response.end('<3'); + }); + + let beforeCount = 0; // Number of times the hooks from `extend` are called + let afterCount = 0; + let beforeCountAdditional = 0; // Number of times the added hooks are called + let afterCountAdditional = 0; + + const beforeHook = () => { + beforeCount++; + }; + + const afterHook = (response: any) => { + afterCount++; + return response; + }; + + const instance = got.extend({ + hooks: { + beforeRequest: [ + beforeHook, + beforeHook + ], + afterResponse: [ + afterHook, + afterHook + ] + }, + pagination: { + paginate: () => false, + countLimit: 2009, + transform: response => [response] + } + }); + + // Add duplicate hooks when calling paginate + const beforeHookAdditional = () => { + beforeCountAdditional++; + }; + + const afterHookAdditional = (response: any) => { + afterCountAdditional++; + return response; + }; + + await instance.paginate.all('get', { + hooks: { + beforeRequest: [ + beforeHook, + beforeHookAdditional, + beforeHookAdditional + ], + afterResponse: [ + afterHook, + afterHookAdditional, + afterHookAdditional + ] + } + }); + + t.is(beforeCount, 3); + t.is(afterCount, 3); + t.is(beforeCountAdditional, 2); + t.is(afterCountAdditional, 2); +}); + +test('no duplicate hook calls when returning original request options', withServer, async (t, server, got) => { + server.get('/get', (_request, response) => { + response.end('i <3 unicorns'); + }); + + let requestNumber = 0; + let beforeHookCount = 0; + let afterHookCount = 0; + + const hooks = { + beforeRequest: [ + () => { + beforeHookCount++; + } + ], + afterResponse: [ + (response: any) => { + afterHookCount++; + return response; + } + ] + }; + + // Test only two requests, one after another + const paginate = (response: Response) => requestNumber++ === 0 ? response.request.options : false; + + const instance = got.extend({ + hooks, + pagination: { + paginate, + countLimit: 2009, + transform: response => [response] + } + }); + + await instance.paginate.all('get'); + + t.is(beforeHookCount, 2); + t.is(afterHookCount, 2); + requestNumber = 0; + + await got.paginate.all('get', { + hooks, + pagination: { + paginate, + transform: response => [response] + } + }); + + t.is(beforeHookCount, 4); + t.is(afterHookCount, 4); +}); From 043c9501b85172e09819d44ac8eb49c574b27bda Mon Sep 17 00:00:00 2001 From: Szymon Marczak <36894700+szmarczak@users.noreply.github.com> Date: Tue, 28 Jul 2020 10:59:11 +0200 Subject: [PATCH 19/23] Fix `dnsCache: true` having no effect Fixes #1372 --- source/core/index.ts | 4 +++- test/create.ts | 12 ++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/source/core/index.ts b/source/core/index.ts index f49889235..b51a6ae52 100644 --- a/source/core/index.ts +++ b/source/core/index.ts @@ -28,6 +28,8 @@ import {DnsLookupIpVersion, isDnsLookupIpVersion, dnsLookupIpVersionToFamily} fr import deprecationWarning from '../utils/deprecation-warning'; import {PromiseOnly} from '../as-promise/types'; +const globalDnsCache = new CacheableLookup(); + type HttpRequestFunction = typeof httpRequest; type Error = NodeJS.ErrnoException; @@ -888,7 +890,7 @@ export default class Request extends Duplex implements RequestEvents { // `options.dnsCache` if (options.dnsCache === true) { - options.dnsCache = new CacheableLookup(); + options.dnsCache = globalDnsCache; } else if (!is.undefined(options.dnsCache) && !options.dnsCache.lookup) { throw new TypeError(`Parameter \`dnsCache\` must be a CacheableLookup instance or a boolean, got ${is(options.dnsCache)}`); } diff --git a/test/create.ts b/test/create.ts index d8589653f..75f47cdc1 100644 --- a/test/create.ts +++ b/test/create.ts @@ -312,3 +312,15 @@ test('async handlers can throw', async t => { await t.throwsAsync(instance('https://example.com'), {message}); }); + +test('setting dnsCache to true points to global cache', t => { + const a = got.extend({ + dnsCache: true + }); + + const b = got.extend({ + dnsCache: true + }); + + t.is(a.defaults.options.dnsCache, b.defaults.options.dnsCache); +}); From ddd793062b02efef00ef70f6b68d09b8b01580d6 Mon Sep 17 00:00:00 2001 From: Sam Verschueren Date: Tue, 28 Jul 2020 11:02:58 +0200 Subject: [PATCH 20/23] Add got4aws to AWS section (#1380) --- readme.md | 25 ++++--------------------- 1 file changed, 4 insertions(+), 21 deletions(-) diff --git a/readme.md b/readme.md index 846f437bc..0cf6cbdc1 100644 --- a/readme.md +++ b/readme.md @@ -699,8 +699,6 @@ Default: `[]` Called with [normalized](source/core/index.ts) [request options](#options). Got will make no further changes to the request before it is sent. This is especially useful in conjunction with [`got.extend()`](#instances) when you want to create an API client that, for example, uses HMAC-signing. -See the [AWS section](#aws) for an example. - **Tip:** You can override the `request` function by returning a [`ClientRequest`-like](https://nodejs.org/api/http.html#http_class_http_clientrequest) instance or a [`IncomingMessage`-like](https://nodejs.org/api/http.html#http_class_http_incomingmessage) instance. This is very useful when creating a custom cache mechanism. ###### hooks.beforeRedirect @@ -1860,29 +1858,14 @@ got('unix:/var/run/docker.sock:/containers/json'); ## AWS -Requests to AWS services need to have their headers signed. This can be accomplished by using the [`aws4`](https://www.npmjs.com/package/aws4) package. This is an example for querying an ["API Gateway"](https://docs.aws.amazon.com/apigateway/api-reference/signing-requests/) with a signed request. +Requests to AWS services need to have their headers signed. This can be accomplished by using the [`got4aws`](https://www.npmjs.com/package/got4aws) package. This is an example for querying an ["API Gateway"](https://docs.aws.amazon.com/apigateway/api-reference/signing-requests/) with a signed request. ```js -const got = require('got'); -const AWS = require('aws-sdk'); -const aws4 = require('aws4'); - -const chain = new AWS.CredentialProviderChain(); +const got4aws = require('got4aws');; -// Create a Got instance to use relative paths and signed requests -const awsClient = got.extend({ - prefixUrl: 'https://.execute-api..amazonaws.com//', - hooks: { - beforeRequest: [ - async options => { - const credentials = await chain.resolvePromise(); - aws4.sign(options, credentials); - } - ] - } -}); +const awsClient = got4aws(); -const response = await awsClient('endpoint/path', { +const response = await awsClient('https://.execute-api..amazonaws.com//endpoint/path', { // Request-specific options }); ``` From a748343363ecb0347897264fb49df4a8f4793997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jesper=20Br=C3=A4nn?= Date: Wed, 5 Aug 2020 11:53:37 +0200 Subject: [PATCH 21/23] Mention header lowercasing in`request` migration guide (#1387) --- documentation/migration-guides.md | 1 + 1 file changed, 1 insertion(+) diff --git a/documentation/migration-guides.md b/documentation/migration-guides.md index 2831f9c23..3717c36f6 100644 --- a/documentation/migration-guides.md +++ b/documentation/migration-guides.md @@ -75,6 +75,7 @@ To use streams, just call `got.stream(url, options)` or `got(url, {isStream: tru - The `json` option is not a `boolean`, it's an `Object`. It will be stringified and used as a body. - The `form` option is an `Object`. It can be a plain object or a [`form-data` instance](https://github.com/sindresorhus/got/#form-data). +- Got will lowercase all custom headers, even if they are specified to not be. - No `oauth`/`hawk`/`aws`/`httpSignature` option. To sign requests, you need to create a [custom instance](advanced-creation.md#signing-requests). - No `agentClass`/`agentOptions`/`pool` option. - No `forever` option. You need to use [forever-agent](https://github.com/request/forever-agent). From 9a309bdbe7e2552c5bffbea253d58c39d6e6c3e3 Mon Sep 17 00:00:00 2001 From: Giovanni Minotti Date: Thu, 6 Aug 2020 04:13:37 +0200 Subject: [PATCH 22/23] Fixed deprecationWarning on https options (#1391) Co-authored-by: Aleksey Timchenko --- source/core/index.ts | 27 +++++++++++++++++++++++++++ test/https.ts | 42 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/source/core/index.ts b/source/core/index.ts index b51a6ae52..21f579bcc 100644 --- a/source/core/index.ts +++ b/source/core/index.ts @@ -1543,6 +1543,33 @@ export default class Request extends Duplex implements RequestEvents { options.timeout = timeout; options.agent = agent; + // HTTPS options restore + if (options.https) { + if ('rejectUnauthorized' in options.https) { + delete requestOptions.rejectUnauthorized; + } + + if (options.https.checkServerIdentity) { + delete requestOptions.checkServerIdentity; + } + + if (options.https.certificateAuthority) { + delete requestOptions.ca; + } + + if (options.https.certificate) { + delete requestOptions.cert; + } + + if (options.https.key) { + delete requestOptions.key; + } + + if (options.https.passphrase) { + delete requestOptions.passphrase; + } + } + if (isClientRequest(requestOrResponse)) { this._onRequest(requestOrResponse); diff --git a/test/https.ts b/test/https.ts index 9ead3ff92..d5a55866d 100644 --- a/test/https.ts +++ b/test/https.ts @@ -31,6 +31,48 @@ test('https request with ca', withServer, async (t, server, got) => { t.is(body, 'ok'); }); +test('https request with ca and afterResponse hook', withServer, async (t, server, got) => { + server.get('/', (_request, response) => { + response.end('ok'); + }); + + const warningListener = (warning: any) => { + if ( + warning.name === 'DeprecationWarning' && + warning.message === 'Got: "options.ca" was never documented, please use ' + + '"options.https.certificateAuthority"' + ) { + process.off('warning', warningListener); + t.fail('unexpected deprecation warning'); + } + }; + + process.once('warning', warningListener); + + let shouldRetry = true; + const {body} = await got.secure({ + https: { + certificateAuthority: server.caCert + }, + headers: {host: 'example.com'}, + hooks: { + afterResponse: [ + (response, retry) => { + if (shouldRetry) { + shouldRetry = false; + + return retry({}); + } + + return response; + } + ] + } + }); + + t.is(body, 'ok'); +}); + test('https request with `checkServerIdentity` OK', withServer, async (t, server, got) => { server.get('/', (_request, response) => { response.end('ok'); From 82d547c9125789259f865088a8964ac32e8b5741 Mon Sep 17 00:00:00 2001 From: Giovanni Minotti Date: Mon, 31 Aug 2020 16:44:52 +0200 Subject: [PATCH 23/23] Added `pfx` to the "HTTPS options restore" section --- source/core/index.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/core/index.ts b/source/core/index.ts index 3ed685622..73aab4fed 100644 --- a/source/core/index.ts +++ b/source/core/index.ts @@ -1578,6 +1578,10 @@ export default class Request extends Duplex implements RequestEvents { if (options.https.passphrase) { delete requestOptions.passphrase; } + + if (options.https.pfx) { + delete requestOptions.pfx; + } } if (isClientRequest(requestOrResponse)) {