diff --git a/documentation/migration-guides.md b/documentation/migration-guides.md index 2831f9c23..3717c36f6 100644 --- a/documentation/migration-guides.md +++ b/documentation/migration-guides.md @@ -75,6 +75,7 @@ To use streams, just call `got.stream(url, options)` or `got(url, {isStream: tru - The `json` option is not a `boolean`, it's an `Object`. It will be stringified and used as a body. - The `form` option is an `Object`. It can be a plain object or a [`form-data` instance](https://github.com/sindresorhus/got/#form-data). +- Got will lowercase all custom headers, even if they are specified to not be. - No `oauth`/`hawk`/`aws`/`httpSignature` option. To sign requests, you need to create a [custom instance](advanced-creation.md#signing-requests). - No `agentClass`/`agentOptions`/`pool` option. - No `forever` option. You need to use [forever-agent](https://github.com/request/forever-agent). diff --git a/readme.md b/readme.md index 9f40ed17a..fa2ac38de 100644 --- a/readme.md +++ b/readme.md @@ -699,8 +699,6 @@ Default: `[]` Called with [normalized](source/core/index.ts) [request options](#options). Got will make no further changes to the request before it is sent. This is especially useful in conjunction with [`got.extend()`](#instances) when you want to create an API client that, for example, uses HMAC-signing. -See the [AWS section](#aws) for an example. - **Tip:** You can override the `request` function by returning a [`ClientRequest`-like](https://nodejs.org/api/http.html#http_class_http_clientrequest) instance or a [`IncomingMessage`-like](https://nodejs.org/api/http.html#http_class_http_incomingmessage) instance. This is very useful when creating a custom cache mechanism. ###### hooks.beforeRedirect @@ -985,7 +983,24 @@ Type: `string` The passphrase to decrypt the `options.https.key` (if different keys have different passphrases refer to `options.https.key` documentation). -##### Examples for `https.key`, `https.certificate` and `https.passphrase` +##### https.pfx + +Type: `string | Buffer | Array` + +[PFX or PKCS12](https://en.wikipedia.org/wiki/PKCS_12) encoded private key and certificate chain. Using `options.https.pfx` is an alternative to providing `options.https.key` and `options.https.certificate` individually. A PFX is usually encrypted, and if it is, `options.https.passphrase` will be used to decrypt it. + +Multiple PFX's can be be provided as an array of unencrypted buffers or an array of objects like: + +```ts +{ + buffer: string | Buffer, + passphrase?: string +} +``` + +This object form can only occur in an array. If the provided buffers are encrypted, `object.passphrase` can be used to decrypt them. If `object.passphrase` is not provided, `options.https.passphrase` will be used for decryption. + +##### Examples for `https.key`, `https.certificate`, `https.passphrase`, and `https.pfx` ```js // Single key with certificate @@ -1032,6 +1047,45 @@ got('https://example.com', { ] } }); + +// Single encrypted PFX with passphrase +got('https://example.com', { + https: { + pfx: fs.readFileSync('./fake.pfx'), + passphrase: 'passphrase' + } +}); + +// Multiple encrypted PFX's with different passphrases +got('https://example.com', { + https: { + pfx: [ + { + buffer: fs.readFileSync('./key1.pfx'), + passphrase: 'passphrase1' + }, + { + buffer: fs.readFileSync('./key2.pfx'), + passphrase: 'passphrase2' + } + ] + } +}); + +// Multiple encrypted PFX's with single passphrase +got('https://example.com', { + https: { + passphrase: 'passphrase', + pfx: [ + { + buffer: fs.readFileSync('./key1.pfx') + }, + { + buffer: fs.readFileSync('./key2.pfx') + } + ] + } +}); ``` ##### https.rejectUnauthorized @@ -1750,6 +1804,26 @@ got('https://sindresorhus.com', { }); ``` +Otherwise, you can use the [`hpagent`](https://github.com/delvedor/hpagent) package, which keeps the internal sockets alive to be reused. + +```js +const got = require('got'); +const {HttpsProxyAgent} = require('hpagent'); + +got('https://sindresorhus.com', { + agent: { + https: new HttpsProxyAgent({ + keepAlive: true, + keepAliveMsecs: 1000, + maxSockets: 256, + maxFreeSockets: 256, + scheduling: 'lifo', + proxy: 'https://localhost:8080' + }) + } +}); +``` + Alternatively, use [`global-agent`](https://github.com/gajus/global-agent) to configure a global proxy for all HTTP/HTTPS traffic in your program. Read the [`http2-wrapper`](https://github.com/szmarczak/http2-wrapper/#proxy-support) docs to learn about proxying for HTTP/2. @@ -1840,29 +1914,14 @@ got('unix:/var/run/docker.sock:/containers/json'); ## AWS -Requests to AWS services need to have their headers signed. This can be accomplished by using the [`aws4`](https://www.npmjs.com/package/aws4) package. This is an example for querying an ["API Gateway"](https://docs.aws.amazon.com/apigateway/api-reference/signing-requests/) with a signed request. +Requests to AWS services need to have their headers signed. This can be accomplished by using the [`got4aws`](https://www.npmjs.com/package/got4aws) package. This is an example for querying an ["API Gateway"](https://docs.aws.amazon.com/apigateway/api-reference/signing-requests/) with a signed request. ```js -const got = require('got'); -const AWS = require('aws-sdk'); -const aws4 = require('aws4'); - -const chain = new AWS.CredentialProviderChain(); +const got4aws = require('got4aws');; -// Create a Got instance to use relative paths and signed requests -const awsClient = got.extend({ - prefixUrl: 'https://.execute-api..amazonaws.com//', - hooks: { - beforeRequest: [ - async options => { - const credentials = await chain.resolvePromise(); - aws4.sign(options, credentials); - } - ] - } -}); +const awsClient = got4aws(); -const response = await awsClient('endpoint/path', { +const response = await awsClient('https://.execute-api..amazonaws.com//endpoint/path', { // Request-specific options }); ``` diff --git a/source/core/index.ts b/source/core/index.ts index f49889235..73aab4fed 100644 --- a/source/core/index.ts +++ b/source/core/index.ts @@ -28,6 +28,8 @@ import {DnsLookupIpVersion, isDnsLookupIpVersion, dnsLookupIpVersionToFamily} fr import deprecationWarning from '../utils/deprecation-warning'; import {PromiseOnly} from '../as-promise/types'; +const globalDnsCache = new CacheableLookup(); + type HttpRequestFunction = typeof httpRequest; type Error = NodeJS.ErrnoException; @@ -187,6 +189,7 @@ export interface HTTPSOptions { key?: SecureContextOptions['key']; certificate?: SecureContextOptions['cert']; passphrase?: SecureContextOptions['passphrase']; + pfx?: SecureContextOptions['pfx']; } interface NormalizedPlainOptions extends PlainOptions { @@ -684,6 +687,7 @@ export default class Request extends Duplex implements RequestEvents { assert.any([is.string, is.object, is.array, is.undefined], options.https.key); assert.any([is.string, is.object, is.array, is.undefined], options.https.certificate); assert.any([is.string, is.undefined], options.https.passphrase); + assert.any([is.string, is.buffer, is.array, is.undefined], options.https.pfx); } // `options.method` @@ -888,7 +892,7 @@ export default class Request extends Duplex implements RequestEvents { // `options.dnsCache` if (options.dnsCache === true) { - options.dnsCache = new CacheableLookup(); + options.dnsCache = globalDnsCache; } else if (!is.undefined(options.dnsCache) && !options.dnsCache.lookup) { throw new TypeError(`Parameter \`dnsCache\` must be a CacheableLookup instance or a boolean, got ${is(options.dnsCache)}`); } @@ -975,6 +979,10 @@ export default class Request extends Duplex implements RequestEvents { deprecationWarning('"options.passphrase" was never documented, please use "options.https.passphrase"'); } + if ('pfx' in options) { + deprecationWarning('"options.pfx" was never documented, please use "options.https.pfx"'); + } + // Other options if ('followRedirects' in options) { throw new TypeError('The `followRedirects` option does not exist. Use `followRedirect` instead.'); @@ -1527,6 +1535,10 @@ export default class Request extends Duplex implements RequestEvents { if (options.https.passphrase) { requestOptions.passphrase = options.https.passphrase; } + + if (options.https.pfx) { + requestOptions.pfx = options.https.pfx; + } } try { @@ -1541,6 +1553,37 @@ export default class Request extends Duplex implements RequestEvents { options.timeout = timeout; options.agent = agent; + // HTTPS options restore + if (options.https) { + if ('rejectUnauthorized' in options.https) { + delete requestOptions.rejectUnauthorized; + } + + if (options.https.checkServerIdentity) { + delete requestOptions.checkServerIdentity; + } + + if (options.https.certificateAuthority) { + delete requestOptions.ca; + } + + if (options.https.certificate) { + delete requestOptions.cert; + } + + if (options.https.key) { + delete requestOptions.key; + } + + if (options.https.passphrase) { + delete requestOptions.passphrase; + } + + if (options.https.pfx) { + delete requestOptions.pfx; + } + } + if (isClientRequest(requestOrResponse)) { this._onRequest(requestOrResponse); diff --git a/source/create.ts b/source/create.ts index a930dd230..09231a42d 100644 --- a/source/create.ts +++ b/source/create.ts @@ -38,7 +38,7 @@ import { StreamOptions } from './types'; import createRejection from './as-promise/create-rejection'; -import Request, {kIsNormalizedAlready, setNonEnumerableProperties} from './core'; +import Request, {kIsNormalizedAlready, setNonEnumerableProperties, Defaults} from './core'; import deepFreeze from './utils/deep-freeze'; const errors = { @@ -114,7 +114,7 @@ const create = (defaults: InstanceDefaults): Got => { })); // Got interface - const got: Got = ((url: string | URL, options?: Options): GotReturn => { + const got: Got = ((url: string | URL, options?: Options, _defaults?: Defaults): GotReturn => { let iteration = 0; const iterateHandlers = (newOptions: NormalizedOptions): GotReturn => { return defaults.handlers[iteration++]( @@ -147,7 +147,7 @@ const create = (defaults: InstanceDefaults): Got => { } // Normalize options & call handlers - const normalizedOptions = normalizeArguments(url, options, defaults.options); + const normalizedOptions = normalizeArguments(url, options, _defaults ?? defaults.options); normalizedOptions[kIsNormalizedAlready] = true; if (initHookError) { @@ -199,7 +199,7 @@ const create = (defaults: InstanceDefaults): Got => { }; // Pagination - const paginateEach = (async function * (url: string | URL, options?: OptionsWithPagination) { + const paginateEach = (async function * (url: string | URL, options?: OptionsWithPagination): AsyncIterableIterator { // TODO: Remove this `@ts-expect-error` when upgrading to TypeScript 4. // Error: Argument of type 'Merge> | undefined' is not assignable to parameter of type 'Options | undefined'. // @ts-expect-error @@ -222,9 +222,10 @@ const create = (defaults: InstanceDefaults): Got => { await delay(pagination.backoff); } + // @ts-expect-error FIXME! // TODO: Throw when result is not an instance of Response // eslint-disable-next-line no-await-in-loop - const result = (await got(normalizedOptions)) as Response; + const result = (await got(undefined, undefined, normalizedOptions)) as Response; // eslint-disable-next-line no-await-in-loop const parsed = await pagination.transform(result); @@ -236,7 +237,7 @@ const create = (defaults: InstanceDefaults): Got => { return; } - yield item; + yield item as T; if (pagination.stackAllItems) { all.push(item as T); @@ -266,14 +267,12 @@ const create = (defaults: InstanceDefaults): Got => { } }); - got.paginate = ((url: string | URL, options?: OptionsWithPagination) => { - return paginateEach(url, options); - }) as GotPaginate; + got.paginate = paginateEach as GotPaginate; got.paginate.all = (async (url: string | URL, options?: OptionsWithPagination) => { const results: T[] = []; - for await (const item of got.paginate(url, options)) { + for await (const item of paginateEach(url, options)) { results.push(item); } diff --git a/test/create.ts b/test/create.ts index d8589653f..75f47cdc1 100644 --- a/test/create.ts +++ b/test/create.ts @@ -312,3 +312,15 @@ test('async handlers can throw', async t => { await t.throwsAsync(instance('https://example.com'), {message}); }); + +test('setting dnsCache to true points to global cache', t => { + const a = got.extend({ + dnsCache: true + }); + + const b = got.extend({ + dnsCache: true + }); + + t.is(a.defaults.options.dnsCache, b.defaults.options.dnsCache); +}); diff --git a/test/hooks.ts b/test/hooks.ts index a72d44247..130b8199e 100644 --- a/test/hooks.ts +++ b/test/hooks.ts @@ -7,7 +7,7 @@ import sinon = require('sinon'); import delay = require('delay'); import {Handler} from 'express'; import Responselike = require('responselike'); -import got, {RequestError, HTTPError} from '../source'; +import got, {RequestError, HTTPError, Response} from '../source'; import withServer from './helpers/with-server'; const errorString = 'oops'; @@ -950,3 +950,247 @@ test('beforeRequest hook respect `url` option', withServer, async (t, server, go } })).body, 'ok'); }); + +test('no duplicate hook calls in single-page paginated requests', withServer, async (t, server, got) => { + server.get('/get', (_request, response) => { + response.end('i <3 koalas'); + }); + + let beforeHookCount = 0; + let beforeHookCountAdditional = 0; + let afterHookCount = 0; + let afterHookCountAdditional = 0; + + const hooks = { + beforeRequest: [ + () => { + beforeHookCount++; + } + ], + afterResponse: [ + (response: any) => { + afterHookCount++; + return response; + } + ] + }; + + // Test only one request + const instance = got.extend({ + hooks, + pagination: { + paginate: () => false, + countLimit: 2009, + transform: response => [response] + } + }); + + await instance.paginate.all('get'); + t.is(beforeHookCount, 1); + t.is(afterHookCount, 1); + + await instance.paginate.all('get', { + hooks: { + beforeRequest: [ + () => { + beforeHookCountAdditional++; + } + ], + afterResponse: [ + (response: any) => { + afterHookCountAdditional++; + return response; + } + ] + } + }); + t.is(beforeHookCount, 2); + t.is(afterHookCount, 2); + t.is(beforeHookCountAdditional, 1); + t.is(afterHookCountAdditional, 1); + + await got.paginate.all('get', { + hooks, + pagination: { + paginate: () => false, + transform: response => [response] + } + }); + + t.is(beforeHookCount, 3); + t.is(afterHookCount, 3); +}); + +test('no duplicate hook calls in sequential paginated requests', withServer, async (t, server, got) => { + server.get('/get', (_request, response) => { + response.end('i <3 unicorns'); + }); + + let requestNumber = 0; + let beforeHookCount = 0; + let afterHookCount = 0; + + const hooks = { + beforeRequest: [ + () => { + beforeHookCount++; + } + ], + afterResponse: [ + (response: any) => { + afterHookCount++; + return response; + } + ] + }; + + // Test only two requests, one after another + const paginate = () => requestNumber++ === 0 ? {} : false; + + const instance = got.extend({ + hooks, + pagination: { + paginate, + countLimit: 2009, + transform: response => [response] + } + }); + + await instance.paginate.all('get'); + + t.is(beforeHookCount, 2); + t.is(afterHookCount, 2); + requestNumber = 0; + + await got.paginate.all('get', { + hooks, + pagination: { + paginate, + transform: response => [response] + } + }); + + t.is(beforeHookCount, 4); + t.is(afterHookCount, 4); +}); + +test('intentional duplicate hooks in pagination with extended instance', withServer, async (t, server, got) => { + server.get('/get', (_request, response) => { + response.end('<3'); + }); + + let beforeCount = 0; // Number of times the hooks from `extend` are called + let afterCount = 0; + let beforeCountAdditional = 0; // Number of times the added hooks are called + let afterCountAdditional = 0; + + const beforeHook = () => { + beforeCount++; + }; + + const afterHook = (response: any) => { + afterCount++; + return response; + }; + + const instance = got.extend({ + hooks: { + beforeRequest: [ + beforeHook, + beforeHook + ], + afterResponse: [ + afterHook, + afterHook + ] + }, + pagination: { + paginate: () => false, + countLimit: 2009, + transform: response => [response] + } + }); + + // Add duplicate hooks when calling paginate + const beforeHookAdditional = () => { + beforeCountAdditional++; + }; + + const afterHookAdditional = (response: any) => { + afterCountAdditional++; + return response; + }; + + await instance.paginate.all('get', { + hooks: { + beforeRequest: [ + beforeHook, + beforeHookAdditional, + beforeHookAdditional + ], + afterResponse: [ + afterHook, + afterHookAdditional, + afterHookAdditional + ] + } + }); + + t.is(beforeCount, 3); + t.is(afterCount, 3); + t.is(beforeCountAdditional, 2); + t.is(afterCountAdditional, 2); +}); + +test('no duplicate hook calls when returning original request options', withServer, async (t, server, got) => { + server.get('/get', (_request, response) => { + response.end('i <3 unicorns'); + }); + + let requestNumber = 0; + let beforeHookCount = 0; + let afterHookCount = 0; + + const hooks = { + beforeRequest: [ + () => { + beforeHookCount++; + } + ], + afterResponse: [ + (response: any) => { + afterHookCount++; + return response; + } + ] + }; + + // Test only two requests, one after another + const paginate = (response: Response) => requestNumber++ === 0 ? response.request.options : false; + + const instance = got.extend({ + hooks, + pagination: { + paginate, + countLimit: 2009, + transform: response => [response] + } + }); + + await instance.paginate.all('get'); + + t.is(beforeHookCount, 2); + t.is(afterHookCount, 2); + requestNumber = 0; + + await got.paginate.all('get', { + hooks, + pagination: { + paginate, + transform: response => [response] + } + }); + + t.is(beforeHookCount, 4); + t.is(afterHookCount, 4); +}); diff --git a/test/https.ts b/test/https.ts index 9ead3ff92..d5a55866d 100644 --- a/test/https.ts +++ b/test/https.ts @@ -31,6 +31,48 @@ test('https request with ca', withServer, async (t, server, got) => { t.is(body, 'ok'); }); +test('https request with ca and afterResponse hook', withServer, async (t, server, got) => { + server.get('/', (_request, response) => { + response.end('ok'); + }); + + const warningListener = (warning: any) => { + if ( + warning.name === 'DeprecationWarning' && + warning.message === 'Got: "options.ca" was never documented, please use ' + + '"options.https.certificateAuthority"' + ) { + process.off('warning', warningListener); + t.fail('unexpected deprecation warning'); + } + }; + + process.once('warning', warningListener); + + let shouldRetry = true; + const {body} = await got.secure({ + https: { + certificateAuthority: server.caCert + }, + headers: {host: 'example.com'}, + hooks: { + afterResponse: [ + (response, retry) => { + if (shouldRetry) { + shouldRetry = false; + + return retry({}); + } + + return response; + } + ] + } + }); + + t.is(body, 'ok'); +}); + test('https request with `checkServerIdentity` OK', withServer, async (t, server, got) => { server.get('/', (_request, response) => { response.end('ok');