From ea88c523dff7a65b2c7546ab2ec679d60014f864 Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Fri, 26 Nov 2021 09:30:22 -0800 Subject: [PATCH 01/10] chore: code cleanup --- src/common/NetworkManager.ts | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/src/common/NetworkManager.ts b/src/common/NetworkManager.ts index 1b048c01fa564..be3c6c14b78d5 100644 --- a/src/common/NetworkManager.ts +++ b/src/common/NetworkManager.ts @@ -369,17 +369,6 @@ export class NetworkManager extends EventEmitter { } _onRequestPaused(event: Protocol.Fetch.RequestPausedEvent): void { - if ( - !this._userRequestInterceptionEnabled && - this._protocolRequestInterceptionEnabled - ) { - this._client - .send('Fetch.continueRequest', { - requestId: event.requestId, - }) - .catch(debugError); - } - const requestId = event.networkId; const interceptionId = event.requestId; From ce329b497107f1b2f27b6543db78358e8b7632d8 Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Fri, 26 Nov 2021 11:23:27 -0800 Subject: [PATCH 02/10] fix: handle multiple Fetch.requestPaused events --- src/common/NetworkManager.ts | 37 +++++++++++++++++++++++------------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/src/common/NetworkManager.ts b/src/common/NetworkManager.ts index be3c6c14b78d5..a70eff955a0e4 100644 --- a/src/common/NetworkManager.ts +++ b/src/common/NetworkManager.ts @@ -376,22 +376,33 @@ export class NetworkManager extends EventEmitter { return; } - let requestWillBeSentEvent = - this._requestIdToRequestWillBeSentEvent.get(requestId); - - // redirect requests have the same `requestId`, - if ( - requestWillBeSentEvent && - (requestWillBeSentEvent.request.url !== event.request.url || - requestWillBeSentEvent.request.method !== event.request.method) - ) { - this._requestIdToRequestWillBeSentEvent.delete(requestId); - requestWillBeSentEvent = null; - } + /** + * CDP may send a Fetch.requestPaused without or before a + * Network.requestWillBeSent + * + * CDP may send multiple Fetch.requestPaused + * for the same Network.requestWillBeSent. + * + * + */ + const requestWillBeSentEvent = (() => { + const requestWillBeSentEvent = + this._requestIdToRequestWillBeSentEvent.get(requestId); + + // redirect requests have the same `requestId`, + if ( + requestWillBeSentEvent && + (requestWillBeSentEvent.request.url !== event.request.url || + requestWillBeSentEvent.request.method !== event.request.method) + ) { + this._requestIdToRequestWillBeSentEvent.delete(requestId); + return; + } + return requestWillBeSentEvent; + })(); if (requestWillBeSentEvent) { this._onRequest(requestWillBeSentEvent, interceptionId); - this._requestIdToRequestWillBeSentEvent.delete(requestId); } else { this._requestIdToRequestPausedEvent.set(requestId, event); } From d8a6a30a4a07675a627f0c07f31bc301f567abd8 Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Fri, 26 Nov 2021 11:51:22 -0800 Subject: [PATCH 03/10] chore: refactor --- src/common/NetworkManager.ts | 104 ++++++++++++++++++++--------------- 1 file changed, 60 insertions(+), 44 deletions(-) diff --git a/src/common/NetworkManager.ts b/src/common/NetworkManager.ts index a70eff955a0e4..786746072b50d 100644 --- a/src/common/NetworkManager.ts +++ b/src/common/NetworkManager.ts @@ -112,11 +112,11 @@ export class NetworkManager extends EventEmitter { * `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`, ... * (see crbug.com/1196004) */ - _requestIdToRequestWillBeSentEvent = new Map< + _networkRequestIdToRequestWillBeSentEvent = new Map< string, Protocol.Network.RequestWillBeSentEvent >(); - _requestIdToRequestPausedEvent = new Map< + _networkRequestIdToRequestPausedEvent = new Map< string, Protocol.Fetch.RequestPausedEvent >(); @@ -131,18 +131,18 @@ export class NetworkManager extends EventEmitter { * handle redirects, we have to make them Arrays to represent the chain of * events. */ - _requestIdToResponseReceivedExtraInfo = new Map< + _networkRequestIdToResponseReceivedExtraInfo = new Map< string, Protocol.Network.ResponseReceivedExtraInfoEvent[] >(); - _requestIdToQueuedRedirectInfoMap = new Map< + _networkRequestIdToQueuedRedirectInfoMap = new Map< string, Array<{ event: Protocol.Network.RequestWillBeSentEvent; interceptionId?: string; }> >(); - _requestIdToQueuedEvents = new Map< + _networkRequestIdToQueuedEvents = new Map< string, { responseReceived: Protocol.Network.ResponseReceivedEvent; @@ -322,21 +322,28 @@ export class NetworkManager extends EventEmitter { } _onRequestWillBeSent(event: Protocol.Network.RequestWillBeSentEvent): void { + // console.log(`Network.requestWillBeSent`, { event }); // Request interception doesn't happen for data URLs with Network Service. if ( this._userRequestInterceptionEnabled && !event.request.url.startsWith('data:') ) { - const requestId = event.requestId; - const requestPausedEvent = - this._requestIdToRequestPausedEvent.get(requestId); + const { requestId: networkRequestId } = event; - this._requestIdToRequestWillBeSentEvent.set(requestId, event); + this._networkRequestIdToRequestWillBeSentEvent.set( + networkRequestId, + event + ); + /** + * CDP may have sent a Fetch.requestPaused event already. Check for it. + */ + const requestPausedEvent = + this._networkRequestIdToRequestPausedEvent.get(networkRequestId); if (requestPausedEvent) { - const interceptionId = requestPausedEvent.requestId; - this._onRequest(event, interceptionId); - this._requestIdToRequestPausedEvent.delete(requestId); + const { requestId: fetchRequestId } = requestPausedEvent; + this._onRequest(event, fetchRequestId); + this._networkRequestIdToRequestPausedEvent.delete(networkRequestId); } return; @@ -369,10 +376,10 @@ export class NetworkManager extends EventEmitter { } _onRequestPaused(event: Protocol.Fetch.RequestPausedEvent): void { - const requestId = event.networkId; - const interceptionId = event.requestId; + // console.log(`Fetch.requestPaused`, { event }); + const { networkId: networkRequestId, requestId: fetchRequestId } = event; - if (!requestId) { + if (!networkRequestId) { return; } @@ -387,7 +394,7 @@ export class NetworkManager extends EventEmitter { */ const requestWillBeSentEvent = (() => { const requestWillBeSentEvent = - this._requestIdToRequestWillBeSentEvent.get(requestId); + this._networkRequestIdToRequestWillBeSentEvent.get(networkRequestId); // redirect requests have the same `requestId`, if ( @@ -395,36 +402,36 @@ export class NetworkManager extends EventEmitter { (requestWillBeSentEvent.request.url !== event.request.url || requestWillBeSentEvent.request.method !== event.request.method) ) { - this._requestIdToRequestWillBeSentEvent.delete(requestId); + this._networkRequestIdToRequestWillBeSentEvent.delete(networkRequestId); return; } return requestWillBeSentEvent; })(); if (requestWillBeSentEvent) { - this._onRequest(requestWillBeSentEvent, interceptionId); + this._onRequest(requestWillBeSentEvent, fetchRequestId); } else { - this._requestIdToRequestPausedEvent.set(requestId, event); + this._networkRequestIdToRequestPausedEvent.set(networkRequestId, event); } } - _requestIdToQueuedRedirectInfo(requestId: string): Array<{ + _networkRequestIdToQueuedRedirectInfo(requestId: string): Array<{ event: Protocol.Network.RequestWillBeSentEvent; interceptionId?: string; }> { - if (!this._requestIdToQueuedRedirectInfoMap.has(requestId)) { - this._requestIdToQueuedRedirectInfoMap.set(requestId, []); + if (!this._networkRequestIdToQueuedRedirectInfoMap.has(requestId)) { + this._networkRequestIdToQueuedRedirectInfoMap.set(requestId, []); } - return this._requestIdToQueuedRedirectInfoMap.get(requestId); + return this._networkRequestIdToQueuedRedirectInfoMap.get(requestId); } - _requestIdToResponseExtraInfo( + _networkRequestIdToResponseExtraInfo( requestId: string ): Protocol.Network.ResponseReceivedExtraInfoEvent[] { - if (!this._requestIdToResponseReceivedExtraInfo.has(requestId)) { - this._requestIdToResponseReceivedExtraInfo.set(requestId, []); + if (!this._networkRequestIdToResponseReceivedExtraInfo.has(requestId)) { + this._networkRequestIdToResponseReceivedExtraInfo.set(requestId, []); } - return this._requestIdToResponseReceivedExtraInfo.get(requestId); + return this._networkRequestIdToResponseReceivedExtraInfo.get(requestId); } _onRequest( @@ -442,11 +449,11 @@ export class NetworkManager extends EventEmitter { // response/requestfinished. let redirectResponseExtraInfo = null; if (event.redirectHasExtraInfo) { - redirectResponseExtraInfo = this._requestIdToResponseExtraInfo( + redirectResponseExtraInfo = this._networkRequestIdToResponseExtraInfo( event.requestId ).shift(); if (!redirectResponseExtraInfo) { - this._requestIdToQueuedRedirectInfo(event.requestId).push({ + this._networkRequestIdToQueuedRedirectInfo(event.requestId).push({ event, interceptionId, }); @@ -519,7 +526,7 @@ export class NetworkManager extends EventEmitter { // FileUpload sends a response without a matching request. if (!request) return; - const extraInfos = this._requestIdToResponseExtraInfo( + const extraInfos = this._networkRequestIdToResponseExtraInfo( responseReceived.requestId ); if (extraInfos.length) { @@ -542,12 +549,14 @@ export class NetworkManager extends EventEmitter { const request = this._requestIdToRequest.get(event.requestId); let extraInfo = null; if (request && !request._fromMemoryCache && event.hasExtraInfo) { - extraInfo = this._requestIdToResponseExtraInfo(event.requestId).shift(); + extraInfo = this._networkRequestIdToResponseExtraInfo( + event.requestId + ).shift(); if (!extraInfo) { // Wait until we get the corresponding ExtraInfo event. let resolver = null; const promise = new Promise((resolve) => (resolver = resolve)); - this._requestIdToQueuedEvents.set(event.requestId, { + this._networkRequestIdToQueuedEvents.set(event.requestId, { responseReceived: event, promise, resolver, @@ -559,7 +568,8 @@ export class NetworkManager extends EventEmitter { } responseWaitingForExtraInfoPromise(requestId: string): Promise { - const responseReceived = this._requestIdToQueuedEvents.get(requestId); + const responseReceived = + this._networkRequestIdToQueuedEvents.get(requestId); if (!responseReceived) return Promise.resolve(); return responseReceived.promise; } @@ -570,18 +580,20 @@ export class NetworkManager extends EventEmitter { // We may have skipped a redirect response/request pair due to waiting for // this ExtraInfo event. If so, continue that work now that we have the // request. - const redirectInfo = this._requestIdToQueuedRedirectInfo( + const redirectInfo = this._networkRequestIdToQueuedRedirectInfo( event.requestId ).shift(); if (redirectInfo) { - this._requestIdToResponseExtraInfo(event.requestId).push(event); + this._networkRequestIdToResponseExtraInfo(event.requestId).push(event); this._onRequest(redirectInfo.event, redirectInfo.interceptionId); return; } // We may have skipped response and loading events because we didn't have // this ExtraInfo event yet. If so, emit those events now. - const queuedEvents = this._requestIdToQueuedEvents.get(event.requestId); + const queuedEvents = this._networkRequestIdToQueuedEvents.get( + event.requestId + ); if (queuedEvents) { this._emitResponseEvent(queuedEvents.responseReceived, event); if (queuedEvents.loadingFinished) { @@ -595,7 +607,7 @@ export class NetworkManager extends EventEmitter { } // Wait until we get another event that can use this ExtraInfo event. - this._requestIdToResponseExtraInfo(event.requestId).push(event); + this._networkRequestIdToResponseExtraInfo(event.requestId).push(event); } _forgetRequest(request: HTTPRequest, events: boolean): void { @@ -606,18 +618,20 @@ export class NetworkManager extends EventEmitter { this._attemptedAuthentications.delete(interceptionId); if (events) { - this._requestIdToRequestWillBeSentEvent.delete(requestId); - this._requestIdToRequestPausedEvent.delete(requestId); - this._requestIdToQueuedEvents.delete(requestId); - this._requestIdToQueuedRedirectInfoMap.delete(requestId); - this._requestIdToResponseReceivedExtraInfo.delete(requestId); + this._networkRequestIdToRequestWillBeSentEvent.delete(requestId); + this._networkRequestIdToRequestPausedEvent.delete(requestId); + this._networkRequestIdToQueuedEvents.delete(requestId); + this._networkRequestIdToQueuedRedirectInfoMap.delete(requestId); + this._networkRequestIdToResponseReceivedExtraInfo.delete(requestId); } } _onLoadingFinished(event: Protocol.Network.LoadingFinishedEvent): void { // If the response event for this request is still waiting on a // corresponding ExtraInfo event, then wait to emit this event too. - const queuedEvents = this._requestIdToQueuedEvents.get(event.requestId); + const queuedEvents = this._networkRequestIdToQueuedEvents.get( + event.requestId + ); if (queuedEvents) { queuedEvents.loadingFinished = event; } else { @@ -641,7 +655,9 @@ export class NetworkManager extends EventEmitter { _onLoadingFailed(event: Protocol.Network.LoadingFailedEvent): void { // If the response event for this request is still waiting on a // corresponding ExtraInfo event, then wait to emit this event too. - const queuedEvents = this._requestIdToQueuedEvents.get(event.requestId); + const queuedEvents = this._networkRequestIdToQueuedEvents.get( + event.requestId + ); if (queuedEvents) { queuedEvents.loadingFailed = event; } else { From f5842634bd78eda9be8649a56ae1e7f4d00e84ea Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Fri, 26 Nov 2021 12:19:53 -0800 Subject: [PATCH 04/10] chore: testing --- src/common/NetworkManager.ts | 11 ++++ test/NetworkManager.spec.ts | 97 ++++++++++++++++++++++++++++++++++-- 2 files changed, 103 insertions(+), 5 deletions(-) diff --git a/src/common/NetworkManager.ts b/src/common/NetworkManager.ts index 786746072b50d..e98bc4cf388be 100644 --- a/src/common/NetworkManager.ts +++ b/src/common/NetworkManager.ts @@ -377,6 +377,17 @@ export class NetworkManager extends EventEmitter { _onRequestPaused(event: Protocol.Fetch.RequestPausedEvent): void { // console.log(`Fetch.requestPaused`, { event }); + if ( + !this._userRequestInterceptionEnabled && + this._protocolRequestInterceptionEnabled + ) { + this._client + .send('Fetch.continueRequest', { + requestId: event.requestId, + }) + .catch(debugError); + } + const { networkId: networkRequestId, requestId: fetchRequestId } = event; if (!networkRequestId) { diff --git a/test/NetworkManager.spec.ts b/test/NetworkManager.spec.ts index a934c80a25221..15af01346d7f2 100644 --- a/test/NetworkManager.spec.ts +++ b/test/NetworkManager.spec.ts @@ -16,16 +16,21 @@ import { describeChromeOnly } from './mocha-utils'; // eslint-disable-line import/extensions -import { NetworkManager } from '../lib/cjs/puppeteer/common/NetworkManager.js'; +import expect from 'expect'; +import { + NetworkManager, + NetworkManagerEmittedEvents, +} from '../lib/cjs/puppeteer/common/NetworkManager.js'; +import { HTTPRequest } from '../lib/cjs/puppeteer/common/HTTPRequest.js'; import { EventEmitter } from '../lib/cjs/puppeteer/common/EventEmitter.js'; import { Frame } from '../lib/cjs/puppeteer/common/FrameManager.js'; +class MockCDPSession extends EventEmitter { + async send(): Promise {} +} + describeChromeOnly('NetworkManager', () => { it('should process extra info on multiple redirects', async () => { - class MockCDPSession extends EventEmitter { - send(): any {} - } - const mockCDPSession = new MockCDPSession(); new NetworkManager(mockCDPSession, true, { frame(): Frame | null { @@ -456,4 +461,86 @@ describeChromeOnly('NetworkManager', () => { frameId: '099A5216AF03AAFEC988F214B024DF08', }); }); + it(`should handle multiple Fetch.requestPaused events for the same Network.requestWillBeSent event`, async () => { + const mockCDPSession = new MockCDPSession(); + const manager = new NetworkManager(mockCDPSession, true, { + frame(): Frame | null { + return null; + }, + }); + manager.setRequestInterception(true); + + const requests: HTTPRequest[] = []; + manager.on(NetworkManagerEmittedEvents.Request, (request: HTTPRequest) => { + request.continue(); + requests.push(request); + }); + + /** + * This sequence was taken from an actual CDP session produced by the following + * test script: + * + * const browser = await puppeteer.launch({ headless: false }); + * const page = await browser.newPage(); + * await page.setCacheEnabled(false); + * + * await page.setRequestInterception(true) + * page.on('request', (interceptedRequest) => { + * interceptedRequest.continue(); + * }); + * + * await page.goto('https://www.google.com'); + * await browser.close(); + * + */ + mockCDPSession.emit('Network.requestWillBeSent', { + requestId: '11ACE9783588040D644B905E8B55285B', + loaderId: '11ACE9783588040D644B905E8B55285B', + documentURL: 'https://www.google.com/', + request: { + url: 'https://www.google.com/', + method: 'GET', + headers: [Object], + mixedContentType: 'none', + initialPriority: 'VeryHigh', + referrerPolicy: 'strict-origin-when-cross-origin', + isSameSite: true, + }, + timestamp: 224604.980827, + wallTime: 1637955746.786191, + initiator: { type: 'other' }, + redirectHasExtraInfo: false, + type: 'Document', + frameId: '84AC261A351B86932B775B76D1DD79F8', + hasUserGesture: false, + }); + mockCDPSession.emit('Fetch.requestPaused', { + requestId: 'interception-job-1.0', + request: { + url: 'https://www.google.com/', + method: 'GET', + headers: [Object], + initialPriority: 'VeryHigh', + referrerPolicy: 'strict-origin-when-cross-origin', + }, + frameId: '84AC261A351B86932B775B76D1DD79F8', + resourceType: 'Document', + networkId: '11ACE9783588040D644B905E8B55285B', + }); + mockCDPSession.emit('Fetch.requestPaused', { + requestId: 'interception-job-2.0', + request: { + url: 'https://www.google.com/', + method: 'GET', + headers: [Object], + initialPriority: 'VeryHigh', + referrerPolicy: 'strict-origin-when-cross-origin', + }, + frameId: '84AC261A351B86932B775B76D1DD79F8', + resourceType: 'Document', + networkId: '11ACE9783588040D644B905E8B55285B', + }); + + expect(requests.length).toBe(2); + }); }); From 32fa56239a7b0373f678dad40103562256f065de Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Fri, 26 Nov 2021 12:28:02 -0800 Subject: [PATCH 05/10] chore: comment update --- src/common/NetworkManager.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/common/NetworkManager.ts b/src/common/NetworkManager.ts index e98bc4cf388be..7323502a0f399 100644 --- a/src/common/NetworkManager.ts +++ b/src/common/NetworkManager.ts @@ -87,7 +87,8 @@ export class NetworkManager extends EventEmitter { * A. `_onRequestWillBeSent` * B. `_onRequestWillBeSent`, `_onRequestPaused` * C. `_onRequestPaused`, `_onRequestWillBeSent` - * D. `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused` + * D. `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`, + * `_onRequestWillBeSent`, `_onRequestPaused`, `_onRequestPaused` * (see crbug.com/1196004) * * For `_onRequest` we need the event from `_onRequestWillBeSent` and @@ -109,7 +110,8 @@ export class NetworkManager extends EventEmitter { * C. `_onRequestWillBeSent`, `_onRequestPaused`, * `_onRequestPaused`, `_onRequestWillBeSent`, ... * D. `_onRequestPaused`, `_onRequestWillBeSent`, - * `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`, ... + * `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`, + * `_onRequestWillBeSent`, `_onRequestPaused`, `_onRequestPaused`, ... * (see crbug.com/1196004) */ _networkRequestIdToRequestWillBeSentEvent = new Map< From 81cc74c52cf887f4da00155bfe61f45ddf8a3fba Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Fri, 26 Nov 2021 13:37:45 -0800 Subject: [PATCH 06/10] chore: refactor --- src/common/NetworkManager.ts | 134 ++++++++++++++++++----------------- 1 file changed, 70 insertions(+), 64 deletions(-) diff --git a/src/common/NetworkManager.ts b/src/common/NetworkManager.ts index 7323502a0f399..cadd4dcf40d73 100644 --- a/src/common/NetworkManager.ts +++ b/src/common/NetworkManager.ts @@ -74,14 +74,25 @@ interface FrameManager { frame(frameId: string): Frame | null; } +type QueuedEvents = { + responseReceived: Protocol.Network.ResponseReceivedEvent; + promise: Promise; + resolver: () => void; + loadingFinished?: Protocol.Network.LoadingFinishedEvent; + loadingFailed?: Protocol.Network.LoadingFailedEvent; +}; + +type RedirectInfoMap = Array<{ + event: Protocol.Network.RequestWillBeSentEvent; + interceptionId?: string; +}>; + /** * @internal + * + * Helper class to track network events by request ID */ -export class NetworkManager extends EventEmitter { - _client: CDPSession; - _ignoreHTTPSErrors: boolean; - _frameManager: FrameManager; - +class NetworkEventManager { /* * There are four possible orders of events: * A. `_onRequestWillBeSent` @@ -114,15 +125,12 @@ export class NetworkManager extends EventEmitter { * `_onRequestWillBeSent`, `_onRequestPaused`, `_onRequestPaused`, ... * (see crbug.com/1196004) */ - _networkRequestIdToRequestWillBeSentEvent = new Map< + requestWillBeSent = new Map< string, Protocol.Network.RequestWillBeSentEvent >(); - _networkRequestIdToRequestPausedEvent = new Map< - string, - Protocol.Fetch.RequestPausedEvent - >(); - _requestIdToRequest = new Map(); + requestPaused = new Map(); + httpRequest = new Map(); /* * The below maps are used to reconcile Network.responseReceivedExtraInfo @@ -133,27 +141,31 @@ export class NetworkManager extends EventEmitter { * handle redirects, we have to make them Arrays to represent the chain of * events. */ - _networkRequestIdToResponseReceivedExtraInfo = new Map< + responseReceivedExtraInfo = new Map< string, Protocol.Network.ResponseReceivedExtraInfoEvent[] >(); - _networkRequestIdToQueuedRedirectInfoMap = new Map< - string, - Array<{ - event: Protocol.Network.RequestWillBeSentEvent; - interceptionId?: string; - }> - >(); - _networkRequestIdToQueuedEvents = new Map< - string, - { - responseReceived: Protocol.Network.ResponseReceivedEvent; - promise: Promise; - resolver: () => void; - loadingFinished?: Protocol.Network.LoadingFinishedEvent; - loadingFailed?: Protocol.Network.LoadingFailedEvent; - } - >(); + queuedRedirectInfoMap = new Map(); + queuedEvents = new Map(); + + forget(requestId: string) { + this.requestWillBeSent.delete(requestId); + this.requestPaused.delete(requestId); + this.queuedEvents.delete(requestId); + this.queuedRedirectInfoMap.delete(requestId); + this.responseReceivedExtraInfo.delete(requestId); + } +} + +/** + * @internal + */ +export class NetworkManager extends EventEmitter { + _client: CDPSession; + _ignoreHTTPSErrors: boolean; + _frameManager: FrameManager; + + _networkEventMap = new NetworkEventManager(); _extraHTTPHeaders: Record = {}; _credentials?: Credentials = null; @@ -238,7 +250,7 @@ export class NetworkManager extends EventEmitter { } numRequestsInProgress(): number { - return [...this._requestIdToRequest].filter(([, request]) => { + return [...this._networkEventMap.httpRequest].filter(([, request]) => { return !request.response(); }).length; } @@ -332,20 +344,17 @@ export class NetworkManager extends EventEmitter { ) { const { requestId: networkRequestId } = event; - this._networkRequestIdToRequestWillBeSentEvent.set( - networkRequestId, - event - ); + this._networkEventMap.requestWillBeSent.set(networkRequestId, event); /** * CDP may have sent a Fetch.requestPaused event already. Check for it. */ const requestPausedEvent = - this._networkRequestIdToRequestPausedEvent.get(networkRequestId); + this._networkEventMap.requestPaused.get(networkRequestId); if (requestPausedEvent) { const { requestId: fetchRequestId } = requestPausedEvent; this._onRequest(event, fetchRequestId); - this._networkRequestIdToRequestPausedEvent.delete(networkRequestId); + this._networkEventMap.requestPaused.delete(networkRequestId); } return; @@ -407,7 +416,7 @@ export class NetworkManager extends EventEmitter { */ const requestWillBeSentEvent = (() => { const requestWillBeSentEvent = - this._networkRequestIdToRequestWillBeSentEvent.get(networkRequestId); + this._networkEventMap.requestWillBeSent.get(networkRequestId); // redirect requests have the same `requestId`, if ( @@ -415,7 +424,7 @@ export class NetworkManager extends EventEmitter { (requestWillBeSentEvent.request.url !== event.request.url || requestWillBeSentEvent.request.method !== event.request.method) ) { - this._networkRequestIdToRequestWillBeSentEvent.delete(networkRequestId); + this._networkEventMap.requestWillBeSent.delete(networkRequestId); return; } return requestWillBeSentEvent; @@ -424,7 +433,7 @@ export class NetworkManager extends EventEmitter { if (requestWillBeSentEvent) { this._onRequest(requestWillBeSentEvent, fetchRequestId); } else { - this._networkRequestIdToRequestPausedEvent.set(networkRequestId, event); + this._networkEventMap.requestPaused.set(networkRequestId, event); } } @@ -432,19 +441,19 @@ export class NetworkManager extends EventEmitter { event: Protocol.Network.RequestWillBeSentEvent; interceptionId?: string; }> { - if (!this._networkRequestIdToQueuedRedirectInfoMap.has(requestId)) { - this._networkRequestIdToQueuedRedirectInfoMap.set(requestId, []); + if (!this._networkEventMap.queuedRedirectInfoMap.has(requestId)) { + this._networkEventMap.queuedRedirectInfoMap.set(requestId, []); } - return this._networkRequestIdToQueuedRedirectInfoMap.get(requestId); + return this._networkEventMap.queuedRedirectInfoMap.get(requestId); } _networkRequestIdToResponseExtraInfo( requestId: string ): Protocol.Network.ResponseReceivedExtraInfoEvent[] { - if (!this._networkRequestIdToResponseReceivedExtraInfo.has(requestId)) { - this._networkRequestIdToResponseReceivedExtraInfo.set(requestId, []); + if (!this._networkEventMap.responseReceivedExtraInfo.has(requestId)) { + this._networkEventMap.responseReceivedExtraInfo.set(requestId, []); } - return this._networkRequestIdToResponseReceivedExtraInfo.get(requestId); + return this._networkEventMap.responseReceivedExtraInfo.get(requestId); } _onRequest( @@ -474,7 +483,7 @@ export class NetworkManager extends EventEmitter { } } - const request = this._requestIdToRequest.get(event.requestId); + const request = this._networkEventMap.httpRequest.get(event.requestId); // If we connect late to the target, we could have missed the // requestWillBeSent event. if (request) { @@ -497,7 +506,7 @@ export class NetworkManager extends EventEmitter { event, redirectChain ); - this._requestIdToRequest.set(event.requestId, request); + this._networkEventMap.httpRequest.set(event.requestId, request); this.emit(NetworkManagerEmittedEvents.Request, request); request.finalizeInterceptions(); } @@ -505,7 +514,7 @@ export class NetworkManager extends EventEmitter { _onRequestServedFromCache( event: Protocol.Network.RequestServedFromCacheEvent ): void { - const request = this._requestIdToRequest.get(event.requestId); + const request = this._networkEventMap.httpRequest.get(event.requestId); if (request) request._fromMemoryCache = true; this.emit(NetworkManagerEmittedEvents.RequestServedFromCache, request); } @@ -535,7 +544,9 @@ export class NetworkManager extends EventEmitter { responseReceived: Protocol.Network.ResponseReceivedEvent, extraInfo: Protocol.Network.ResponseReceivedExtraInfoEvent ): void { - const request = this._requestIdToRequest.get(responseReceived.requestId); + const request = this._networkEventMap.httpRequest.get( + responseReceived.requestId + ); // FileUpload sends a response without a matching request. if (!request) return; @@ -559,7 +570,7 @@ export class NetworkManager extends EventEmitter { } _onResponseReceived(event: Protocol.Network.ResponseReceivedEvent): void { - const request = this._requestIdToRequest.get(event.requestId); + const request = this._networkEventMap.httpRequest.get(event.requestId); let extraInfo = null; if (request && !request._fromMemoryCache && event.hasExtraInfo) { extraInfo = this._networkRequestIdToResponseExtraInfo( @@ -569,7 +580,7 @@ export class NetworkManager extends EventEmitter { // Wait until we get the corresponding ExtraInfo event. let resolver = null; const promise = new Promise((resolve) => (resolver = resolve)); - this._networkRequestIdToQueuedEvents.set(event.requestId, { + this._networkEventMap.queuedEvents.set(event.requestId, { responseReceived: event, promise, resolver, @@ -581,8 +592,7 @@ export class NetworkManager extends EventEmitter { } responseWaitingForExtraInfoPromise(requestId: string): Promise { - const responseReceived = - this._networkRequestIdToQueuedEvents.get(requestId); + const responseReceived = this._networkEventMap.queuedEvents.get(requestId); if (!responseReceived) return Promise.resolve(); return responseReceived.promise; } @@ -604,7 +614,7 @@ export class NetworkManager extends EventEmitter { // We may have skipped response and loading events because we didn't have // this ExtraInfo event yet. If so, emit those events now. - const queuedEvents = this._networkRequestIdToQueuedEvents.get( + const queuedEvents = this._networkEventMap.queuedEvents.get( event.requestId ); if (queuedEvents) { @@ -627,22 +637,18 @@ export class NetworkManager extends EventEmitter { const requestId = request._requestId; const interceptionId = request._interceptionId; - this._requestIdToRequest.delete(requestId); + this._networkEventMap.httpRequest.delete(requestId); this._attemptedAuthentications.delete(interceptionId); if (events) { - this._networkRequestIdToRequestWillBeSentEvent.delete(requestId); - this._networkRequestIdToRequestPausedEvent.delete(requestId); - this._networkRequestIdToQueuedEvents.delete(requestId); - this._networkRequestIdToQueuedRedirectInfoMap.delete(requestId); - this._networkRequestIdToResponseReceivedExtraInfo.delete(requestId); + this._networkEventMap.forget(requestId); } } _onLoadingFinished(event: Protocol.Network.LoadingFinishedEvent): void { // If the response event for this request is still waiting on a // corresponding ExtraInfo event, then wait to emit this event too. - const queuedEvents = this._networkRequestIdToQueuedEvents.get( + const queuedEvents = this._networkEventMap.queuedEvents.get( event.requestId ); if (queuedEvents) { @@ -653,7 +659,7 @@ export class NetworkManager extends EventEmitter { } _emitLoadingFinished(event: Protocol.Network.LoadingFinishedEvent): void { - const request = this._requestIdToRequest.get(event.requestId); + const request = this._networkEventMap.httpRequest.get(event.requestId); // For certain requestIds we never receive requestWillBeSent event. // @see https://crbug.com/750469 if (!request) return; @@ -668,7 +674,7 @@ export class NetworkManager extends EventEmitter { _onLoadingFailed(event: Protocol.Network.LoadingFailedEvent): void { // If the response event for this request is still waiting on a // corresponding ExtraInfo event, then wait to emit this event too. - const queuedEvents = this._networkRequestIdToQueuedEvents.get( + const queuedEvents = this._networkEventMap.queuedEvents.get( event.requestId ); if (queuedEvents) { @@ -679,7 +685,7 @@ export class NetworkManager extends EventEmitter { } _emitLoadingFailed(event: Protocol.Network.LoadingFailedEvent): void { - const request = this._requestIdToRequest.get(event.requestId); + const request = this._networkEventMap.httpRequest.get(event.requestId); // For certain requestIds we never receive requestWillBeSent event. // @see https://crbug.com/750469 if (!request) return; From bfa1c7f1d483c1518a7764d9fcbdaae258708103 Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Fri, 26 Nov 2021 13:42:07 -0800 Subject: [PATCH 07/10] chore: refactor --- src/common/NetworkEventManager.ts | 83 +++++++++++++++ src/common/NetworkManager.ts | 171 ++++++++++-------------------- 2 files changed, 139 insertions(+), 115 deletions(-) create mode 100644 src/common/NetworkEventManager.ts diff --git a/src/common/NetworkEventManager.ts b/src/common/NetworkEventManager.ts new file mode 100644 index 0000000000000..8a12857b21246 --- /dev/null +++ b/src/common/NetworkEventManager.ts @@ -0,0 +1,83 @@ +import { Protocol } from 'devtools-protocol'; +import { HTTPRequest } from './HTTPRequest.js'; + +type QueuedEvents = { + responseReceived: Protocol.Network.ResponseReceivedEvent; + promise: Promise; + resolver: () => void; + loadingFinished?: Protocol.Network.LoadingFinishedEvent; + loadingFailed?: Protocol.Network.LoadingFailedEvent; +}; +type RedirectInfoMap = Array<{ + event: Protocol.Network.RequestWillBeSentEvent; + interceptionId?: string; +}>; +/** + * @internal + * + * Helper class to track network events by request ID + */ +export class NetworkEventManager { + /* + * There are four possible orders of events: + * A. `_onRequestWillBeSent` + * B. `_onRequestWillBeSent`, `_onRequestPaused` + * C. `_onRequestPaused`, `_onRequestWillBeSent` + * D. `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`, + * `_onRequestWillBeSent`, `_onRequestPaused`, `_onRequestPaused` + * (see crbug.com/1196004) + * + * For `_onRequest` we need the event from `_onRequestWillBeSent` and + * optionally the `interceptionId` from `_onRequestPaused`. + * + * If request interception is disabled, call `_onRequest` once per call to + * `_onRequestWillBeSent`. + * If request interception is enabled, call `_onRequest` once per call to + * `_onRequestPaused` (once per `interceptionId`). + * + * Events are stored to allow for subsequent events to call `_onRequest`. + * + * Note that (chains of) redirect requests have the same `requestId` (!) as + * the original request. We have to anticipate series of events like these: + * A. `_onRequestWillBeSent`, + * `_onRequestWillBeSent`, ... + * B. `_onRequestWillBeSent`, `_onRequestPaused`, + * `_onRequestWillBeSent`, `_onRequestPaused`, ... + * C. `_onRequestWillBeSent`, `_onRequestPaused`, + * `_onRequestPaused`, `_onRequestWillBeSent`, ... + * D. `_onRequestPaused`, `_onRequestWillBeSent`, + * `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`, + * `_onRequestWillBeSent`, `_onRequestPaused`, `_onRequestPaused`, ... + * (see crbug.com/1196004) + */ + requestWillBeSent = new Map< + string, + Protocol.Network.RequestWillBeSentEvent + >(); + requestPaused = new Map(); + httpRequest = new Map(); + + /* + * The below maps are used to reconcile Network.responseReceivedExtraInfo + * events with their corresponding request. Each response and redirect + * response gets an ExtraInfo event, and we don't know which will come first. + * This means that we have to store a Response or an ExtraInfo for each + * response, and emit the event when we get both of them. In addition, to + * handle redirects, we have to make them Arrays to represent the chain of + * events. + */ + responseReceivedExtraInfo = new Map< + string, + Protocol.Network.ResponseReceivedExtraInfoEvent[] + >(); + queuedRedirectInfoMap = new Map(); + queuedEvents = new Map(); + + forget(requestId: string): void { + this.requestWillBeSent.delete(requestId); + this.requestPaused.delete(requestId); + this.queuedEvents.delete(requestId); + this.queuedRedirectInfoMap.delete(requestId); + this.responseReceivedExtraInfo.delete(requestId); + } +} diff --git a/src/common/NetworkManager.ts b/src/common/NetworkManager.ts index cadd4dcf40d73..be9af9d19c64b 100644 --- a/src/common/NetworkManager.ts +++ b/src/common/NetworkManager.ts @@ -22,6 +22,7 @@ import { helper, debugError } from './helper.js'; import { Protocol } from 'devtools-protocol'; import { HTTPRequest } from './HTTPRequest.js'; import { HTTPResponse } from './HTTPResponse.js'; +import { NetworkEventManager } from './NetworkEventManager.js'; /** * @public @@ -74,89 +75,6 @@ interface FrameManager { frame(frameId: string): Frame | null; } -type QueuedEvents = { - responseReceived: Protocol.Network.ResponseReceivedEvent; - promise: Promise; - resolver: () => void; - loadingFinished?: Protocol.Network.LoadingFinishedEvent; - loadingFailed?: Protocol.Network.LoadingFailedEvent; -}; - -type RedirectInfoMap = Array<{ - event: Protocol.Network.RequestWillBeSentEvent; - interceptionId?: string; -}>; - -/** - * @internal - * - * Helper class to track network events by request ID - */ -class NetworkEventManager { - /* - * There are four possible orders of events: - * A. `_onRequestWillBeSent` - * B. `_onRequestWillBeSent`, `_onRequestPaused` - * C. `_onRequestPaused`, `_onRequestWillBeSent` - * D. `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`, - * `_onRequestWillBeSent`, `_onRequestPaused`, `_onRequestPaused` - * (see crbug.com/1196004) - * - * For `_onRequest` we need the event from `_onRequestWillBeSent` and - * optionally the `interceptionId` from `_onRequestPaused`. - * - * If request interception is disabled, call `_onRequest` once per call to - * `_onRequestWillBeSent`. - * If request interception is enabled, call `_onRequest` once per call to - * `_onRequestPaused` (once per `interceptionId`). - * - * Events are stored to allow for subsequent events to call `_onRequest`. - * - * Note that (chains of) redirect requests have the same `requestId` (!) as - * the original request. We have to anticipate series of events like these: - * A. `_onRequestWillBeSent`, - * `_onRequestWillBeSent`, ... - * B. `_onRequestWillBeSent`, `_onRequestPaused`, - * `_onRequestWillBeSent`, `_onRequestPaused`, ... - * C. `_onRequestWillBeSent`, `_onRequestPaused`, - * `_onRequestPaused`, `_onRequestWillBeSent`, ... - * D. `_onRequestPaused`, `_onRequestWillBeSent`, - * `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`, - * `_onRequestWillBeSent`, `_onRequestPaused`, `_onRequestPaused`, ... - * (see crbug.com/1196004) - */ - requestWillBeSent = new Map< - string, - Protocol.Network.RequestWillBeSentEvent - >(); - requestPaused = new Map(); - httpRequest = new Map(); - - /* - * The below maps are used to reconcile Network.responseReceivedExtraInfo - * events with their corresponding request. Each response and redirect - * response gets an ExtraInfo event, and we don't know which will come first. - * This means that we have to store a Response or an ExtraInfo for each - * response, and emit the event when we get both of them. In addition, to - * handle redirects, we have to make them Arrays to represent the chain of - * events. - */ - responseReceivedExtraInfo = new Map< - string, - Protocol.Network.ResponseReceivedExtraInfoEvent[] - >(); - queuedRedirectInfoMap = new Map(); - queuedEvents = new Map(); - - forget(requestId: string) { - this.requestWillBeSent.delete(requestId); - this.requestPaused.delete(requestId); - this.queuedEvents.delete(requestId); - this.queuedRedirectInfoMap.delete(requestId); - this.responseReceivedExtraInfo.delete(requestId); - } -} - /** * @internal */ @@ -165,7 +83,7 @@ export class NetworkManager extends EventEmitter { _ignoreHTTPSErrors: boolean; _frameManager: FrameManager; - _networkEventMap = new NetworkEventManager(); + _networkRequestIdEventMap = new NetworkEventManager(); _extraHTTPHeaders: Record = {}; _credentials?: Credentials = null; @@ -250,9 +168,11 @@ export class NetworkManager extends EventEmitter { } numRequestsInProgress(): number { - return [...this._networkEventMap.httpRequest].filter(([, request]) => { - return !request.response(); - }).length; + return [...this._networkRequestIdEventMap.httpRequest].filter( + ([, request]) => { + return !request.response(); + } + ).length; } async setOfflineMode(value: boolean): Promise { @@ -336,7 +256,6 @@ export class NetworkManager extends EventEmitter { } _onRequestWillBeSent(event: Protocol.Network.RequestWillBeSentEvent): void { - // console.log(`Network.requestWillBeSent`, { event }); // Request interception doesn't happen for data URLs with Network Service. if ( this._userRequestInterceptionEnabled && @@ -344,17 +263,20 @@ export class NetworkManager extends EventEmitter { ) { const { requestId: networkRequestId } = event; - this._networkEventMap.requestWillBeSent.set(networkRequestId, event); + this._networkRequestIdEventMap.requestWillBeSent.set( + networkRequestId, + event + ); /** * CDP may have sent a Fetch.requestPaused event already. Check for it. */ const requestPausedEvent = - this._networkEventMap.requestPaused.get(networkRequestId); + this._networkRequestIdEventMap.requestPaused.get(networkRequestId); if (requestPausedEvent) { const { requestId: fetchRequestId } = requestPausedEvent; this._onRequest(event, fetchRequestId); - this._networkEventMap.requestPaused.delete(networkRequestId); + this._networkRequestIdEventMap.requestPaused.delete(networkRequestId); } return; @@ -387,7 +309,6 @@ export class NetworkManager extends EventEmitter { } _onRequestPaused(event: Protocol.Fetch.RequestPausedEvent): void { - // console.log(`Fetch.requestPaused`, { event }); if ( !this._userRequestInterceptionEnabled && this._protocolRequestInterceptionEnabled @@ -416,7 +337,7 @@ export class NetworkManager extends EventEmitter { */ const requestWillBeSentEvent = (() => { const requestWillBeSentEvent = - this._networkEventMap.requestWillBeSent.get(networkRequestId); + this._networkRequestIdEventMap.requestWillBeSent.get(networkRequestId); // redirect requests have the same `requestId`, if ( @@ -424,7 +345,9 @@ export class NetworkManager extends EventEmitter { (requestWillBeSentEvent.request.url !== event.request.url || requestWillBeSentEvent.request.method !== event.request.method) ) { - this._networkEventMap.requestWillBeSent.delete(networkRequestId); + this._networkRequestIdEventMap.requestWillBeSent.delete( + networkRequestId + ); return; } return requestWillBeSentEvent; @@ -433,7 +356,7 @@ export class NetworkManager extends EventEmitter { if (requestWillBeSentEvent) { this._onRequest(requestWillBeSentEvent, fetchRequestId); } else { - this._networkEventMap.requestPaused.set(networkRequestId, event); + this._networkRequestIdEventMap.requestPaused.set(networkRequestId, event); } } @@ -441,19 +364,26 @@ export class NetworkManager extends EventEmitter { event: Protocol.Network.RequestWillBeSentEvent; interceptionId?: string; }> { - if (!this._networkEventMap.queuedRedirectInfoMap.has(requestId)) { - this._networkEventMap.queuedRedirectInfoMap.set(requestId, []); + if (!this._networkRequestIdEventMap.queuedRedirectInfoMap.has(requestId)) { + this._networkRequestIdEventMap.queuedRedirectInfoMap.set(requestId, []); } - return this._networkEventMap.queuedRedirectInfoMap.get(requestId); + return this._networkRequestIdEventMap.queuedRedirectInfoMap.get(requestId); } _networkRequestIdToResponseExtraInfo( requestId: string ): Protocol.Network.ResponseReceivedExtraInfoEvent[] { - if (!this._networkEventMap.responseReceivedExtraInfo.has(requestId)) { - this._networkEventMap.responseReceivedExtraInfo.set(requestId, []); + if ( + !this._networkRequestIdEventMap.responseReceivedExtraInfo.has(requestId) + ) { + this._networkRequestIdEventMap.responseReceivedExtraInfo.set( + requestId, + [] + ); } - return this._networkEventMap.responseReceivedExtraInfo.get(requestId); + return this._networkRequestIdEventMap.responseReceivedExtraInfo.get( + requestId + ); } _onRequest( @@ -483,7 +413,9 @@ export class NetworkManager extends EventEmitter { } } - const request = this._networkEventMap.httpRequest.get(event.requestId); + const request = this._networkRequestIdEventMap.httpRequest.get( + event.requestId + ); // If we connect late to the target, we could have missed the // requestWillBeSent event. if (request) { @@ -506,7 +438,7 @@ export class NetworkManager extends EventEmitter { event, redirectChain ); - this._networkEventMap.httpRequest.set(event.requestId, request); + this._networkRequestIdEventMap.httpRequest.set(event.requestId, request); this.emit(NetworkManagerEmittedEvents.Request, request); request.finalizeInterceptions(); } @@ -514,7 +446,9 @@ export class NetworkManager extends EventEmitter { _onRequestServedFromCache( event: Protocol.Network.RequestServedFromCacheEvent ): void { - const request = this._networkEventMap.httpRequest.get(event.requestId); + const request = this._networkRequestIdEventMap.httpRequest.get( + event.requestId + ); if (request) request._fromMemoryCache = true; this.emit(NetworkManagerEmittedEvents.RequestServedFromCache, request); } @@ -544,7 +478,7 @@ export class NetworkManager extends EventEmitter { responseReceived: Protocol.Network.ResponseReceivedEvent, extraInfo: Protocol.Network.ResponseReceivedExtraInfoEvent ): void { - const request = this._networkEventMap.httpRequest.get( + const request = this._networkRequestIdEventMap.httpRequest.get( responseReceived.requestId ); // FileUpload sends a response without a matching request. @@ -570,7 +504,9 @@ export class NetworkManager extends EventEmitter { } _onResponseReceived(event: Protocol.Network.ResponseReceivedEvent): void { - const request = this._networkEventMap.httpRequest.get(event.requestId); + const request = this._networkRequestIdEventMap.httpRequest.get( + event.requestId + ); let extraInfo = null; if (request && !request._fromMemoryCache && event.hasExtraInfo) { extraInfo = this._networkRequestIdToResponseExtraInfo( @@ -580,7 +516,7 @@ export class NetworkManager extends EventEmitter { // Wait until we get the corresponding ExtraInfo event. let resolver = null; const promise = new Promise((resolve) => (resolver = resolve)); - this._networkEventMap.queuedEvents.set(event.requestId, { + this._networkRequestIdEventMap.queuedEvents.set(event.requestId, { responseReceived: event, promise, resolver, @@ -592,7 +528,8 @@ export class NetworkManager extends EventEmitter { } responseWaitingForExtraInfoPromise(requestId: string): Promise { - const responseReceived = this._networkEventMap.queuedEvents.get(requestId); + const responseReceived = + this._networkRequestIdEventMap.queuedEvents.get(requestId); if (!responseReceived) return Promise.resolve(); return responseReceived.promise; } @@ -614,7 +551,7 @@ export class NetworkManager extends EventEmitter { // We may have skipped response and loading events because we didn't have // this ExtraInfo event yet. If so, emit those events now. - const queuedEvents = this._networkEventMap.queuedEvents.get( + const queuedEvents = this._networkRequestIdEventMap.queuedEvents.get( event.requestId ); if (queuedEvents) { @@ -637,18 +574,18 @@ export class NetworkManager extends EventEmitter { const requestId = request._requestId; const interceptionId = request._interceptionId; - this._networkEventMap.httpRequest.delete(requestId); + this._networkRequestIdEventMap.httpRequest.delete(requestId); this._attemptedAuthentications.delete(interceptionId); if (events) { - this._networkEventMap.forget(requestId); + this._networkRequestIdEventMap.forget(requestId); } } _onLoadingFinished(event: Protocol.Network.LoadingFinishedEvent): void { // If the response event for this request is still waiting on a // corresponding ExtraInfo event, then wait to emit this event too. - const queuedEvents = this._networkEventMap.queuedEvents.get( + const queuedEvents = this._networkRequestIdEventMap.queuedEvents.get( event.requestId ); if (queuedEvents) { @@ -659,7 +596,9 @@ export class NetworkManager extends EventEmitter { } _emitLoadingFinished(event: Protocol.Network.LoadingFinishedEvent): void { - const request = this._networkEventMap.httpRequest.get(event.requestId); + const request = this._networkRequestIdEventMap.httpRequest.get( + event.requestId + ); // For certain requestIds we never receive requestWillBeSent event. // @see https://crbug.com/750469 if (!request) return; @@ -674,7 +613,7 @@ export class NetworkManager extends EventEmitter { _onLoadingFailed(event: Protocol.Network.LoadingFailedEvent): void { // If the response event for this request is still waiting on a // corresponding ExtraInfo event, then wait to emit this event too. - const queuedEvents = this._networkEventMap.queuedEvents.get( + const queuedEvents = this._networkRequestIdEventMap.queuedEvents.get( event.requestId ); if (queuedEvents) { @@ -685,7 +624,9 @@ export class NetworkManager extends EventEmitter { } _emitLoadingFailed(event: Protocol.Network.LoadingFailedEvent): void { - const request = this._networkEventMap.httpRequest.get(event.requestId); + const request = this._networkRequestIdEventMap.httpRequest.get( + event.requestId + ); // For certain requestIds we never receive requestWillBeSent event. // @see https://crbug.com/750469 if (!request) return; From aa001014a8ea73f5e694e2da97d0967ac7d68803 Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Fri, 26 Nov 2021 13:43:43 -0800 Subject: [PATCH 08/10] chore: cleanup --- src/common/NetworkEventManager.ts | 83 ----------- src/common/NetworkManager.ts | 237 ++++++++++++++++-------------- 2 files changed, 130 insertions(+), 190 deletions(-) delete mode 100644 src/common/NetworkEventManager.ts diff --git a/src/common/NetworkEventManager.ts b/src/common/NetworkEventManager.ts deleted file mode 100644 index 8a12857b21246..0000000000000 --- a/src/common/NetworkEventManager.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { Protocol } from 'devtools-protocol'; -import { HTTPRequest } from './HTTPRequest.js'; - -type QueuedEvents = { - responseReceived: Protocol.Network.ResponseReceivedEvent; - promise: Promise; - resolver: () => void; - loadingFinished?: Protocol.Network.LoadingFinishedEvent; - loadingFailed?: Protocol.Network.LoadingFailedEvent; -}; -type RedirectInfoMap = Array<{ - event: Protocol.Network.RequestWillBeSentEvent; - interceptionId?: string; -}>; -/** - * @internal - * - * Helper class to track network events by request ID - */ -export class NetworkEventManager { - /* - * There are four possible orders of events: - * A. `_onRequestWillBeSent` - * B. `_onRequestWillBeSent`, `_onRequestPaused` - * C. `_onRequestPaused`, `_onRequestWillBeSent` - * D. `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`, - * `_onRequestWillBeSent`, `_onRequestPaused`, `_onRequestPaused` - * (see crbug.com/1196004) - * - * For `_onRequest` we need the event from `_onRequestWillBeSent` and - * optionally the `interceptionId` from `_onRequestPaused`. - * - * If request interception is disabled, call `_onRequest` once per call to - * `_onRequestWillBeSent`. - * If request interception is enabled, call `_onRequest` once per call to - * `_onRequestPaused` (once per `interceptionId`). - * - * Events are stored to allow for subsequent events to call `_onRequest`. - * - * Note that (chains of) redirect requests have the same `requestId` (!) as - * the original request. We have to anticipate series of events like these: - * A. `_onRequestWillBeSent`, - * `_onRequestWillBeSent`, ... - * B. `_onRequestWillBeSent`, `_onRequestPaused`, - * `_onRequestWillBeSent`, `_onRequestPaused`, ... - * C. `_onRequestWillBeSent`, `_onRequestPaused`, - * `_onRequestPaused`, `_onRequestWillBeSent`, ... - * D. `_onRequestPaused`, `_onRequestWillBeSent`, - * `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`, - * `_onRequestWillBeSent`, `_onRequestPaused`, `_onRequestPaused`, ... - * (see crbug.com/1196004) - */ - requestWillBeSent = new Map< - string, - Protocol.Network.RequestWillBeSentEvent - >(); - requestPaused = new Map(); - httpRequest = new Map(); - - /* - * The below maps are used to reconcile Network.responseReceivedExtraInfo - * events with their corresponding request. Each response and redirect - * response gets an ExtraInfo event, and we don't know which will come first. - * This means that we have to store a Response or an ExtraInfo for each - * response, and emit the event when we get both of them. In addition, to - * handle redirects, we have to make them Arrays to represent the chain of - * events. - */ - responseReceivedExtraInfo = new Map< - string, - Protocol.Network.ResponseReceivedExtraInfoEvent[] - >(); - queuedRedirectInfoMap = new Map(); - queuedEvents = new Map(); - - forget(requestId: string): void { - this.requestWillBeSent.delete(requestId); - this.requestPaused.delete(requestId); - this.queuedEvents.delete(requestId); - this.queuedRedirectInfoMap.delete(requestId); - this.responseReceivedExtraInfo.delete(requestId); - } -} diff --git a/src/common/NetworkManager.ts b/src/common/NetworkManager.ts index be9af9d19c64b..d569293d98d38 100644 --- a/src/common/NetworkManager.ts +++ b/src/common/NetworkManager.ts @@ -22,7 +22,6 @@ import { helper, debugError } from './helper.js'; import { Protocol } from 'devtools-protocol'; import { HTTPRequest } from './HTTPRequest.js'; import { HTTPResponse } from './HTTPResponse.js'; -import { NetworkEventManager } from './NetworkEventManager.js'; /** * @public @@ -83,7 +82,76 @@ export class NetworkManager extends EventEmitter { _ignoreHTTPSErrors: boolean; _frameManager: FrameManager; - _networkRequestIdEventMap = new NetworkEventManager(); + /* + * There are four possible orders of events: + * A. `_onRequestWillBeSent` + * B. `_onRequestWillBeSent`, `_onRequestPaused` + * C. `_onRequestPaused`, `_onRequestWillBeSent` + * D. `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused` + * (see crbug.com/1196004) + * + * For `_onRequest` we need the event from `_onRequestWillBeSent` and + * optionally the `interceptionId` from `_onRequestPaused`. + * + * If request interception is disabled, call `_onRequest` once per call to + * `_onRequestWillBeSent`. + * If request interception is enabled, call `_onRequest` once per call to + * `_onRequestPaused` (once per `interceptionId`). + * + * Events are stored to allow for subsequent events to call `_onRequest`. + * + * Note that (chains of) redirect requests have the same `requestId` (!) as + * the original request. We have to anticipate series of events like these: + * A. `_onRequestWillBeSent`, + * `_onRequestWillBeSent`, ... + * B. `_onRequestWillBeSent`, `_onRequestPaused`, + * `_onRequestWillBeSent`, `_onRequestPaused`, ... + * C. `_onRequestWillBeSent`, `_onRequestPaused`, + * `_onRequestPaused`, `_onRequestWillBeSent`, ... + * D. `_onRequestPaused`, `_onRequestWillBeSent`, + * `_onRequestPaused`, `_onRequestWillBeSent`, `_onRequestPaused`, ... + * (see crbug.com/1196004) + */ + _requestIdToRequestWillBeSentEvent = new Map< + string, + Protocol.Network.RequestWillBeSentEvent + >(); + _requestIdToRequestPausedEvent = new Map< + string, + Protocol.Fetch.RequestPausedEvent + >(); + _requestIdToRequest = new Map(); + + /* + * The below maps are used to reconcile Network.responseReceivedExtraInfo + * events with their corresponding request. Each response and redirect + * response gets an ExtraInfo event, and we don't know which will come first. + * This means that we have to store a Response or an ExtraInfo for each + * response, and emit the event when we get both of them. In addition, to + * handle redirects, we have to make them Arrays to represent the chain of + * events. + */ + _requestIdToResponseReceivedExtraInfo = new Map< + string, + Protocol.Network.ResponseReceivedExtraInfoEvent[] + >(); + _requestIdToQueuedRedirectInfoMap = new Map< + string, + Array<{ + event: Protocol.Network.RequestWillBeSentEvent; + interceptionId?: string; + }> + >(); + _requestIdToQueuedEvents = new Map< + string, + { + responseReceived: Protocol.Network.ResponseReceivedEvent; + promise: Promise; + resolver: () => void; + loadingFinished?: Protocol.Network.LoadingFinishedEvent; + loadingFailed?: Protocol.Network.LoadingFailedEvent; + } + >(); _extraHTTPHeaders: Record = {}; _credentials?: Credentials = null; @@ -168,11 +236,9 @@ export class NetworkManager extends EventEmitter { } numRequestsInProgress(): number { - return [...this._networkRequestIdEventMap.httpRequest].filter( - ([, request]) => { - return !request.response(); - } - ).length; + return [...this._requestIdToRequest].filter(([, request]) => { + return !request.response(); + }).length; } async setOfflineMode(value: boolean): Promise { @@ -261,22 +327,16 @@ export class NetworkManager extends EventEmitter { this._userRequestInterceptionEnabled && !event.request.url.startsWith('data:') ) { - const { requestId: networkRequestId } = event; + const requestId = event.requestId; + const requestPausedEvent = + this._requestIdToRequestPausedEvent.get(requestId); - this._networkRequestIdEventMap.requestWillBeSent.set( - networkRequestId, - event - ); + this._requestIdToRequestWillBeSentEvent.set(requestId, event); - /** - * CDP may have sent a Fetch.requestPaused event already. Check for it. - */ - const requestPausedEvent = - this._networkRequestIdEventMap.requestPaused.get(networkRequestId); if (requestPausedEvent) { - const { requestId: fetchRequestId } = requestPausedEvent; - this._onRequest(event, fetchRequestId); - this._networkRequestIdEventMap.requestPaused.delete(networkRequestId); + const interceptionId = requestPausedEvent.requestId; + this._onRequest(event, interceptionId); + this._requestIdToRequestPausedEvent.delete(requestId); } return; @@ -320,70 +380,50 @@ export class NetworkManager extends EventEmitter { .catch(debugError); } - const { networkId: networkRequestId, requestId: fetchRequestId } = event; + const requestId = event.networkId; + const interceptionId = event.requestId; - if (!networkRequestId) { + if (!requestId) { return; } - /** - * CDP may send a Fetch.requestPaused without or before a - * Network.requestWillBeSent - * - * CDP may send multiple Fetch.requestPaused - * for the same Network.requestWillBeSent. - * - * - */ - const requestWillBeSentEvent = (() => { - const requestWillBeSentEvent = - this._networkRequestIdEventMap.requestWillBeSent.get(networkRequestId); - - // redirect requests have the same `requestId`, - if ( - requestWillBeSentEvent && - (requestWillBeSentEvent.request.url !== event.request.url || - requestWillBeSentEvent.request.method !== event.request.method) - ) { - this._networkRequestIdEventMap.requestWillBeSent.delete( - networkRequestId - ); - return; - } - return requestWillBeSentEvent; - })(); + let requestWillBeSentEvent = + this._requestIdToRequestWillBeSentEvent.get(requestId); + + // redirect requests have the same `requestId`, + if ( + requestWillBeSentEvent && + (requestWillBeSentEvent.request.url !== event.request.url || + requestWillBeSentEvent.request.method !== event.request.method) + ) { + this._requestIdToRequestWillBeSentEvent.delete(requestId); + requestWillBeSentEvent = null; + } if (requestWillBeSentEvent) { - this._onRequest(requestWillBeSentEvent, fetchRequestId); + this._onRequest(requestWillBeSentEvent, interceptionId); } else { - this._networkRequestIdEventMap.requestPaused.set(networkRequestId, event); + this._requestIdToRequestPausedEvent.set(requestId, event); } } - _networkRequestIdToQueuedRedirectInfo(requestId: string): Array<{ + _requestIdToQueuedRedirectInfo(requestId: string): Array<{ event: Protocol.Network.RequestWillBeSentEvent; interceptionId?: string; }> { - if (!this._networkRequestIdEventMap.queuedRedirectInfoMap.has(requestId)) { - this._networkRequestIdEventMap.queuedRedirectInfoMap.set(requestId, []); + if (!this._requestIdToQueuedRedirectInfoMap.has(requestId)) { + this._requestIdToQueuedRedirectInfoMap.set(requestId, []); } - return this._networkRequestIdEventMap.queuedRedirectInfoMap.get(requestId); + return this._requestIdToQueuedRedirectInfoMap.get(requestId); } - _networkRequestIdToResponseExtraInfo( + _requestIdToResponseExtraInfo( requestId: string ): Protocol.Network.ResponseReceivedExtraInfoEvent[] { - if ( - !this._networkRequestIdEventMap.responseReceivedExtraInfo.has(requestId) - ) { - this._networkRequestIdEventMap.responseReceivedExtraInfo.set( - requestId, - [] - ); + if (!this._requestIdToResponseReceivedExtraInfo.has(requestId)) { + this._requestIdToResponseReceivedExtraInfo.set(requestId, []); } - return this._networkRequestIdEventMap.responseReceivedExtraInfo.get( - requestId - ); + return this._requestIdToResponseReceivedExtraInfo.get(requestId); } _onRequest( @@ -401,11 +441,11 @@ export class NetworkManager extends EventEmitter { // response/requestfinished. let redirectResponseExtraInfo = null; if (event.redirectHasExtraInfo) { - redirectResponseExtraInfo = this._networkRequestIdToResponseExtraInfo( + redirectResponseExtraInfo = this._requestIdToResponseExtraInfo( event.requestId ).shift(); if (!redirectResponseExtraInfo) { - this._networkRequestIdToQueuedRedirectInfo(event.requestId).push({ + this._requestIdToQueuedRedirectInfo(event.requestId).push({ event, interceptionId, }); @@ -413,9 +453,7 @@ export class NetworkManager extends EventEmitter { } } - const request = this._networkRequestIdEventMap.httpRequest.get( - event.requestId - ); + const request = this._requestIdToRequest.get(event.requestId); // If we connect late to the target, we could have missed the // requestWillBeSent event. if (request) { @@ -438,7 +476,7 @@ export class NetworkManager extends EventEmitter { event, redirectChain ); - this._networkRequestIdEventMap.httpRequest.set(event.requestId, request); + this._requestIdToRequest.set(event.requestId, request); this.emit(NetworkManagerEmittedEvents.Request, request); request.finalizeInterceptions(); } @@ -446,9 +484,7 @@ export class NetworkManager extends EventEmitter { _onRequestServedFromCache( event: Protocol.Network.RequestServedFromCacheEvent ): void { - const request = this._networkRequestIdEventMap.httpRequest.get( - event.requestId - ); + const request = this._requestIdToRequest.get(event.requestId); if (request) request._fromMemoryCache = true; this.emit(NetworkManagerEmittedEvents.RequestServedFromCache, request); } @@ -478,13 +514,11 @@ export class NetworkManager extends EventEmitter { responseReceived: Protocol.Network.ResponseReceivedEvent, extraInfo: Protocol.Network.ResponseReceivedExtraInfoEvent ): void { - const request = this._networkRequestIdEventMap.httpRequest.get( - responseReceived.requestId - ); + const request = this._requestIdToRequest.get(responseReceived.requestId); // FileUpload sends a response without a matching request. if (!request) return; - const extraInfos = this._networkRequestIdToResponseExtraInfo( + const extraInfos = this._requestIdToResponseExtraInfo( responseReceived.requestId ); if (extraInfos.length) { @@ -504,19 +538,15 @@ export class NetworkManager extends EventEmitter { } _onResponseReceived(event: Protocol.Network.ResponseReceivedEvent): void { - const request = this._networkRequestIdEventMap.httpRequest.get( - event.requestId - ); + const request = this._requestIdToRequest.get(event.requestId); let extraInfo = null; if (request && !request._fromMemoryCache && event.hasExtraInfo) { - extraInfo = this._networkRequestIdToResponseExtraInfo( - event.requestId - ).shift(); + extraInfo = this._requestIdToResponseExtraInfo(event.requestId).shift(); if (!extraInfo) { // Wait until we get the corresponding ExtraInfo event. let resolver = null; const promise = new Promise((resolve) => (resolver = resolve)); - this._networkRequestIdEventMap.queuedEvents.set(event.requestId, { + this._requestIdToQueuedEvents.set(event.requestId, { responseReceived: event, promise, resolver, @@ -528,8 +558,7 @@ export class NetworkManager extends EventEmitter { } responseWaitingForExtraInfoPromise(requestId: string): Promise { - const responseReceived = - this._networkRequestIdEventMap.queuedEvents.get(requestId); + const responseReceived = this._requestIdToQueuedEvents.get(requestId); if (!responseReceived) return Promise.resolve(); return responseReceived.promise; } @@ -540,20 +569,18 @@ export class NetworkManager extends EventEmitter { // We may have skipped a redirect response/request pair due to waiting for // this ExtraInfo event. If so, continue that work now that we have the // request. - const redirectInfo = this._networkRequestIdToQueuedRedirectInfo( + const redirectInfo = this._requestIdToQueuedRedirectInfo( event.requestId ).shift(); if (redirectInfo) { - this._networkRequestIdToResponseExtraInfo(event.requestId).push(event); + this._requestIdToResponseExtraInfo(event.requestId).push(event); this._onRequest(redirectInfo.event, redirectInfo.interceptionId); return; } // We may have skipped response and loading events because we didn't have // this ExtraInfo event yet. If so, emit those events now. - const queuedEvents = this._networkRequestIdEventMap.queuedEvents.get( - event.requestId - ); + const queuedEvents = this._requestIdToQueuedEvents.get(event.requestId); if (queuedEvents) { this._emitResponseEvent(queuedEvents.responseReceived, event); if (queuedEvents.loadingFinished) { @@ -567,27 +594,29 @@ export class NetworkManager extends EventEmitter { } // Wait until we get another event that can use this ExtraInfo event. - this._networkRequestIdToResponseExtraInfo(event.requestId).push(event); + this._requestIdToResponseExtraInfo(event.requestId).push(event); } _forgetRequest(request: HTTPRequest, events: boolean): void { const requestId = request._requestId; const interceptionId = request._interceptionId; - this._networkRequestIdEventMap.httpRequest.delete(requestId); + this._requestIdToRequest.delete(requestId); this._attemptedAuthentications.delete(interceptionId); if (events) { - this._networkRequestIdEventMap.forget(requestId); + this._requestIdToRequestWillBeSentEvent.delete(requestId); + this._requestIdToRequestPausedEvent.delete(requestId); + this._requestIdToQueuedEvents.delete(requestId); + this._requestIdToQueuedRedirectInfoMap.delete(requestId); + this._requestIdToResponseReceivedExtraInfo.delete(requestId); } } _onLoadingFinished(event: Protocol.Network.LoadingFinishedEvent): void { // If the response event for this request is still waiting on a // corresponding ExtraInfo event, then wait to emit this event too. - const queuedEvents = this._networkRequestIdEventMap.queuedEvents.get( - event.requestId - ); + const queuedEvents = this._requestIdToQueuedEvents.get(event.requestId); if (queuedEvents) { queuedEvents.loadingFinished = event; } else { @@ -596,9 +625,7 @@ export class NetworkManager extends EventEmitter { } _emitLoadingFinished(event: Protocol.Network.LoadingFinishedEvent): void { - const request = this._networkRequestIdEventMap.httpRequest.get( - event.requestId - ); + const request = this._requestIdToRequest.get(event.requestId); // For certain requestIds we never receive requestWillBeSent event. // @see https://crbug.com/750469 if (!request) return; @@ -613,9 +640,7 @@ export class NetworkManager extends EventEmitter { _onLoadingFailed(event: Protocol.Network.LoadingFailedEvent): void { // If the response event for this request is still waiting on a // corresponding ExtraInfo event, then wait to emit this event too. - const queuedEvents = this._networkRequestIdEventMap.queuedEvents.get( - event.requestId - ); + const queuedEvents = this._requestIdToQueuedEvents.get(event.requestId); if (queuedEvents) { queuedEvents.loadingFailed = event; } else { @@ -624,9 +649,7 @@ export class NetworkManager extends EventEmitter { } _emitLoadingFailed(event: Protocol.Network.LoadingFailedEvent): void { - const request = this._networkRequestIdEventMap.httpRequest.get( - event.requestId - ); + const request = this._requestIdToRequest.get(event.requestId); // For certain requestIds we never receive requestWillBeSent event. // @see https://crbug.com/750469 if (!request) return; From 72c28537144b8f0d1f6e15394dd775a6b74dec83 Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Fri, 26 Nov 2021 13:54:49 -0800 Subject: [PATCH 09/10] chore: document --- src/common/NetworkManager.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/common/NetworkManager.ts b/src/common/NetworkManager.ts index d569293d98d38..c075a2e7b14c4 100644 --- a/src/common/NetworkManager.ts +++ b/src/common/NetworkManager.ts @@ -368,6 +368,15 @@ export class NetworkManager extends EventEmitter { .catch(debugError); } + /** + * CDP may send a Fetch.requestPaused without or before a + * Network.requestWillBeSent + * + * CDP may send multiple Fetch.requestPaused + * for the same Network.requestWillBeSent. + * + * + */ _onRequestPaused(event: Protocol.Fetch.RequestPausedEvent): void { if ( !this._userRequestInterceptionEnabled && From 51c244bde932bfa36c5520330b4c8c33dc03f735 Mon Sep 17 00:00:00 2001 From: Ben Allfree Date: Sun, 28 Nov 2021 08:37:03 -0800 Subject: [PATCH 10/10] chore: comments --- src/common/NetworkManager.ts | 9 --------- test/NetworkManager.spec.ts | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/src/common/NetworkManager.ts b/src/common/NetworkManager.ts index c075a2e7b14c4..d569293d98d38 100644 --- a/src/common/NetworkManager.ts +++ b/src/common/NetworkManager.ts @@ -368,15 +368,6 @@ export class NetworkManager extends EventEmitter { .catch(debugError); } - /** - * CDP may send a Fetch.requestPaused without or before a - * Network.requestWillBeSent - * - * CDP may send multiple Fetch.requestPaused - * for the same Network.requestWillBeSent. - * - * - */ _onRequestPaused(event: Protocol.Fetch.RequestPausedEvent): void { if ( !this._userRequestInterceptionEnabled && diff --git a/test/NetworkManager.spec.ts b/test/NetworkManager.spec.ts index 15af01346d7f2..de1339754f41b 100644 --- a/test/NetworkManager.spec.ts +++ b/test/NetworkManager.spec.ts @@ -461,7 +461,7 @@ describeChromeOnly('NetworkManager', () => { frameId: '099A5216AF03AAFEC988F214B024DF08', }); }); - it(`should handle multiple Fetch.requestPaused events for the same Network.requestWillBeSent event`, async () => { + it(`should handle "double pause" (crbug.com/1196004) Fetch.requestPaused events for the same Network.requestWillBeSent event`, async () => { const mockCDPSession = new MockCDPSession(); const manager = new NetworkManager(mockCDPSession, true, { frame(): Frame | null {