Merge branch 'main' into patch-4

.github/workflows/main.yml

@@ -27,7 +27,7 @@ jobs:
           fetch-depth: 2
 
       - name: Set up Node.js
-        uses: actions/setup-node@v3.1.0
+        uses: actions/setup-node@v3.1.1
         with:
           node-version: ${{ matrix.node }}
 
@@ -107,7 +107,7 @@ jobs:
           fetch-depth: 2
 
       - name: Set up Node.js
-        uses: actions/setup-node@v3.1.0
+        uses: actions/setup-node@v3.1.1
         with:
           # Test only the oldest maintenance LTS Node.js version.
           # https://github.com/nodejs/Release#release-schedule
@@ -148,7 +148,7 @@ jobs:
           fetch-depth: 2
 
       - name: Set up Node.js
-        uses: actions/setup-node@v3.1.0
+        uses: actions/setup-node@v3.1.1
         with:
           # Test only the oldest maintenance LTS Node.js version.
           # https://github.com/nodejs/Release#release-schedule

.github/workflows/scorecards-analysis.yml

@@ -46,6 +46,6 @@ jobs:
 
       # Upload the results to GitHub’s code scanning dashboard.
       - name: 'Upload to code-scanning'
-        uses: github/codeql-action/upload-sarif@0182a2c78c8a55b763909348834ed54d735ab3e2 # v1.0.26
+        uses: github/codeql-action/upload-sarif@1ed1437484560351c5be56cf73a48a279d116b78 # v1.0.26
         with:
           sarif_file: results.sarif

docs/api.md

@@ -721,7 +721,7 @@ The product is set by the `PUPPETEER_PRODUCT` environment variable or the `produ
 - `name` <[string]> The name that the custom query handler will be registered under.
 - `queryHandler` <[CustomQueryHandler]> The [custom query handler](#interface-customqueryhandler) to register.
 
-Registers a [custom query handler](#interface-customqueryhandler). After registration, the handler can be used everywhere where a selector is expected by prepending the selection string with `<name>/`. The name is only allowed to consist of lower and upper case Latin letters.
+Registers a [custom query handler](#interface-customqueryhandler).
 
 Example:
 
@@ -5525,7 +5525,11 @@ This method is identical to `off` and maintained for compatibility with Node's E
 
 ### interface: CustomQueryHandler
 
-Contains two functions `queryOne` and `queryAll` that can be [registered](#puppeteerregistercustomqueryhandlername-queryhandler) as alternative querying strategies. The functions `queryOne` and `queryAll` are executed in the page context. `queryOne` should take an `Element` and a selector string as argument and return a single `Element` or `null` if no element is found. `queryAll` takes the same arguments but should instead return a `NodeList<Element>` or `Array<Element>` with all the elements that match the given query selector.
+After [registration](#puppeteerregistercustomqueryhandlername-queryhandler), the handler can be used everywhere where a selector is expected by prepending the selection string with `<name>/`. The name is only allowed to consist of lower and upper case Latin letters.
+
+Contains two functions `queryOne` and `queryAll` that are executed in the page context. `queryOne` should take an `Element` and a selector string as argument and return a single `Element` or `null` if no element is found. `queryAll` takes the same arguments but should instead return a `NodeList<Element>` or `Array<Element>` with all the elements that match the given query selector.
+
+**NOTE:** Because the function code needs to be serialized, it is **not possible** to access anything outside its scope or use imports. See e.g. [the pierce handler](https://github.com/puppeteer/puppeteer/blob/v13.5.2/src/common/QueryHandler.ts#L115) which has redundancies because of this limitation.
 
 ### interface: Selector
 
@@ -5534,7 +5538,7 @@ The default behavior is to regard the string as a [CSS selector] and query using
 If a selector string contains a forward slash `/` the selector is instead regarded as custom selector where everything before the slash is the [custom handler](#puppeteerregistercustomqueryhandlername-queryhandler) name and everything after is the selector: `<handler>/<selector>`.
 Puppeteer ships with two such custom handlers pre-registered:
 
-- `aria/`: Queries the accessibilty tree for computed accessibility properties.
+- `aria`: Queries the accessibilty tree for computed accessibility properties.
   The selectors consist of an accessible name to query for and optionally
   further aria attributes on the form `[<attribute>=<value>]`.
   Currently, we only support the `name` and `role` attribute.

package.json

@@ -98,7 +98,7 @@
     "@web/test-runner": "0.13.27",
     "commonmark": "0.30.0",
     "cross-env": "7.0.3",
-    "eslint": "8.12.0",
+    "eslint": "8.13.0",
     "eslint-config-prettier": "8.5.0",
     "eslint-plugin-import": "2.26.0",
     "eslint-plugin-mocha": "10.0.3",

src/common/NetworkEventManager.ts

@@ -184,4 +184,8 @@ export class NetworkEventManager {
   ): void {
     this._queuedEventGroupMap.set(networkRequestId, event);
   }
+
+  forgetQueuedEventGroup(networkRequestId: NetworkRequestId): void {
+    this._queuedEventGroupMap.delete(networkRequestId);
+  }
 }

src/common/NetworkManager.ts

@@ -519,6 +519,7 @@ export class NetworkManager extends EventEmitter {
       event.requestId
     );
     if (queuedEvents) {
+      this._networkEventManager.forgetQueuedEventGroup(event.requestId);
       this._emitResponseEvent(queuedEvents.responseReceivedEvent, event);
       if (queuedEvents.loadingFinishedEvent) {
         this._emitLoadingFinished(queuedEvents.loadingFinishedEvent);

src/node/BrowserRunner.ts

@@ -164,7 +164,7 @@ export class BrowserRunner {
 
   close(): Promise<void> {
     if (this._closed) return Promise.resolve();
-    if (this._isTempUserDataDir && this._product !== 'firefox') {
+    if (this._isTempUserDataDir) {
       this.kill();
     } else if (this.connection) {
       // Attempt to close the browser gracefully

test/NetworkManager.spec.ts

@@ -543,4 +543,121 @@ describeChromeOnly('NetworkManager', () => {
 
     expect(requests.length).toBe(2);
   });
+  it(`should handle Network.responseReceivedExtraInfo event after Network.responseReceived event (github.com/puppeteer/puppeteer/issues/8234)`, async () => {
+    const mockCDPSession = new MockCDPSession();
+    const manager = new NetworkManager(mockCDPSession, true, {
+      frame(): Frame | null {
+        return null;
+      },
+    });
+
+    const requests: HTTPRequest[] = [];
+    manager.on(
+      NetworkManagerEmittedEvents.RequestFinished,
+      (request: HTTPRequest) => {
+        requests.push(request);
+      }
+    );
+
+    mockCDPSession.emit('Network.requestWillBeSent', {
+      requestId: '1360.2',
+      loaderId: '9E86B0282CC98B77FB0ABD49156DDFDD',
+      documentURL: 'http://this.is.the.start.page.com/',
+      request: {
+        url: 'http://this.is.a.test.com:1080/test.js',
+        method: 'GET',
+        headers: {
+          'Accept-Language': 'en-US,en;q=0.9',
+          Referer: 'http://this.is.the.start.page.com/',
+          'User-Agent':
+            'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.0 Safari/537.36',
+        },
+        mixedContentType: 'none',
+        initialPriority: 'High',
+        referrerPolicy: 'strict-origin-when-cross-origin',
+        isSameSite: false,
+      },
+      timestamp: 10959.020087,
+      wallTime: 1649712607.861365,
+      initiator: {
+        type: 'parser',
+        url: 'http://this.is.the.start.page.com/',
+        lineNumber: 9,
+        columnNumber: 80,
+      },
+      redirectHasExtraInfo: false,
+      type: 'Script',
+      frameId: '60E6C35E7E519F28E646056820095498',
+      hasUserGesture: false,
+    });
+    mockCDPSession.emit('Network.responseReceived', {
+      requestId: '1360.2',
+      loaderId: '9E86B0282CC98B77FB0ABD49156DDFDD',
+      timestamp: 10959.042529,
+      type: 'Script',
+      response: {
+        url: 'http://this.is.a.test.com:1080',
+        status: 200,
+        statusText: 'OK',
+        headers: {
+          connection: 'keep-alive',
+          'content-length': '85862',
+        },
+        mimeType: 'text/plain',
+        connectionReused: false,
+        connectionId: 119,
+        remoteIPAddress: '127.0.0.1',
+        remotePort: 1080,
+        fromDiskCache: false,
+        fromServiceWorker: false,
+        fromPrefetchCache: false,
+        encodedDataLength: 66,
+        timing: {
+          requestTime: 10959.023904,
+          proxyStart: -1,
+          proxyEnd: -1,
+          dnsStart: 0.328,
+          dnsEnd: 2.183,
+          connectStart: 2.183,
+          connectEnd: 2.798,
+          sslStart: -1,
+          sslEnd: -1,
+          workerStart: -1,
+          workerReady: -1,
+          workerFetchStart: -1,
+          workerRespondWithSettled: -1,
+          sendStart: 2.982,
+          sendEnd: 3.757,
+          pushStart: 0,
+          pushEnd: 0,
+          receiveHeadersEnd: 16.373,
+        },
+        responseTime: 1649712607880.971,
+        protocol: 'http/1.1',
+        securityState: 'insecure',
+      },
+      hasExtraInfo: true,
+      frameId: '60E6C35E7E519F28E646056820095498',
+    });
+    mockCDPSession.emit('Network.responseReceivedExtraInfo', {
+      requestId: '1360.2',
+      blockedCookies: [],
+      headers: {
+        connection: 'keep-alive',
+        'content-length': '85862',
+      },
+      resourceIPAddressSpace: 'Private',
+      statusCode: 200,
+      headersText:
+        'HTTP/1.1 200 OK\r\nconnection: keep-alive\r\ncontent-length: 85862\r\n\r\n',
+    });
+    mockCDPSession.emit('Network.loadingFinished', {
+      requestId: '1360.2',
+      timestamp: 10959.060708,
+      encodedDataLength: 85928,
+      shouldReportCorbBlocking: false,
+    });
+
+    expect(requests.length).toBe(1);
+  });
 });

test/launcher.spec.ts

@@ -404,7 +404,7 @@ describe('Launcher specs', function () {
         await page.close();
         await browser.close();
       });
-      it('should filter out ignored default arguments', async () => {
+      itChromeOnly('should filter out ignored default arguments', async () => {
         const { defaultBrowserOptions, puppeteer } = getTestState();
         // Make sure we launch with `--enable-automation` by default.
         const defaultArgs = puppeteer.defaultArgs();
@@ -423,6 +423,25 @@ describe('Launcher specs', function () {
         expect(spawnargs.indexOf(defaultArgs[2])).toBe(-1);
         await browser.close();
       });
+      itFirefoxOnly('should filter out ignored default arguments', async () => {
+        const { defaultBrowserOptions, puppeteer } = getTestState();
+
+        const defaultArgs = puppeteer.defaultArgs();
+        const browser = await puppeteer.launch(
+          Object.assign({}, defaultBrowserOptions, {
+            // Only the first argument is fixed, others are optional.
+            ignoreDefaultArgs: [defaultArgs[0]],
+          })
+        );
+        const spawnargs = browser.process().spawnargs;
+        if (!spawnargs) {
+          throw new Error('spawnargs not present');
+        }
+        expect(spawnargs.indexOf(defaultArgs[0])).toBe(-1);
+        expect(spawnargs.indexOf(defaultArgs[1])).not.toBe(-1);
+        await browser.close();
+      });
+
       it('should have default URL when launching browser', async function () {
         const { defaultBrowserOptions, puppeteer } = getTestState();
         const browser = await puppeteer.launch(defaultBrowserOptions);