cherry-pick(1.15): support options in playwright._newRequest (#9061)

docs/src/api/class-playwright.md

@@ -83,6 +83,35 @@ class PlaywrightExample
 }
 ```
 
+## async method: Playwright._newRequest
+* langs: js
+- returns: <[FetchRequest]>
+
+**experimental** Creates new instances of [FetchRequest].
+
+### option: Playwright._newRequest.useragent = %%-context-option-useragent-%%
+
+### option: Playwright._newRequest.extraHTTPHeaders = %%-context-option-extrahttpheaders-%%
+
+### option: Playwright._newRequest.httpCredentials = %%-context-option-httpcredentials-%%
+
+### option: Playwright._newRequest.proxy = %%-browser-option-proxy-%%
+
+### option: Playwright._newRequest.timeout
+- `timeout` <[float]>
+
+Maximum time in milliseconds to wait for the response. Defaults to
+`30000` (30 seconds). Pass `0` to disable timeout.
+
+### option: Playwright._newRequest.ignoreHTTPSErrors = %%-context-option-ignorehttpserrors-%%
+
+### option: Playwright._newRequest.baseURL
+- `baseURL` <[string]>
+
+When using [`method: FetchRequest.get`], [`method: FetchRequest.post`], [`method: FetchRequest.fetch`] it takes the base URL in consideration by using the [`URL()`](https://developer.mozilla.org/en-US/docs/Web/API/URL/URL) constructor for building the corresponding URL. Examples:
+* baseURL: `http://localhost:3000` and sending rquest to `/bar.html` results in `http://localhost:3000/bar.html`
+* baseURL: `http://localhost:3000/foo/` and sending rquest to `./bar.html` results in `http://localhost:3000/foo/bar.html`
+
 ## property: Playwright.chromium
 - type: <[BrowserType]>
 

src/client/playwright.ts

@@ -20,13 +20,14 @@ import util from 'util';
 import * as channels from '../protocol/channels';
 import { TimeoutError } from '../utils/errors';
 import { createSocket } from '../utils/netUtils';
+import { headersObjectToArray } from '../utils/utils';
 import { Android } from './android';
 import { BrowserType } from './browserType';
 import { ChannelOwner } from './channelOwner';
 import { Electron } from './electron';
 import { FetchRequest } from './fetch';
 import { Selectors, SelectorsOwner, sharedSelectors } from './selectors';
-import { Size } from './types';
+import { NewRequestOptions, Size } from './types';
 const dnsLookupAsync = util.promisify(dns.lookup);
 
 type DeviceDescriptor = {
@@ -69,9 +70,12 @@ export class Playwright extends ChannelOwner<channels.PlaywrightChannel, channel
     this.selectors._addChannel(this._selectorsOwner);
   }
 
-  async _newRequest(options?: {}): Promise<FetchRequest> {
+  async _newRequest(options: NewRequestOptions = {}): Promise<FetchRequest> {
     return await this._wrapApiCall(async (channel: channels.PlaywrightChannel) => {
-      return FetchRequest.from((await channel.newRequest({})).request);
+      return FetchRequest.from((await channel.newRequest({
+        ...options,
+        extraHTTPHeaders: options.extraHTTPHeaders ? headersObjectToArray(options.extraHTTPHeaders) : undefined
+      })).request);
     });
   }
 

src/client/types.ts

@@ -17,7 +17,7 @@
 
 import * as channels from '../protocol/channels';
 import type { Size } from '../common/types';
-export { Size, Point, Rect, Quad, URLMatch, TimeoutOptions, HeadersArray } from '../common/types';
+export { Size, Point, Rect, Quad, URLMatch, TimeoutOptions, HeadersArray, NewRequestOptions } from '../common/types';
 
 type LoggerSeverity = 'verbose' | 'info' | 'warning' | 'error';
 export interface Logger {

src/common/types.ts

@@ -22,3 +22,20 @@ export type URLMatch = string | RegExp | ((url: URL) => boolean);
 export type TimeoutOptions = { timeout?: number };
 export type NameValue = { name: string, value: string };
 export type HeadersArray = NameValue[];
+export type NewRequestOptions = {
+  baseURL?: string;
+  extraHTTPHeaders?: { [key: string]: string; };
+  httpCredentials?: {
+    username: string;
+    password: string;
+  };
+  ignoreHTTPSErrors?: boolean;
+  proxy?: {
+    server: string;
+    bypass?: string;
+    username?: string;
+    password?: string;
+  };
+  timeout?: number;
+  userAgent?: string;
+};

src/dispatchers/playwrightDispatcher.ts

@@ -16,18 +16,18 @@
 
 import net, { AddressInfo } from 'net';
 import * as channels from '../protocol/channels';
+import { GlobalFetchRequest } from '../server/fetch';
 import { Playwright } from '../server/playwright';
+import * as types from '../server/types';
+import { debugLogger } from '../utils/debugLogger';
+import { SocksConnection, SocksConnectionClient } from '../utils/socksProxy';
+import { createGuid } from '../utils/utils';
 import { AndroidDispatcher } from './androidDispatcher';
 import { BrowserTypeDispatcher } from './browserTypeDispatcher';
 import { Dispatcher, DispatcherScope } from './dispatcher';
 import { ElectronDispatcher } from './electronDispatcher';
-import { SelectorsDispatcher } from './selectorsDispatcher';
-import * as types from '../server/types';
-import { SocksConnection, SocksConnectionClient } from '../utils/socksProxy';
-import { createGuid } from '../utils/utils';
-import { debugLogger } from '../utils/debugLogger';
-import { GlobalFetchRequest } from '../server/fetch';
 import { FetchRequestDispatcher } from './networkDispatchers';
+import { SelectorsDispatcher } from './selectorsDispatcher';
 
 export class PlaywrightDispatcher extends Dispatcher<Playwright, channels.PlaywrightInitializer, channels.PlaywrightEvents> implements channels.PlaywrightChannel {
   private _socksProxy: SocksProxy | undefined;
@@ -75,7 +75,7 @@ export class PlaywrightDispatcher extends Dispatcher<Playwright, channels.Playwr
   }
 
   async newRequest(params: channels.PlaywrightNewRequestParams, metadata?: channels.Metadata): Promise<channels.PlaywrightNewRequestResult> {
-    const request = new GlobalFetchRequest(this._object);
+    const request = new GlobalFetchRequest(this._object, params);
     return { request: FetchRequestDispatcher.from(this._scope, request) };
   }
 }

src/protocol/channels.ts

@@ -322,10 +322,38 @@ export type PlaywrightSocksEndOptions = {
 };
 export type PlaywrightSocksEndResult = void;
 export type PlaywrightNewRequestParams = {
+  baseURL?: string,
+  userAgent?: string,
   ignoreHTTPSErrors?: boolean,
+  extraHTTPHeaders?: NameValue[],
+  httpCredentials?: {
+    username: string,
+    password: string,
+  },
+  proxy?: {
+    server: string,
+    bypass?: string,
+    username?: string,
+    password?: string,
+  },
+  timeout?: number,
 };
 export type PlaywrightNewRequestOptions = {
+  baseURL?: string,
+  userAgent?: string,
   ignoreHTTPSErrors?: boolean,
+  extraHTTPHeaders?: NameValue[],
+  httpCredentials?: {
+    username: string,
+    password: string,
+  },
+  proxy?: {
+    server: string,
+    bypass?: string,
+    username?: string,
+    password?: string,
+  },
+  timeout?: number,
 };
 export type PlaywrightNewRequestResult = {
   request: FetchRequestChannel,

src/protocol/protocol.yml

@@ -455,7 +455,26 @@ Playwright:
 
     newRequest:
       parameters:
+        baseURL: string?
+        userAgent: string?
         ignoreHTTPSErrors: boolean?
+        extraHTTPHeaders:
+          type: array?
+          items: NameValue
+        httpCredentials:
+          type: object?
+          properties:
+            username: string
+            password: string
+        proxy:
+          type: object?
+          properties:
+            server: string
+            bypass: string?
+            username: string?
+            password: string?
+        timeout: number?
+
       returns:
         request: FetchRequest
 

src/protocol/validator.ts

@@ -195,7 +195,21 @@ export function createScheme(tChannel: (name: string) => Validator): Scheme {
     uid: tString,
   });
   scheme.PlaywrightNewRequestParams = tObject({
+    baseURL: tOptional(tString),
+    userAgent: tOptional(tString),
     ignoreHTTPSErrors: tOptional(tBoolean),
+    extraHTTPHeaders: tOptional(tArray(tType('NameValue'))),
+    httpCredentials: tOptional(tObject({
+      username: tString,
+      password: tString,
+    })),
+    proxy: tOptional(tObject({
+      server: tString,
+      bypass: tOptional(tString),
+      username: tOptional(tString),
+      password: tOptional(tString),
+    })),
+    timeout: tOptional(tNumber),
   });
   scheme.SelectorsRegisterParams = tObject({
     name: tString,

src/server/fetch.ts

@@ -14,24 +14,25 @@
  * limitations under the License.
  */
 
-import { HttpsProxyAgent } from 'https-proxy-agent';
-import url from 'url';
-import zlib from 'zlib';
 import * as http from 'http';
 import * as https from 'https';
-import { BrowserContext } from './browserContext';
-import * as types from './types';
+import { HttpsProxyAgent } from 'https-proxy-agent';
 import { pipeline, Readable, Transform } from 'stream';
+import url from 'url';
+import zlib from 'zlib';
+import { HTTPCredentials } from '../../types/types';
+import { NameValue, NewRequestOptions } from '../common/types';
+import { TimeoutSettings } from '../utils/timeoutSettings';
 import { createGuid, isFilePayload, monotonicTime } from '../utils/utils';
+import { BrowserContext } from './browserContext';
+import { MultipartFormData } from './formData';
 import { SdkObject } from './instrumentation';
 import { Playwright } from './playwright';
+import * as types from './types';
 import { HeadersArray, ProxySettings } from './types';
-import { HTTPCredentials } from '../../types/types';
-import { TimeoutSettings } from '../utils/timeoutSettings';
-import { MultipartFormData } from './formData';
 
 
-type FetchRequestOptions = {
+export type FetchRequestOptions = {
   userAgent: string;
   extraHTTPHeaders?: HeadersArray;
   httpCredentials?: HTTPCredentials;
@@ -333,23 +334,37 @@ export class BrowserContextFetchRequest extends FetchRequest {
 
 
 export class GlobalFetchRequest extends FetchRequest {
-  constructor(playwright: Playwright) {
+  private readonly _options: FetchRequestOptions;
+  constructor(playwright: Playwright, options: Omit<NewRequestOptions, 'extraHTTPHeaders'> & { extraHTTPHeaders?: NameValue[] }) {
     super(playwright);
+    const timeoutSettings = new TimeoutSettings();
+    if (options.timeout !== undefined)
+      timeoutSettings.setDefaultTimeout(options.timeout);
+    const proxy = options.proxy;
+    if (proxy?.server) {
+      let url = proxy?.server.trim();
+      if (!/^\w+:\/\//.test(url))
+        url = 'http://' + url;
+      proxy.server = url;
+    }
+    this._options = {
+      baseURL: options.baseURL,
+      userAgent: options.userAgent || '',
+      extraHTTPHeaders: options.extraHTTPHeaders,
+      ignoreHTTPSErrors: !!options.ignoreHTTPSErrors,
+      httpCredentials: options.httpCredentials,
+      proxy,
+      timeoutSettings,
+    };
+
   }
 
   override dispose() {
     this._disposeImpl();
   }
 
   _defaultOptions(): FetchRequestOptions {
-    return {
-      userAgent: '',
-      extraHTTPHeaders: undefined,
-      proxy: undefined,
-      timeoutSettings: new TimeoutSettings(),
-      ignoreHTTPSErrors: false,
-      baseURL: undefined,
-    };
+    return this._options;
   }
 
   async _addCookies(cookies: types.SetNetworkCookieParam[]): Promise<void> {

tests/browsercontext-fetch.spec.ts

@@ -42,19 +42,6 @@ it.afterAll(() => {
   http.globalAgent = prevAgent;
 });
 
-it('global get should work', async ({playwright, context, server}) => {
-  const request = await playwright._newRequest();
-  const response = await request.get(server.PREFIX + '/simple.json');
-  expect(response.url()).toBe(server.PREFIX + '/simple.json');
-  expect(response.status()).toBe(200);
-  expect(response.statusText()).toBe('OK');
-  expect(response.ok()).toBeTruthy();
-  expect(response.url()).toBe(server.PREFIX + '/simple.json');
-  expect(response.headers()['content-type']).toBe('application/json; charset=utf-8');
-  expect(response.headersArray()).toContainEqual({ name: 'Content-Type', value: 'application/json; charset=utf-8' });
-  expect(await response.text()).toBe('{"foo": "bar"}\n');
-});
-
 it('get should work', async ({context, server}) => {
   const response = await context._request.get(server.PREFIX + '/simple.json');
   expect(response.url()).toBe(server.PREFIX + '/simple.json');
@@ -692,15 +679,6 @@ it('should dispose when context closes', async function({context, server}) {
   expect(error.message).toContain('Response has been disposed');
 });
 
-it('should dispose global request', async function({playwright, context, server}) {
-  const request = await playwright._newRequest();
-  const response = await request.get(server.PREFIX + '/simple.json');
-  expect(await response.json()).toEqual({ foo: 'bar' });
-  await request.dispose();
-  const error = await response.body().catch(e => e);
-  expect(error.message).toContain('Response has been disposed');
-});
-
 it('should throw on invalid first argument', async function({context}) {
   const error = await context._request.get({} as any).catch(e => e);
   expect(error.message).toContain('First argument must be either URL string or Request');