diff --git a/browser-versions.json b/browser-versions.json index afde97bc3133..678439a4b20f 100644 --- a/browser-versions.json +++ b/browser-versions.json @@ -1,4 +1,4 @@ { - "chrome:beta": "92.0.4515.40", - "chrome:stable": "91.0.4472.77" + "chrome:beta": "92.0.4515.51", + "chrome:stable": "91.0.4472.101" } diff --git a/circle.yml b/circle.yml index 29893111719a..6bbe30f82b9b 100644 --- a/circle.yml +++ b/circle.yml @@ -975,14 +975,14 @@ jobs: - run: yarn test-mocha # test binary build code - run: yarn test-scripts - # check for compile errors with the releaserc scripts - - run: yarn test-npm-package-release-script # make sure our snapshots are compared correctly - run: yarn test-mocha-snapshot # make sure packages with TypeScript can be transpiled to JS - run: yarn lerna run build-prod --stream # run unit tests from each individual package - run: yarn test + # check for compile errors with the releaserc scripts + - run: yarn test-npm-package-release-script - verify-mocha-results: expectedResultCount: 9 - store_test_results: diff --git a/packages/driver/cypress/integration/commands/net_stubbing_spec.ts b/packages/driver/cypress/integration/commands/net_stubbing_spec.ts index 599174dcc270..558ebb6f165e 100644 --- a/packages/driver/cypress/integration/commands/net_stubbing_spec.ts +++ b/packages/driver/cypress/integration/commands/net_stubbing_spec.ts @@ -480,6 +480,54 @@ describe('network stubbing', { retries: { runMode: 2, openMode: 0 } }, function }).visit('/dump-method').contains('GET') }) }) + + // https://github.com/cypress-io/cypress/issues/16292 + describe('multibyte utf8', () => { + const multibyteUtf8 = [ + // 1. When there's problem in the chunkEnd + // * 2 bytes + '12345678901234567890123Ф', + // * 3 bytes + '12345678901234567890123안', + '1234567890123456789012안', + // * 4 bytes + '12345678901234567890123😀', + '1234567890123456789012😀', + '123456789012345678901😀', + // 2. When there's a problem in the chunkBegin + // * 2 bytes + 'dummyФ12345678901234567890123', + // * 3 bytes + 'dummy안12345678901234567890123', + 'dummy안1234567890123456789012', + // * 4 bytes + 'dummy😀12345678901234567890123', + 'dummy😀1234567890123456789012', + 'dummy😀123456789012345678901', + ] + + multibyteUtf8.forEach((str) => { + it(str, () => { + cy.intercept('https://example.com/test', { + body: { result: 'ok' }, + }).as('testRequest') + + cy.window().then(() => { + let xhr = new XMLHttpRequest() + + xhr.open('POST', 'https://example.com/test') + xhr.setRequestHeader('Content-Type', 'application/json') + xhr.send(str) + }) + + cy.wait('@testRequest') + .its('request') + .then((req) => { + expect(req.body).to.eq(str) + }) + }) + }) + }) }) context('logging', function () { diff --git a/packages/net-stubbing/lib/external-types.ts b/packages/net-stubbing/lib/external-types.ts index 54cd65585e1e..9a2361bf0dfe 100644 --- a/packages/net-stubbing/lib/external-types.ts +++ b/packages/net-stubbing/lib/external-types.ts @@ -79,7 +79,7 @@ export namespace CyHttpMessages { /** * The headers of the HTTP message. */ - headers: { [key: string]: string } + headers: { [key: string]: string | string[] } } export type IncomingResponse = BaseMessage & { diff --git a/packages/net-stubbing/lib/server/util.ts b/packages/net-stubbing/lib/server/util.ts index c5b54810f6d8..bf1d6971a413 100644 --- a/packages/net-stubbing/lib/server/util.ts +++ b/packages/net-stubbing/lib/server/util.ts @@ -241,7 +241,8 @@ export function getBodyEncoding (req: CyHttpMessages.IncomingRequest): BodyEncod // a simple heuristic for detecting UTF8 encoded requests if (req.headers && req.headers['content-type']) { - const contentType = req.headers['content-type'].toLowerCase() + const contentTypeHeader = req.headers['content-type'] as string + const contentType = contentTypeHeader.toLowerCase() if (contentType.includes('charset=utf-8') || contentType.includes('charset="utf-8"')) { return 'utf8' diff --git a/packages/proxy/lib/http/index.ts b/packages/proxy/lib/http/index.ts index a286a8747c7d..665ec59f7c5c 100644 --- a/packages/proxy/lib/http/index.ts +++ b/packages/proxy/lib/http/index.ts @@ -3,10 +3,12 @@ import CyServer from '@packages/server' import { CypressIncomingRequest, CypressOutgoingResponse, + BrowserPreRequest, } from '@packages/proxy' -import debugModule from 'debug' +import Debug from 'debug' import ErrorMiddleware from './error-middleware' import { HttpBuffers } from './util/buffers' +import { GetPreRequestCb, PreRequests } from './util/prerequests' import { IncomingMessage } from 'http' import { NetStubbingState } from '@packages/net-stubbing' import Bluebird from 'bluebird' @@ -16,7 +18,7 @@ import RequestMiddleware from './request-middleware' import ResponseMiddleware from './response-middleware' import { DeferredSourceMapCache } from '@packages/rewriter' -const debug = debugModule('cypress:proxy:http') +const debugRequests = Debug('cypress-verbose:proxy:http') export enum HttpStages { IncomingRequest, @@ -35,9 +37,12 @@ export type HttpMiddlewareStacks = { type HttpMiddlewareCtx = { req: CypressIncomingRequest res: CypressOutgoingResponse - + shouldCorrelatePreRequests: () => boolean + stage: HttpStages + debug: Debug.Debugger middleware: HttpMiddlewareStacks deferSourceMapRewrite: (opts: { js: string, url: string }) => string + getPreRequest: (cb: GetPreRequestCb) => void } & T export const defaultMiddleware = { @@ -48,6 +53,7 @@ export const defaultMiddleware = { export type ServerCtx = Readonly<{ config: CyServer.Config + shouldCorrelatePreRequests?: () => boolean getFileServerToken: () => string getRemoteState: CyServer.getRemoteState netStubbingState: NetStubbingState @@ -82,10 +88,8 @@ type HttpMiddlewareThis = HttpMiddlewareCtx & ServerCtx & Readonly<{ skipMiddleware: (name: string) => void }> -export function _runStage (type: HttpStages, ctx: any) { - const stage = HttpStages[type] - - debug('Entering stage %o', { stage }) +export function _runStage (type: HttpStages, ctx: any, onError) { + ctx.stage = HttpStages[type] const runMiddlewareStack = () => { const middlewares = ctx.middleware[type] @@ -131,8 +135,6 @@ export function _runStage (type: HttpStages, ctx: any) { return resolve() } - debug('Running middleware %o', { stage, middlewareName }) - const fullCtx = { next: () => { copyChangedCtx() @@ -147,21 +149,19 @@ export function _runStage (type: HttpStages, ctx: any) { _end() }, onError: (error: Error) => { - debug('Error in middleware %o', { stage, middlewareName, error }) + ctx.debug('Error in middleware %o', { middlewareName, error }) if (type === HttpStages.Error) { return } ctx.error = error - - _end(_runStage(HttpStages.Error, ctx)) + onError(error) + _end(_runStage(HttpStages.Error, ctx, onError)) }, - skipMiddleware: (name) => { ctx.middleware[type] = _.omit(ctx.middleware[type], name) }, - ...ctx, } @@ -174,19 +174,18 @@ export function _runStage (type: HttpStages, ctx: any) { } return runMiddlewareStack() - .then(() => { - debug('Leaving stage %o', { stage }) - }) } export class Http { buffers: HttpBuffers config: CyServer.Config + shouldCorrelatePreRequests: () => boolean deferredSourceMapCache: DeferredSourceMapCache getFileServerToken: () => string getRemoteState: () => any middleware: HttpMiddlewareStacks netStubbingState: NetStubbingState + preRequests: PreRequests = new PreRequests() request: any socket: CyServer.Socket @@ -195,6 +194,7 @@ export class Http { this.deferredSourceMapCache = new DeferredSourceMapCache(opts.request) this.config = opts.config + this.shouldCorrelatePreRequests = opts.shouldCorrelatePreRequests || (() => false) this.getFileServerToken = opts.getFileServerToken this.getRemoteState = opts.getRemoteState this.middleware = opts.middleware @@ -213,27 +213,43 @@ export class Http { res, buffers: this.buffers, config: this.config, + shouldCorrelatePreRequests: this.shouldCorrelatePreRequests, getFileServerToken: this.getFileServerToken, getRemoteState: this.getRemoteState, request: this.request, middleware: _.cloneDeep(this.middleware), netStubbingState: this.netStubbingState, socket: this.socket, + debug: (formatter, ...args) => { + debugRequests(`%s %s %s ${formatter}`, ctx.req.method, ctx.req.proxiedUrl, ctx.stage, ...args) + }, deferSourceMapRewrite: (opts) => { this.deferredSourceMapCache.defer({ resHeaders: ctx.incomingRes.headers, ...opts, }) }, + getPreRequest: (cb) => { + this.preRequests.get(ctx.req, ctx.debug, cb) + }, + } + + const onError = () => { + if (ctx.req.browserPreRequest) { + // browsers will retry requests in the event of network errors, but they will not send pre-requests, + // so try to re-use the current browserPreRequest for the next retry + ctx.debug('Re-using pre-request data %o', ctx.req.browserPreRequest) + this.addPendingBrowserPreRequest(ctx.req.browserPreRequest) + } } - return _runStage(HttpStages.IncomingRequest, ctx) + return _runStage(HttpStages.IncomingRequest, ctx, onError) .then(() => { if (ctx.incomingRes) { - return _runStage(HttpStages.IncomingResponse, ctx) + return _runStage(HttpStages.IncomingResponse, ctx, onError) } - return debug('warning: Request was not fulfilled with a response.') + return ctx.debug('Warning: Request was not fulfilled with a response.') }) } @@ -253,9 +269,14 @@ export class Http { reset () { this.buffers.reset() + this.preRequests = new PreRequests() } setBuffer (buffer) { return this.buffers.set(buffer) } + + addPendingBrowserPreRequest (browserPreRequest: BrowserPreRequest) { + this.preRequests.addPending(browserPreRequest) + } } diff --git a/packages/proxy/lib/http/request-middleware.ts b/packages/proxy/lib/http/request-middleware.ts index f68d2254b47b..1b6452a1f210 100644 --- a/packages/proxy/lib/http/request-middleware.ts +++ b/packages/proxy/lib/http/request-middleware.ts @@ -19,11 +19,40 @@ const LogRequest: RequestMiddleware = function () { this.next() } +const CorrelateBrowserPreRequest: RequestMiddleware = async function () { + if (!this.shouldCorrelatePreRequests()) { + return this.next() + } + + if (this.req.headers['x-cypress-resolving-url']) { + this.debug('skipping prerequest for resolve:url') + delete this.req.headers['x-cypress-resolving-url'] + + return this.next() + } + + this.debug('waiting for prerequest') + this.getPreRequest(((browserPreRequest) => { + this.req.browserPreRequest = browserPreRequest + this.next() + })) +} + +const SendToDriver: RequestMiddleware = function () { + const { browserPreRequest } = this.req + + if (browserPreRequest) { + this.socket.toDriver('proxy:incoming:request', browserPreRequest) + } + + this.next() +} + const MaybeEndRequestWithBufferedResponse: RequestMiddleware = function () { const buffer = this.buffers.take(this.req.proxiedUrl) if (buffer) { - debug('got a buffer %o', _.pick(buffer, 'url')) + this.debug('ending request with buffered response') this.res.wantsInjection = 'full' return this.onResponse(buffer.response, buffer.stream) @@ -52,10 +81,7 @@ const EndRequestsToBlockedHosts: RequestMiddleware = function () { if (matches) { this.res.set('x-cypress-matched-blocked-host', matches) - debug('blocking request %o', { - url: this.req.proxiedUrl, - matches, - }) + this.debug('blocking request %o', { matches }) this.res.status(503).end() @@ -127,7 +153,7 @@ const SendRequestOutgoing: RequestMiddleware = function () { req.on('error', this.onError) req.on('response', (incomingRes) => this.onResponse(incomingRes, req)) this.req.on('aborted', () => { - debug('request aborted') + this.debug('request aborted') req.abort() }) @@ -141,6 +167,8 @@ const SendRequestOutgoing: RequestMiddleware = function () { export default { LogRequest, + CorrelateBrowserPreRequest, + SendToDriver, MaybeEndRequestWithBufferedResponse, InterceptRequest, RedirectToClientRouteIfUnloaded, diff --git a/packages/proxy/lib/http/util/prerequests.ts b/packages/proxy/lib/http/util/prerequests.ts new file mode 100644 index 000000000000..f710d409ab21 --- /dev/null +++ b/packages/proxy/lib/http/util/prerequests.ts @@ -0,0 +1,112 @@ +import { + CypressIncomingRequest, + BrowserPreRequest, +} from '@packages/proxy' +import Debug from 'debug' +import _ from 'lodash' + +const debug = Debug('cypress:proxy:http:util:prerequests') +const debugVerbose = Debug('cypress-verbose:proxy:http:util:prerequests') + +const metrics: any = { + browserPreRequestsReceived: 0, + proxyRequestsReceived: 0, + immediatelyMatchedRequests: 0, + eventuallyReceivedPreRequest: [], + neverReceivedPreRequest: [], +} + +process.once('exit', () => { + debug('metrics: %o', metrics) +}) + +function removeOne (a: Array, predicate: (v: T) => boolean): T | void { + for (const i in a) { + const v = a[i] + + if (predicate(v)) { + a.splice(i as unknown as number, 1) + + return v + } + } +} + +function matches (preRequest: BrowserPreRequest, req: Pick) { + return preRequest.method === req.method && preRequest.url === req.proxiedUrl +} + +export type GetPreRequestCb = (browserPreRequest?: BrowserPreRequest) => void + +export class PreRequests { + pendingBrowserPreRequests: Array = [] + requestsPendingPreRequestCbs: Array<{ + cb: (browserPreRequest: BrowserPreRequest) => void + method: string + proxiedUrl: string + }> = [] + + get (req: CypressIncomingRequest, ctxDebug, cb: GetPreRequestCb) { + metrics.proxyRequestsReceived++ + + const pendingBrowserPreRequest = removeOne(this.pendingBrowserPreRequests, (browserPreRequest) => { + return matches(browserPreRequest, req) + }) + + if (pendingBrowserPreRequest) { + metrics.immediatelyMatchedRequests++ + + ctxDebug('matches pending pre-request %o', pendingBrowserPreRequest) + + return cb(pendingBrowserPreRequest) + } + + const timeout = setTimeout(() => { + metrics.neverReceivedPreRequest.push({ url: req.proxiedUrl }) + ctxDebug('500ms passed without a pre-request, continuing request with an empty pre-request field!') + + remove() + cb() + }, 500) + + const startedMs = Date.now() + const remove = _.once(() => removeOne(this.requestsPendingPreRequestCbs, (v) => v === requestPendingPreRequestCb)) + + const requestPendingPreRequestCb = { + cb: (browserPreRequest) => { + const afterMs = Date.now() - startedMs + + metrics.eventuallyReceivedPreRequest.push({ url: browserPreRequest.url, afterMs }) + ctxDebug('received pre-request after %dms %o', afterMs, browserPreRequest) + clearTimeout(timeout) + remove() + cb(browserPreRequest) + }, + proxiedUrl: req.proxiedUrl, + method: req.method, + } + + this.requestsPendingPreRequestCbs.push(requestPendingPreRequestCb) + } + + addPending (browserPreRequest: BrowserPreRequest) { + if (this.pendingBrowserPreRequests.indexOf(browserPreRequest) !== -1) { + return + } + + metrics.browserPreRequestsReceived++ + + const requestPendingPreRequestCb = removeOne(this.requestsPendingPreRequestCbs, (req) => { + return matches(browserPreRequest, req) + }) + + if (requestPendingPreRequestCb) { + debugVerbose('immediately matched pre-request %o', browserPreRequest) + + return requestPendingPreRequestCb.cb(browserPreRequest) + } + + debugVerbose('queuing pre-request to be matched later %o %o', browserPreRequest, this.pendingBrowserPreRequests) + this.pendingBrowserPreRequests.push(browserPreRequest) + } +} diff --git a/packages/proxy/lib/network-proxy.ts b/packages/proxy/lib/network-proxy.ts index cfbb82f124fc..84decf291368 100644 --- a/packages/proxy/lib/network-proxy.ts +++ b/packages/proxy/lib/network-proxy.ts @@ -1,4 +1,5 @@ import { Http, ServerCtx } from './http' +import { BrowserPreRequest } from './types' export class NetworkProxy { http: Http @@ -7,6 +8,10 @@ export class NetworkProxy { this.http = new Http(opts) } + addPendingBrowserPreRequest (preRequest: BrowserPreRequest) { + this.http.addPendingBrowserPreRequest(preRequest) + } + handleHttpRequest (req, res) { this.http.handle(req, res) } diff --git a/packages/proxy/lib/types.ts b/packages/proxy/lib/types.ts index 05977a326bc0..46a42b9c42cc 100644 --- a/packages/proxy/lib/types.ts +++ b/packages/proxy/lib/types.ts @@ -8,6 +8,7 @@ export type CypressIncomingRequest = Request & { proxiedUrl: string abort: () => void requestId: string + browserPreRequest?: BrowserPreRequest body?: string responseTimeout?: number followRedirect?: boolean @@ -28,3 +29,16 @@ export { ErrorMiddleware } from './http/error-middleware' export { RequestMiddleware } from './http/request-middleware' export { ResponseMiddleware } from './http/response-middleware' + +export type ResourceType = 'fetch' | 'xhr' | 'websocket' | 'stylesheet' | 'script' | 'image' | 'font' | 'cspviolationreport' | 'ping' | 'manifest' | 'other' + +/** + * Metadata about an HTTP request, according to the browser's pre-request event. + */ +export type BrowserPreRequest = { + requestId: string + method: string + url: string + resourceType: ResourceType + originalResourceType: string | undefined +} diff --git a/packages/proxy/test/unit/http/request-middleware.spec.ts b/packages/proxy/test/unit/http/request-middleware.spec.ts index 3298c9479a03..bdd139549471 100644 --- a/packages/proxy/test/unit/http/request-middleware.spec.ts +++ b/packages/proxy/test/unit/http/request-middleware.spec.ts @@ -6,6 +6,8 @@ describe('http/request-middleware', function () { it('exports the members in the correct order', function () { expect(_.keys(RequestMiddleware)).to.have.ordered.members([ 'LogRequest', + 'CorrelateBrowserPreRequest', + 'SendToDriver', 'MaybeEndRequestWithBufferedResponse', 'InterceptRequest', 'RedirectToClientRouteIfUnloaded', diff --git a/packages/proxy/test/unit/http/util/prerequests.spec.ts b/packages/proxy/test/unit/http/util/prerequests.spec.ts new file mode 100644 index 000000000000..c05fead6d3a5 --- /dev/null +++ b/packages/proxy/test/unit/http/util/prerequests.spec.ts @@ -0,0 +1,34 @@ +import { PreRequests } from '@packages/proxy/lib/http/util/prerequests' +import { BrowserPreRequest, CypressIncomingRequest } from '@packages/proxy' +import { expect } from 'chai' +import sinon from 'sinon' + +describe('http/util/prerequests', () => { + let preRequests: PreRequests + + beforeEach(() => { + preRequests = new PreRequests() + }) + + it('synchronously matches a pre-request that existed at the time of the request', () => { + preRequests.addPending({ requestId: '1234', url: 'foo', method: 'GET' } as BrowserPreRequest) + + const cb = sinon.stub() + + preRequests.get({ proxiedUrl: 'foo', method: 'GET' } as CypressIncomingRequest, () => {}, cb) + + const { args } = cb.getCall(0) + + expect(args[0]).to.include({ requestId: '1234', url: 'foo', method: 'GET' }) + }) + + it('synchronously matches a pre-request added after the request', (done) => { + const cb = (preRequest) => { + expect(preRequest).to.include({ requestId: '1234', url: 'foo', method: 'GET' }) + done() + } + + preRequests.get({ proxiedUrl: 'foo', method: 'GET' } as CypressIncomingRequest, () => {}, cb) + preRequests.addPending({ requestId: '1234', url: 'foo', method: 'GET' } as BrowserPreRequest) + }) +}) diff --git a/packages/server-ct/src/project-ct.ts b/packages/server-ct/src/project-ct.ts index 3c10c331b387..5f7e6b8fe916 100644 --- a/packages/server-ct/src/project-ct.ts +++ b/packages/server-ct/src/project-ct.ts @@ -45,7 +45,7 @@ export class ProjectCt extends ProjectBase { return this._initPlugins(cfgForComponentTesting, options) .then(({ cfg, specsStore }) => { - return this.server.open(cfg, specsStore, this, options.onError, options.onWarning) + return this.server.open(cfg, specsStore, this, options.onError, options.onWarning, this.shouldCorrelatePreRequests) .then(([port, warning]) => { return { cfg, @@ -89,6 +89,13 @@ export class ProjectCt extends ProjectBase { updatedConfig.componentTesting = true + // This value is normally set up in the `packages/server/lib/plugins/index.js#110` + // But if we don't return it in the plugins function, it never gets set + // Since there is no chance that it will have any other value here, we set it to "component" + // This allows users to not return config in the `cypress/plugins/index.js` file + // https://github.com/cypress-io/cypress/issues/16860 + updatedConfig.resolved.testingType = { value: 'component' } + debug('updated config: %o', updatedConfig) return updatedConfig diff --git a/packages/server-ct/src/server-ct.ts b/packages/server-ct/src/server-ct.ts index e9b5d6889247..9439c6d7911f 100644 --- a/packages/server-ct/src/server-ct.ts +++ b/packages/server-ct/src/server-ct.ts @@ -13,7 +13,7 @@ type WarningErr = Record const debug = Debug('cypress:server-ct:server') export class ServerCt extends ServerBase { - open (config, specsStore, project, onError, onWarning) { + open (config, specsStore, project, onError, onWarning, shouldCorrelatePreRequests) { debug('server open') return Bluebird.try(() => { @@ -35,7 +35,7 @@ export class ServerCt extends ServerBase { return this._getRemoteState() } - this.createNetworkProxy(config, getRemoteState) + this.createNetworkProxy(config, getRemoteState, shouldCorrelatePreRequests) createRoutes({ app, diff --git a/packages/server-ct/tsconfig.json b/packages/server-ct/tsconfig.json index 84d7a0f6dc38..69a8a9e7bede 100644 --- a/packages/server-ct/tsconfig.json +++ b/packages/server-ct/tsconfig.json @@ -7,5 +7,5 @@ "files": [ "index.ts", "./../ts/index.d.ts" - ] + ], } diff --git a/packages/server/lib/automation/automation.ts b/packages/server/lib/automation/automation.ts index fdaa79ee24f7..17b124b00391 100644 --- a/packages/server/lib/automation/automation.ts +++ b/packages/server/lib/automation/automation.ts @@ -2,9 +2,12 @@ import Bluebird from 'bluebird' import { v4 as uuidv4 } from 'uuid' import { Cookies } from './cookies' import { Screenshot } from './screenshot' +import { BrowserPreRequest } from '@packages/proxy' type NullableMiddlewareHook = (() => void) | null +export type OnBrowserPreRequest = (browserPreRequest: BrowserPreRequest) => void + interface IMiddleware { onPush: NullableMiddlewareHook onBeforeRequest: NullableMiddlewareHook @@ -19,7 +22,7 @@ export class Automation { private cookies: Cookies private screenshot: { capture: (data: any, automate: any) => any } - constructor (cyNamespace: string, cookieNamespace: string, screenshotsFolder: string) { + constructor (cyNamespace: string, cookieNamespace: string, screenshotsFolder: string, public onBrowserPreRequest: OnBrowserPreRequest) { this.requests = {} // set the middleware diff --git a/packages/server/lib/browsers/cdp_automation.ts b/packages/server/lib/browsers/cdp_automation.ts index ce12431e3eb8..9c245910e475 100644 --- a/packages/server/lib/browsers/cdp_automation.ts +++ b/packages/server/lib/browsers/cdp_automation.ts @@ -1,8 +1,12 @@ +/// + import _ from 'lodash' import Bluebird from 'bluebird' import cdp from 'devtools-protocol' import { cors } from '@packages/network' import debugModule from 'debug' +import { Automation } from '../automation' +import { ResourceType, BrowserPreRequest } from '@packages/proxy' const debugVerbose = debugModule('cypress-verbose:server:browsers:cdp_automation') @@ -11,6 +15,10 @@ export type CyCookie = Pick Bluebird - export const _domainIsWithinSuperdomain = (domain: string, suffix: string) => { const suffixParts = suffix.split('.').filter(_.identity) const domainParts = domain.split('.').filter(_.identity) @@ -60,74 +62,128 @@ export const _cookieMatches = (cookie: CyCookie, filter: CyCookieFilter) => { return true } -export const CdpAutomation = (sendDebuggerCommandFn: SendDebuggerCommand) => { - const normalizeGetCookieProps = (cookie: cdp.Network.Cookie): CyCookie => { - if (cookie.expires === -1) { - // @ts-ignore - delete cookie.expires +const normalizeGetCookieProps = (cookie: cdp.Network.Cookie): CyCookie => { + if (cookie.expires === -1) { + // @ts-ignore + delete cookie.expires + } + + // @ts-ignore + cookie.sameSite = convertSameSiteCdpToExtension(cookie.sameSite) + + // @ts-ignore + cookie.expirationDate = cookie.expires + // @ts-ignore + delete cookie.expires + + // @ts-ignore + return cookie +} + +const normalizeGetCookies = (cookies: cdp.Network.Cookie[]) => { + return _.map(cookies, normalizeGetCookieProps) +} + +const normalizeSetCookieProps = (cookie: CyCookie): cdp.Network.SetCookieRequest => { + // this logic forms a SetCookie request that will be received by Chrome + // see MakeCookieFromProtocolValues for information on how this cookie data will be parsed + // @see https://cs.chromium.org/chromium/src/content/browser/devtools/protocol/network_handler.cc?l=246&rcl=786a9194459684dc7a6fded9cabfc0c9b9b37174 + + const setCookieRequest: cdp.Network.SetCookieRequest = _({ + domain: cookie.domain, + path: cookie.path, + secure: cookie.secure, + httpOnly: cookie.httpOnly, + sameSite: convertSameSiteExtensionToCdp(cookie.sameSite), + expires: cookie.expirationDate, + }) + // Network.setCookie will error on any undefined/null parameters + .omitBy(_.isNull) + .omitBy(_.isUndefined) + // set name and value at the end to get the correct typing + .extend({ + name: cookie.name || '', + value: cookie.value || '', + }) + .value() + + // without this logic, a cookie being set on 'foo.com' will only be set for 'foo.com', not other subdomains + if (!cookie.hostOnly && cookie.domain[0] !== '.') { + const parsedDomain = cors.parseDomain(cookie.domain) + + // normally, a non-hostOnly cookie should be prefixed with a . + // so if it's not a top-level domain (localhost, ...) or IP address + // prefix it with a . so it becomes a non-hostOnly cookie + if (parsedDomain && parsedDomain.tld !== cookie.domain) { + setCookieRequest.domain = `.${cookie.domain}` } + } - // @ts-ignore - cookie.sameSite = convertSameSiteCdpToExtension(cookie.sameSite) + if (setCookieRequest.name.startsWith('__Host-')) { + setCookieRequest.url = `https://${cookie.domain}` + delete setCookieRequest.domain + } - // @ts-ignore - cookie.expirationDate = cookie.expires - // @ts-ignore - delete cookie.expires + return setCookieRequest +} - // @ts-ignore - return cookie +const normalizeResourceType = (resourceType: string | undefined): ResourceType => { + resourceType = resourceType ? resourceType.toLowerCase() : 'unknown' + if (validResourceTypes.includes(resourceType as ResourceType)) { + return resourceType as ResourceType } - const normalizeGetCookies = (cookies: cdp.Network.Cookie[]) => { - return _.map(cookies, normalizeGetCookieProps) + if (resourceType === 'img') { + return 'image' } - const normalizeSetCookieProps = (cookie: CyCookie): cdp.Network.SetCookieRequest => { - // this logic forms a SetCookie request that will be received by Chrome - // see MakeCookieFromProtocolValues for information on how this cookie data will be parsed - // @see https://cs.chromium.org/chromium/src/content/browser/devtools/protocol/network_handler.cc?l=246&rcl=786a9194459684dc7a6fded9cabfc0c9b9b37174 + return ffToStandardResourceTypeMap[resourceType] || 'other' +} - const setCookieRequest: cdp.Network.SetCookieRequest = _({ - domain: cookie.domain, - path: cookie.path, - secure: cookie.secure, - httpOnly: cookie.httpOnly, - sameSite: convertSameSiteExtensionToCdp(cookie.sameSite), - expires: cookie.expirationDate, - }) - // Network.setCookie will error on any undefined/null parameters - .omitBy(_.isNull) - .omitBy(_.isUndefined) - // set name and value at the end to get the correct typing - .extend({ - name: cookie.name || '', - value: cookie.value || '', +type SendDebuggerCommand = (message: string, data?: any) => Bluebird +type OnFn = (eventName: string, cb: Function) => void + +// the intersection of what's valid in CDP and what's valid in FFCDP +// Firefox: https://searchfox.org/mozilla-central/rev/98a9257ca2847fad9a19631ac76199474516b31e/remote/cdp/domains/parent/Network.jsm#22 +// CDP: https://chromedevtools.github.io/devtools-protocol/tot/Network/#type-ResourceType +const validResourceTypes: ResourceType[] = ['fetch', 'xhr', 'websocket', 'stylesheet', 'script', 'image', 'font', 'cspviolationreport', 'ping', 'manifest', 'other'] +const ffToStandardResourceTypeMap: { [ff: string]: ResourceType } = { + 'img': 'image', + 'csp': 'cspviolationreport', + 'webmanifest': 'manifest', +} + +export class CdpAutomation { + constructor (private sendDebuggerCommandFn: SendDebuggerCommand, onFn: OnFn, private automation: Automation) { + onFn('Network.requestWillBeSent', this.onNetworkRequestWillBeSent) + sendDebuggerCommandFn('Network.enable', { + maxTotalBufferSize: 0, + maxResourceBufferSize: 0, + maxPostDataSize: 0, }) - .value() - - // without this logic, a cookie being set on 'foo.com' will only be set for 'foo.com', not other subdomains - if (!cookie.hostOnly && cookie.domain[0] !== '.') { - const parsedDomain = cors.parseDomain(cookie.domain) - - // normally, a non-hostOnly cookie should be prefixed with a . - // so if it's not a top-level domain (localhost, ...) or IP address - // prefix it with a . so it becomes a non-hostOnly cookie - if (parsedDomain && parsedDomain.tld !== cookie.domain) { - setCookieRequest.domain = `.${cookie.domain}` - } - } + } + + private onNetworkRequestWillBeSent = (params: cdp.Network.RequestWillBeSentEvent) => { + let url = params.request.url - if (setCookieRequest.name.startsWith('__Host-')) { - setCookieRequest.url = `https://${cookie.domain}` - delete setCookieRequest.domain + // in Firefox, the hash is incorrectly included in the URL: https://bugzilla.mozilla.org/show_bug.cgi?id=1715366 + if (url.includes('#')) url = url.slice(0, url.indexOf('#')) + + // Firefox: https://searchfox.org/mozilla-central/rev/98a9257ca2847fad9a19631ac76199474516b31e/remote/cdp/domains/parent/Network.jsm#397 + // Firefox lacks support for urlFragment and initiator, two nice-to-haves + const browserPreRequest: BrowserPreRequest = { + requestId: params.requestId, + method: params.request.method, + url, + resourceType: normalizeResourceType(params.type), + originalResourceType: params.type, } - return setCookieRequest + this.automation.onBrowserPreRequest(browserPreRequest) } - const getAllCookies = (filter: CyCookieFilter) => { - return sendDebuggerCommandFn('Network.getAllCookies') + private getAllCookies = (filter: CyCookieFilter) => { + return this.sendDebuggerCommandFn('Network.getAllCookies') .then((result: cdp.Network.GetAllCookiesResponse) => { return normalizeGetCookies(result.cookies) .filter((cookie: CyCookie) => { @@ -140,8 +196,8 @@ export const CdpAutomation = (sendDebuggerCommandFn: SendDebuggerCommand) => { }) } - const getCookiesByUrl = (url): Bluebird => { - return sendDebuggerCommandFn('Network.getCookies', { + private getCookiesByUrl = (url): Bluebird => { + return this.sendDebuggerCommandFn('Network.getCookies', { urls: [url], }) .then((result: cdp.Network.GetCookiesResponse) => { @@ -152,29 +208,29 @@ export const CdpAutomation = (sendDebuggerCommandFn: SendDebuggerCommand) => { }) } - const getCookie = (filter: CyCookieFilter): Bluebird => { - return getAllCookies(filter) + private getCookie = (filter: CyCookieFilter): Bluebird => { + return this.getAllCookies(filter) .then((cookies) => { return _.get(cookies, 0, null) }) } - const onRequest = (message, data) => { + onRequest = (message, data) => { let setCookie switch (message) { case 'get:cookies': if (data.url) { - return getCookiesByUrl(data.url) + return this.getCookiesByUrl(data.url) } - return getAllCookies(data) + return this.getAllCookies(data) case 'get:cookie': - return getCookie(data) + return this.getCookie(data) case 'set:cookie': setCookie = normalizeSetCookieProps(data) - return sendDebuggerCommandFn('Network.setCookie', setCookie) + return this.sendDebuggerCommandFn('Network.setCookie', setCookie) .then((result: cdp.Network.SetCookieResponse) => { if (!result.success) { // i wish CDP provided some more detail here, but this is really it in v1.3 @@ -182,10 +238,10 @@ export const CdpAutomation = (sendDebuggerCommandFn: SendDebuggerCommand) => { throw new Error(`Network.setCookie failed to set cookie: ${JSON.stringify(setCookie)}`) } - return getCookie(data) + return this.getCookie(data) }) case 'clear:cookie': - return getCookie(data) + return this.getCookie(data) // tap, so we can resolve with the value of the removed cookie // also, getting the cookie via CDP first will ensure that we send a cookie `domain` to CDP // that matches the cookie domain that is really stored @@ -194,14 +250,14 @@ export const CdpAutomation = (sendDebuggerCommandFn: SendDebuggerCommand) => { return } - return sendDebuggerCommandFn('Network.deleteCookies', _.pick(cookieToBeCleared, 'name', 'domain')) + return this.sendDebuggerCommandFn('Network.deleteCookies', _.pick(cookieToBeCleared, 'name', 'domain')) }) case 'is:automation:client:connected': return true case 'remote:debugger:protocol': - return sendDebuggerCommandFn(data.command, data.params) + return this.sendDebuggerCommandFn(data.command, data.params) case 'take:screenshot': - return sendDebuggerCommandFn('Page.captureScreenshot', { format: 'png' }) + return this.sendDebuggerCommandFn('Page.captureScreenshot', { format: 'png' }) .catch((err) => { throw new Error(`The browser responded with an error when Cypress attempted to take a screenshot.\n\nDetails:\n${err.message}`) }) @@ -212,6 +268,4 @@ export const CdpAutomation = (sendDebuggerCommandFn: SendDebuggerCommand) => { throw new Error(`No automation handler registered for: '${message}'`) } } - - return { onRequest } } diff --git a/packages/server/lib/browsers/chrome.ts b/packages/server/lib/browsers/chrome.ts index ab916a51353e..4ccebd83d798 100644 --- a/packages/server/lib/browsers/chrome.ts +++ b/packages/server/lib/browsers/chrome.ts @@ -177,12 +177,6 @@ const _writeChromePreferences = (userDir: string, originalPrefs: ChromePreferenc .return() } -const getRemoteDebuggingPort = async () => { - const port = Number(process.env.CYPRESS_REMOTE_DEBUGGING_PORT) - - return port || utils.getPort() -} - /** * Merge the different `--load-extension` arguments into one. * @@ -252,13 +246,13 @@ const _disableRestorePagesPrompt = function (userDir) { // After the browser has been opened, we can connect to // its remote interface via a websocket. -const _connectToChromeRemoteInterface = function (port, onError) { +const _connectToChromeRemoteInterface = function (port, onError, browserDisplayName) { // @ts-ignore la(check.userPort(port), 'expected port number to connect CRI to', port) debug('connecting to Chrome remote interface at random port %d', port) - return protocol.getWsTargetFor(port) + return protocol.getWsTargetFor(port, browserDisplayName) .then((wsUrl) => { debug('received wsUrl %s for port %d', wsUrl, port) @@ -338,7 +332,7 @@ const _handleDownloads = async function (client, dir, automation) { const _setAutomation = (client, automation) => { return automation.use( - CdpAutomation(client.send), + new CdpAutomation(client.send, client.on, automation), ) } @@ -451,7 +445,7 @@ export = { const userDir = utils.getProfileDir(browser, isTextTerminal) const [port, preferences] = await Bluebird.all([ - getRemoteDebuggingPort(), + protocol.getRemoteDebuggingPort(), _getChromePreferences(userDir), ]) @@ -504,7 +498,7 @@ export = { // SECOND connect to the Chrome remote interface // and when the connection is ready // navigate to the actual url - const criClient = await this._connectToChromeRemoteInterface(port, options.onError) + const criClient = await this._connectToChromeRemoteInterface(port, options.onError, browser.displayName) la(criClient, 'expected Chrome remote interface reference', criClient) diff --git a/packages/server/lib/browsers/cri-client.ts b/packages/server/lib/browsers/cri-client.ts index 82e7c9fa947c..41a84bb84b5a 100644 --- a/packages/server/lib/browsers/cri-client.ts +++ b/packages/server/lib/browsers/cri-client.ts @@ -30,12 +30,14 @@ namespace CRI { 'Page.navigate' | 'Page.startScreencast' | 'Page.screencastFrameAck' | - 'Page.setDownloadBehavior' + 'Page.setDownloadBehavior' | + string export type EventName = 'Page.screencastFrame' | 'Page.downloadWillBegin' | - 'Page.downloadProgress' + 'Page.downloadProgress' | + string } /** diff --git a/packages/server/lib/browsers/electron.js b/packages/server/lib/browsers/electron.js index 9eb2e2a7c398..3fb42ae21532 100644 --- a/packages/server/lib/browsers/electron.js +++ b/packages/server/lib/browsers/electron.js @@ -35,7 +35,7 @@ const tryToCall = function (win, method) { } } -const _getAutomation = function (win, options) { +const _getAutomation = function (win, options, parent) { const sendCommand = Bluebird.method((...args) => { return tryToCall(win, () => { return win.webContents.debugger.sendCommand @@ -43,7 +43,15 @@ const _getAutomation = function (win, options) { }) }) - const automation = CdpAutomation(sendCommand) + const on = (eventName, cb) => { + win.webContents.debugger.on('message', (event, method, params) => { + if (method === eventName) { + cb(params) + } + }) + } + + const automation = new CdpAutomation(sendCommand, on, parent) if (!options.onScreencastFrame) { // after upgrading to Electron 8, CDP screenshots can hang if a screencast is not also running @@ -177,10 +185,9 @@ module.exports = { win.maximize() } - automation.use(_getAutomation(win, preferences)) - return this._launch(win, url, automation, preferences) .tap(_maybeRecordVideo(win.webContents, preferences)) + .tap(() => automation.use(_getAutomation(win, preferences, automation))) }, _launchChild (e, url, parent, projectRoot, state, options, automation) { diff --git a/packages/server/lib/browsers/firefox-util.ts b/packages/server/lib/browsers/firefox-util.ts index 13066c3eefa1..62d9d9750708 100644 --- a/packages/server/lib/browsers/firefox-util.ts +++ b/packages/server/lib/browsers/firefox-util.ts @@ -6,6 +6,8 @@ import { Command } from 'marionette-client/lib/marionette/message.js' import util from 'util' import Foxdriver from '@benmalka/foxdriver' import * as protocol from './protocol' +import { CdpAutomation } from './cdp_automation' +import * as CriClient from './cri-client' const errors = require('../errors') @@ -93,6 +95,13 @@ const attachToTabMemory = Bluebird.method((tab) => { }) }) +async function setupRemote (remotePort, automation, onError) { + const wsUrl = await protocol.getWsTargetFor(remotePort, 'Firefox') + const criClient = await CriClient.create(wsUrl, onError) + + new CdpAutomation(criClient.send, criClient.on, automation) +} + const logGcDetails = () => { const reducedTimings = { ...timings, @@ -158,14 +167,18 @@ export default { }, setup ({ + automation, extensions, + onError, url, marionettePort, foxdriverPort, + remotePort, }) { return Bluebird.all([ this.setupFoxdriver(foxdriverPort), this.setupMarionette(extensions, url, marionettePort), + remotePort && setupRemote(remotePort, automation, onError), ]) }, diff --git a/packages/server/lib/browsers/firefox.ts b/packages/server/lib/browsers/firefox.ts index 59022a91a59a..7d97645d19e8 100644 --- a/packages/server/lib/browsers/firefox.ts +++ b/packages/server/lib/browsers/firefox.ts @@ -15,6 +15,7 @@ import { EventEmitter } from 'events' import os from 'os' import treeKill from 'tree-kill' import mimeDb from 'mime-db' +import { getRemoteDebuggingPort } from './protocol' const errors = require('../errors') @@ -356,7 +357,9 @@ export function _createDetachedInstance (browserInstance: BrowserInstance): Brow return detachedInstance } -export async function open (browser: Browser, url, options: any = {}): Promise { +export async function open (browser: Browser, url, options: any = {}, automation): Promise { + // see revision comment here https://wiki.mozilla.org/index.php?title=WebDriver/RemoteProtocol&oldid=1234946 + const hasCdp = browser.majorVersion >= 86 const defaultLaunchOptions = utils.getDefaultLaunchOptions({ extensions: [] as string[], preferences: _.extend({}, defaultPreferences), @@ -369,6 +372,14 @@ export async function open (browser: Browser, url, options: any = {}): Promise { + try { + await firefoxUtil.setup({ automation, extensions: launchOptions.extensions, url, foxdriverPort, marionettePort, remotePort, onError: options.onError }) + } catch (err) { errors.throw('FIREFOX_COULD_NOT_CONNECT', err) - }) + } if (os.platform() === 'win32') { // override the .kill method for Windows so that the detached Firefox process closes between specs diff --git a/packages/server/lib/browsers/protocol.ts b/packages/server/lib/browsers/protocol.ts index 2ee5e1c9caed..850b0b330fb0 100644 --- a/packages/server/lib/browsers/protocol.ts +++ b/packages/server/lib/browsers/protocol.ts @@ -5,12 +5,13 @@ import Bluebird from 'bluebird' import la from 'lazy-ass' import Debug from 'debug' import { Socket } from 'net' +import utils from './utils' const errors = require('../errors') const is = require('check-more-types') const debug = Debug('cypress:server:browsers:protocol') -export function _getDelayMsForRetry (i) { +export function _getDelayMsForRetry (i, browserName) { if (i < 10) { return 100 } @@ -20,7 +21,7 @@ export function _getDelayMsForRetry (i) { } if (i < 63) { // after 5 seconds, begin logging and retrying - errors.warning('CDP_RETRYING_CONNECTION', i) + errors.warning('CDP_RETRYING_CONNECTION', i, browserName) return 1000 } @@ -73,11 +74,18 @@ const findStartPageTarget = (connectOpts) => { return CRI.List(_.clone(connectOpts)).then(findStartPage) } +export async function getRemoteDebuggingPort () { + const port = Number(process.env.CYPRESS_REMOTE_DEBUGGING_PORT) + + return port || utils.getPort() +} + /** * Waits for the port to respond with connection to Chrome Remote Interface * @param {number} port Port number to connect to + * @param {string} browserName Browser name, for warning/error messages */ -export const getWsTargetFor = (port) => { +export const getWsTargetFor = (port: number, browserName: string) => { debug('Getting WS connection to CRI on port %d', port) la(is.port(port), 'expected port number', port) @@ -91,7 +99,7 @@ export const getWsTargetFor = (port) => { getDelayMsForRetry: (i) => { retryIndex = i - return _getDelayMsForRetry(i) + return _getDelayMsForRetry(i, browserName) }, } @@ -103,7 +111,7 @@ export const getWsTargetFor = (port) => { return findStartPageTarget(connectOpts) .catch((err) => { retryIndex++ - const delay = _getDelayMsForRetry(retryIndex) + const delay = _getDelayMsForRetry(retryIndex, browserName) debug('error finding CRI target, maybe retrying %o', { delay, err }) @@ -120,6 +128,6 @@ export const getWsTargetFor = (port) => { }) .catch((err) => { debug('failed to connect to CDP %o', { connectOpts, err }) - errors.throw('CDP_COULD_NOT_CONNECT', port, err) + errors.throw('CDP_COULD_NOT_CONNECT', port, err, browserName) }) } diff --git a/packages/server/lib/errors.js b/packages/server/lib/errors.js index f9c77de130da..2b0de7396a10 100644 --- a/packages/server/lib/errors.js +++ b/packages/server/lib/errors.js @@ -843,7 +843,7 @@ const getMsgByType = function (type, arg1 = {}, arg2, arg3) { return stripIndent`\ Cypress failed to make a connection to the Chrome DevTools Protocol after retrying for 50 seconds. - This usually indicates there was a problem opening the Chrome browser. + This usually indicates there was a problem opening the ${arg3} browser. The CDP port requested was ${chalk.yellow(arg1)}. @@ -865,7 +865,7 @@ const getMsgByType = function (type, arg1 = {}, arg2, arg3) { ${arg1.stack}` case 'CDP_RETRYING_CONNECTION': - return `Failed to connect to Chrome, retrying in 1 second (attempt ${chalk.yellow(arg1)}/62)` + return `Failed to connect to ${arg2}, retrying in 1 second (attempt ${chalk.yellow(arg1)}/62)` case 'DEPRECATED_BEFORE_BROWSER_LAUNCH_ARGS': return stripIndent`\ Deprecation Warning: The \`before:browser:launch\` plugin event changed its signature in version \`4.0.0\` diff --git a/packages/server/lib/project-base.ts b/packages/server/lib/project-base.ts index 4d549e7cf22f..3066605f4f2a 100644 --- a/packages/server/lib/project-base.ts +++ b/packages/server/lib/project-base.ts @@ -373,7 +373,11 @@ export class ProjectBase extends EE { reporter = Reporter.create(reporter, cfg.reporterOptions, projectRoot) } - this._automation = new Automation(cfg.namespace, cfg.socketIoCookie, cfg.screenshotsFolder) + const onBrowserPreRequest = (browserPreRequest) => { + this.server.addBrowserPreRequest(browserPreRequest) + } + + this._automation = new Automation(cfg.namespace, cfg.socketIoCookie, cfg.screenshotsFolder, onBrowserPreRequest) this.server.startWebsockets(this.automation, cfg, { onReloadBrowser: options.onReloadBrowser, @@ -439,6 +443,16 @@ export class ProjectBase extends EE { this.server.changeToUrl(url) } + shouldCorrelatePreRequests = () => { + if (!this.browser) { + return false + } + + const { family, majorVersion } = this.browser + + return family === 'chromium' || (family === 'firefox' && majorVersion >= 86) + } + setCurrentSpecAndBrowser (spec, browser: Cypress.Browser) { this.spec = spec this.browser = browser diff --git a/packages/server/lib/project-e2e.ts b/packages/server/lib/project-e2e.ts index aeeea311577c..54cd2edbc0fd 100644 --- a/packages/server/lib/project-e2e.ts +++ b/packages/server/lib/project-e2e.ts @@ -30,7 +30,7 @@ export class ProjectE2E extends ProjectBase { return updatedConfig }) .then((cfg) => { - return this.server.open(cfg, this, options.onError, options.onWarning) + return this.server.open(cfg, this, options.onError, options.onWarning, this.shouldCorrelatePreRequests) .then(([port, warning]) => { return { cfg, diff --git a/packages/server/lib/server-base.ts b/packages/server/lib/server-base.ts index 65f6a1a311f9..0ab1e52713fe 100644 --- a/packages/server/lib/server-base.ts +++ b/packages/server/lib/server-base.ts @@ -12,7 +12,7 @@ import url from 'url' import httpsProxy from '@packages/https-proxy' import { netStubbingState, NetStubbingState } from '@packages/net-stubbing' import { agent, cors, httpUtils, uri } from '@packages/network' -import { NetworkProxy } from '@packages/proxy' +import { NetworkProxy, BrowserPreRequest } from '@packages/proxy' import { SocketCt } from '@packages/server-ct' import errors from './errors' import logger from './logger' @@ -197,7 +197,7 @@ export class ServerBase { return e } - createNetworkProxy (config, getRemoteState) { + createNetworkProxy (config, getRemoteState, shouldCorrelatePreRequests) { const getFileServerToken = () => { return this._fileServer.token } @@ -206,6 +206,7 @@ export class ServerBase { // @ts-ignore this._networkProxy = new NetworkProxy({ config, + shouldCorrelatePreRequests, getRemoteState, getFileServerToken, socket: this.socket, @@ -236,6 +237,10 @@ export class ServerBase { }) } + addBrowserPreRequest (browserPreRequest: BrowserPreRequest) { + this.networkProxy.addPendingBrowserPreRequest(browserPreRequest) + } + _createHttpServer (app): DestroyableHttpServer { const svr = http.createServer(httpUtils.lenientOptions, app) diff --git a/packages/server/lib/server-e2e.ts b/packages/server/lib/server-e2e.ts index 0ff2f75d4e1d..0f0dd973aa17 100644 --- a/packages/server/lib/server-e2e.ts +++ b/packages/server/lib/server-e2e.ts @@ -53,7 +53,7 @@ export class ServerE2E extends ServerBase { this._urlResolver = null } - open (config: Record = {}, project, onError, onWarning) { + open (config: Record = {}, project, onError, onWarning, shouldCorrelatePreRequests) { debug('server open') la(_.isPlainObject(config), 'expected plain config object', config) @@ -70,11 +70,8 @@ export class ServerE2E extends ServerBase { return this._getRemoteState() } - this.createNetworkProxy(config, getRemoteState) + this.createNetworkProxy(config, getRemoteState, shouldCorrelatePreRequests) - // TODO: this does not look like a good idea - // since we would be spawning new workers on every - // open + close of a project... if (config.experimentalSourceRewriting) { createInitialWorkers() } @@ -423,9 +420,12 @@ export class ServerE2E extends ServerBase { if (matchesNetStubbingRoute(options)) { // TODO: this is being used to force cy.visits to be interceptable by network stubbing // however, network errors will be obsfucated by the proxying so this is not an ideal solution - _.assign(options, { + _.merge(options, { proxy: `http://127.0.0.1:${this._port()}`, agent: null, + headers: { + 'x-cypress-resolving-url': '1', + }, }) } diff --git a/packages/server/package.json b/packages/server/package.json index c8a1a43ede60..e79e412b907c 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -185,7 +185,7 @@ "ts-loader": "7.0.4", "tsconfig-paths": "3.9.0", "webpack": "4.43.0", - "ws": "5.2.2", + "ws": "5.2.3", "xvfb": "cypress-io/node-xvfb#22e3783c31d81ebe64d8c0df491ea00cdc74726a", "xvfb-maybe": "0.2.1" }, diff --git a/packages/server/test/unit/browsers/cdp_automation_spec.ts b/packages/server/test/unit/browsers/cdp_automation_spec.ts index bcf3319e356d..26c784a58ca1 100644 --- a/packages/server/test/unit/browsers/cdp_automation_spec.ts +++ b/packages/server/test/unit/browsers/cdp_automation_spec.ts @@ -69,8 +69,9 @@ context('lib/browsers/cdp_automation', () => { context('.CdpAutomation', () => { beforeEach(function () { this.sendDebuggerCommand = sinon.stub() + this.onFn = sinon.stub() - this.automation = CdpAutomation(this.sendDebuggerCommand) + this.automation = new CdpAutomation(this.sendDebuggerCommand, this.onFn) this.sendDebuggerCommand .throws(new Error('not stubbed')) diff --git a/packages/server/test/unit/browsers/chrome_spec.js b/packages/server/test/unit/browsers/chrome_spec.js index e7345b10146c..355431af1704 100644 --- a/packages/server/test/unit/browsers/chrome_spec.js +++ b/packages/server/test/unit/browsers/chrome_spec.js @@ -66,12 +66,13 @@ describe('lib/browsers/chrome', () => { return chrome.open('chrome', 'http://', {}, this.automation) .then(() => { expect(utils.getPort).to.have.been.calledOnce // to get remote interface port - expect(this.criClient.send.callCount).to.equal(4) + expect(this.criClient.send.callCount).to.equal(5) expect(this.criClient.send).to.have.been.calledWith('Page.bringToFront') expect(this.criClient.send).to.have.been.calledWith('Page.navigate') expect(this.criClient.send).to.have.been.calledWith('Page.enable') expect(this.criClient.send).to.have.been.calledWith('Page.setDownloadBehavior') + expect(this.criClient.send).to.have.been.calledWith('Network.enable') }) }) diff --git a/packages/server/test/unit/browsers/electron_spec.js b/packages/server/test/unit/browsers/electron_spec.js index 572131cd8f5a..a6044c24bb53 100644 --- a/packages/server/test/unit/browsers/electron_spec.js +++ b/packages/server/test/unit/browsers/electron_spec.js @@ -289,6 +289,7 @@ describe('lib/browsers/electron', () => { this.newWin = { maximize: sinon.stub(), setSize: sinon.stub(), + webContents: this.win.webContents, } this.preferences = { ...this.options } diff --git a/packages/server/test/unit/browsers/protocol_spec.ts b/packages/server/test/unit/browsers/protocol_spec.ts index 77b6996ea36f..99475110be18 100644 --- a/packages/server/test/unit/browsers/protocol_spec.ts +++ b/packages/server/test/unit/browsers/protocol_spec.ts @@ -25,7 +25,7 @@ describe('lib/browsers/protocol', () => { let delay: number let i = 0 - while ((delay = protocol._getDelayMsForRetry(i))) { + while ((delay = protocol._getDelayMsForRetry(i, 'FooBrowser'))) { delays.push(delay) i++ } @@ -35,7 +35,7 @@ describe('lib/browsers/protocol', () => { log.getCalls().forEach((log, i) => { const line = stripAnsi(log.args[0]) - expect(line).to.include(`Failed to connect to Chrome, retrying in 1 second (attempt ${i + 18}/62)`) + expect(line).to.include(`Failed to connect to FooBrowser, retrying in 1 second (attempt ${i + 18}/62)`) }) snapshot(delays) @@ -46,7 +46,7 @@ describe('lib/browsers/protocol', () => { const expectedCdpFailedError = stripIndents` Cypress failed to make a connection to the Chrome DevTools Protocol after retrying for 50 seconds. - This usually indicates there was a problem opening the Chrome browser. + This usually indicates there was a problem opening the FooBrowser browser. The CDP port requested was ${chalk.yellow('12345')}. @@ -57,7 +57,7 @@ describe('lib/browsers/protocol', () => { const innerErr = new Error('cdp connection failure') sinon.stub(connect, 'createRetryingSocket').callsArgWith(1, innerErr) - const p = protocol.getWsTargetFor(12345) + const p = protocol.getWsTargetFor(12345, 'FooBrowser') return expect(p).to.eventually.be.rejected .and.property('message').include(expectedCdpFailedError) @@ -77,7 +77,7 @@ describe('lib/browsers/protocol', () => { sinon.stub(connect, 'createRetryingSocket').callsArgWith(1, null, { end }) - const p = protocol.getWsTargetFor(12345) + const p = protocol.getWsTargetFor(12345, 'FooBrowser') return expect(p).to.eventually.be.rejected .and.property('message').include(expectedCdpFailedError) @@ -106,7 +106,7 @@ describe('lib/browsers/protocol', () => { sinon.stub(connect, 'createRetryingSocket').callsArgWith(1, null, { end }) - const p = protocol.getWsTargetFor(12345) + const p = protocol.getWsTargetFor(12345, 'FooBrowser') await expect(p).to.eventually.equal('bar') expect(end).to.be.calledOnce @@ -139,7 +139,7 @@ describe('lib/browsers/protocol', () => { .onSecondCall().resolves([]) .onThirdCall().resolves(targets) - const targetUrl = await protocol.getWsTargetFor(port) + const targetUrl = await protocol.getWsTargetFor(port, 'FooBrowser') expect(criList).to.have.been.calledThrice expect(targetUrl).to.equal('ws://debug-url') @@ -169,7 +169,7 @@ describe('lib/browsers/protocol', () => { .onSecondCall().resolves([]) .onThirdCall().resolves(targets) - const targetUrl = await protocol.getWsTargetFor(port) + const targetUrl = await protocol.getWsTargetFor(port, 'FooBrowser') expect(criList).to.have.been.calledThrice expect(targetUrl).to.equal('ws://debug-url') @@ -180,7 +180,7 @@ describe('lib/browsers/protocol', () => { log.getCalls().forEach((log, i) => { const line = stripAnsi(log.args[0]) - expect(line).to.include(`Failed to connect to Chrome, retrying in 1 second (attempt ${i + 18}/62)`) + expect(line).to.include(`Failed to connect to FooBrowser, retrying in 1 second (attempt ${i + 18}/62)`) }) }) }) diff --git a/packages/server/tsconfig.json b/packages/server/tsconfig.json index 22ca75091ddd..fbab78576fa0 100644 --- a/packages/server/tsconfig.json +++ b/packages/server/tsconfig.json @@ -8,6 +8,6 @@ "./../ts/index.d.ts" ], "compilerOptions": { - "types": ["mocha", "node", "chrome"] + "types": ["mocha", "node"] } } diff --git a/patches/istextorbinary+5.12.0.patch b/patches/istextorbinary+5.12.0.patch new file mode 100644 index 000000000000..704207c792e7 --- /dev/null +++ b/patches/istextorbinary+5.12.0.patch @@ -0,0 +1,145 @@ +diff --git a/node_modules/istextorbinary/edition-esnext/index.js b/node_modules/istextorbinary/edition-esnext/index.js +index 2781914..504ade7 100644 +--- a/node_modules/istextorbinary/edition-esnext/index.js ++++ b/node_modules/istextorbinary/edition-esnext/index.js +@@ -132,7 +132,20 @@ function getEncoding(buffer, opts) { + return encoding + } else { + // Extract +- const chunkEnd = Math.min(buffer.length, chunkBegin + chunkLength) ++ chunkBegin = getChunkBegin(buffer, chunkBegin) ++ if (chunkBegin === -1) { ++ return binaryEncoding ++ } ++ ++ const chunkEnd = getChunkEnd( ++ buffer, ++ Math.min(buffer.length, chunkBegin + chunkLength) ++ ) ++ ++ if (chunkEnd > buffer.length) { ++ return binaryEncoding ++ } ++ + const contentChunkUTF8 = buffer.toString(textEncoding, chunkBegin, chunkEnd) + // Detect encoding + for (let i = 0; i < contentChunkUTF8.length; ++i) { +@@ -149,3 +162,117 @@ function getEncoding(buffer, opts) { + } + } + exports.getEncoding = getEncoding ++ ++// The functions below are created to handle multibyte utf8 characters. ++// To understand how the encoding works, ++// check this article: https://en.wikipedia.org/wiki/UTF-8#Encoding ++ ++function getChunkBegin(buf, chunkBegin) { ++ // If it's the beginning, just return. ++ if (chunkBegin === 0) { ++ return 0 ++ } ++ ++ if (!isLaterByteOfUtf8(buf[chunkBegin])) { ++ return chunkBegin ++ } ++ ++ let begin = chunkBegin - 3 ++ ++ if (begin >= 0) { ++ if (isFirstByteOf4ByteChar(buf[begin])) { ++ return begin ++ } ++ } ++ ++ begin = chunkBegin - 2 ++ ++ if (begin >= 0) { ++ if ( ++ isFirstByteOf4ByteChar(buf[begin]) || ++ isFirstByteOf3ByteChar(buf[begin]) ++ ) { ++ return begin ++ } ++ } ++ ++ begin = chunkBegin - 1 ++ ++ if (begin >= 0) { ++ // Is it a 4-byte, 3-byte utf8 character? ++ if ( ++ isFirstByteOf4ByteChar(buf[begin]) || ++ isFirstByteOf3ByteChar(buf[begin]) || ++ isFirstByteOf2ByteChar(buf[begin]) ++ ) { ++ return begin ++ } ++ } ++ ++ return -1 ++} ++ ++function getChunkEnd(buf, chunkEnd) { ++ // If it's the end, just return. ++ if (chunkEnd === buf.length) { ++ return chunkEnd ++ } ++ ++ let index = chunkEnd - 3 ++ ++ if (index >= 0) { ++ if (isFirstByteOf4ByteChar(buf[index])) { ++ return chunkEnd + 1 ++ } ++ } ++ ++ index = chunkEnd - 2 ++ ++ if (index >= 0) { ++ if (isFirstByteOf4ByteChar(buf[index])) { ++ return chunkEnd + 2 ++ } ++ ++ if (isFirstByteOf3ByteChar(buf[index])) { ++ return chunkEnd + 1 ++ } ++ } ++ ++ index = chunkEnd - 1 ++ ++ if (index >= 0) { ++ if (isFirstByteOf4ByteChar(buf[index])) { ++ return chunkEnd + 3 ++ } ++ ++ if (isFirstByteOf3ByteChar(buf[index])) { ++ return chunkEnd + 2 ++ } ++ ++ if (isFirstByteOf2ByteChar(buf[index])) { ++ return chunkEnd + 1 ++ } ++ } ++ ++ return chunkEnd ++} ++ ++function isFirstByteOf4ByteChar(byte) { ++ // eslint-disable-next-line no-bitwise ++ return byte >> 3 === 30 // 11110xxx? ++} ++ ++function isFirstByteOf3ByteChar(byte) { ++ // eslint-disable-next-line no-bitwise ++ return byte >> 4 === 14 // 1110xxxx? ++} ++ ++function isFirstByteOf2ByteChar(byte) { ++ // eslint-disable-next-line no-bitwise ++ return byte >> 5 === 6 // 110xxxxx? ++} ++ ++function isLaterByteOfUtf8(byte) { ++ // eslint-disable-next-line no-bitwise ++ return byte >> 6 === 2 // 10xxxxxx? ++} +\ No newline at end of file diff --git a/yarn.lock b/yarn.lock index 6d5ccb31336e..85aa8ee4dc08 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2366,16 +2366,6 @@ lodash.merge "^4.6.2" lodash.omit "^4.5.0" -"@cypress/listr-verbose-renderer@^0.4.1": - version "0.4.1" - resolved "https://registry.yarnpkg.com/@cypress/listr-verbose-renderer/-/listr-verbose-renderer-0.4.1.tgz#a77492f4b11dcc7c446a34b3e28721afd33c642a" - integrity sha1-p3SS9LEdzHxEajSz4ochr9M8ZCo= - dependencies: - chalk "^1.1.3" - cli-cursor "^1.0.2" - date-fns "^1.27.2" - figures "^1.7.0" - "@cypress/mocha-teamcity-reporter@1.0.0": version "1.0.0" resolved "https://registry.yarnpkg.com/@cypress/mocha-teamcity-reporter/-/mocha-teamcity-reporter-1.0.0.tgz#efc8ab938c99f9654f438bef412bce1cd5e129d7" @@ -13624,13 +13614,6 @@ cli-columns@^3.1.2: string-width "^2.0.0" strip-ansi "^3.0.1" -cli-cursor@^1.0.2: - version "1.0.2" - resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-1.0.2.tgz#64da3f7d56a54412e59794bd62dc35295e8f2987" - integrity sha1-ZNo/fValRBLll5S9Ytw1KV6PKYc= - dependencies: - restore-cursor "^1.0.1" - cli-cursor@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-2.1.0.tgz#b35dac376479facc3e94747d41d0d0f5238ffcb5" @@ -15710,11 +15693,6 @@ date-fns@2.13.0: resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-2.13.0.tgz#d7b8a0a2d392e8d88a8024d0a46b980bbfdbd708" integrity sha512-xm0c61mevGF7f0XpCGtDTGpzEFC/1fpLXHbmFpxZZQJuvByIK2ozm6cSYuU+nxFYOPh2EuCfzUwlTEFwKG+h5w== -date-fns@^1.27.2: - version "1.30.1" - resolved "https://registry.yarnpkg.com/date-fns/-/date-fns-1.30.1.tgz#2e71bf0b119153dbb4cc4e88d9ea5acfb50dc05c" - integrity sha512-hBSVCvSmWC+QypYObzwGOd9wqdDpOt+0wl0KbU+R+uuZBS1jN8VsD1ss3irQDknRj5NvxiTF6oj/nDRnN/UQNw== - dateformat@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/dateformat/-/dateformat-2.2.0.tgz#4065e2013cf9fb916ddfd82efb506ad4c6769062" @@ -18286,11 +18264,6 @@ exif-parser@^0.1.12: resolved "https://registry.yarnpkg.com/exif-parser/-/exif-parser-0.1.12.tgz#58a9d2d72c02c1f6f02a0ef4a9166272b7760922" integrity sha1-WKnS1ywCwfbwKg70qRZicrd2CSI= -exit-hook@^1.0.0: - version "1.1.1" - resolved "https://registry.yarnpkg.com/exit-hook/-/exit-hook-1.1.1.tgz#f05ca233b48c05d54fff07765df8507e95c02ff8" - integrity sha1-8FyiM7SMBdVP/wd2XfhQfpXAL/g= - exit-on-epipe@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/exit-on-epipe/-/exit-on-epipe-1.0.1.tgz#0bdd92e87d5285d267daa8171d0eb06159689692" @@ -18717,14 +18690,6 @@ figgy-pudding@^3.4.1, figgy-pudding@^3.5.1: resolved "https://registry.yarnpkg.com/figgy-pudding/-/figgy-pudding-3.5.2.tgz#b4eee8148abb01dcf1d1ac34367d59e12fa61d6e" integrity sha512-0btnI/H8f2pavGMN8w40mlSKOfTK2SVJmBfBeVIj3kNw0swwgzyRq0d5TJVOwodFmtvpPeWPN/MCcfuWF0Ezbw== -figures@^1.7.0: - version "1.7.0" - resolved "https://registry.yarnpkg.com/figures/-/figures-1.7.0.tgz#cbe1e3affcf1cd44b80cadfed28dc793a9701d2e" - integrity sha1-y+Hjr/zxzUS4DK3+0o3Hk6lwHS4= - dependencies: - escape-string-regexp "^1.0.5" - object-assign "^4.1.0" - figures@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/figures/-/figures-2.0.0.tgz#3ab1a2d2a62c8bfb431a0c94cb797a2fce27c962" @@ -28574,11 +28539,6 @@ once@1.x, once@^1.3.0, once@^1.3.1, once@^1.3.2, once@^1.3.3, once@^1.4.0, once@ dependencies: wrappy "1" -onetime@^1.0.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/onetime/-/onetime-1.1.0.tgz#a1f7838f8314c516f05ecefcbc4ccfe04b4ed789" - integrity sha1-ofeDj4MUxRbwXs78vEzP4EtO14k= - onetime@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/onetime/-/onetime-2.0.1.tgz#067428230fd67443b2794b22bba528b6867962d4" @@ -33577,14 +33537,6 @@ resq@1.8.0: dependencies: fast-deep-equal "^2.0.1" -restore-cursor@^1.0.1: - version "1.0.1" - resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-1.0.1.tgz#34661f46886327fed2991479152252df92daa541" - integrity sha1-NGYfRohjJ/7SmRR5FSJS35LapUE= - dependencies: - exit-hook "^1.0.0" - onetime "^1.0.0" - restore-cursor@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-2.0.0.tgz#9f7ee287f82fd326d4fd162923d62129eee0dfaf" @@ -40254,7 +40206,14 @@ ws@3.3.x, ws@^3.2.0: safe-buffer "~5.1.0" ultron "~1.1.0" -ws@5.2.2, ws@^5.2.0: +ws@5.2.3: + version "5.2.3" + resolved "https://registry.yarnpkg.com/ws/-/ws-5.2.3.tgz#05541053414921bc29c63bee14b8b0dd50b07b3d" + integrity sha512-jZArVERrMsKUatIdnLzqvcfydI85dvd/Fp1u/VOpfdDWQ4c9qWXe+VIeAbQ5FrDwciAkr+lzofXLz3Kuf26AOA== + dependencies: + async-limiter "~1.0.0" + +ws@^5.2.0: version "5.2.2" resolved "https://registry.yarnpkg.com/ws/-/ws-5.2.2.tgz#dffef14866b8e8dc9133582514d1befaf96e980f" integrity sha512-jaHFD6PFv6UgoIVda6qZllptQsMlDEJkTQcybzzXDYM1XO9Y8em691FGMPmM46WGyLU4z9KMgQN+qrux/nhlHA==