Merge branch 'develop' into feature-multidomain

browser-versions.json

@@ -1,4 +1,4 @@
 {
-  "chrome:beta": "92.0.4515.40",
-  "chrome:stable": "91.0.4472.77"
+  "chrome:beta": "92.0.4515.51",
+  "chrome:stable": "91.0.4472.101"
 }

circle.yml

@@ -975,14 +975,14 @@ jobs:
       - run: yarn test-mocha
       # test binary build code
       - run: yarn test-scripts
-      # check for compile errors with the releaserc scripts
-      - run: yarn test-npm-package-release-script
       # make sure our snapshots are compared correctly
       - run: yarn test-mocha-snapshot
       # make sure packages with TypeScript can be transpiled to JS
       - run: yarn lerna run build-prod --stream
       # run unit tests from each individual package
       - run: yarn test
+      # check for compile errors with the releaserc scripts
+      - run: yarn test-npm-package-release-script
       - verify-mocha-results:
           expectedResultCount: 9
       - store_test_results:

packages/driver/cypress/integration/commands/net_stubbing_spec.ts

@@ -480,6 +480,54 @@ describe('network stubbing', { retries: { runMode: 2, openMode: 0 } }, function
           }).visit('/dump-method').contains('GET')
         })
       })
+
+      // https://github.com/cypress-io/cypress/issues/16292
+      describe('multibyte utf8', () => {
+        const multibyteUtf8 = [
+          // 1. When there's problem in the chunkEnd
+          // * 2 bytes
+          '12345678901234567890123Ф',
+          // * 3 bytes
+          '12345678901234567890123안',
+          '1234567890123456789012안',
+          // * 4 bytes
+          '12345678901234567890123😀',
+          '1234567890123456789012😀',
+          '123456789012345678901😀',
+          // 2. When there's a problem in the chunkBegin
+          // * 2 bytes
+          'dummyФ12345678901234567890123',
+          // * 3 bytes
+          'dummy안12345678901234567890123',
+          'dummy안1234567890123456789012',
+          // * 4 bytes
+          'dummy😀12345678901234567890123',
+          'dummy😀1234567890123456789012',
+          'dummy😀123456789012345678901',
+        ]
+
+        multibyteUtf8.forEach((str) => {
+          it(str, () => {
+            cy.intercept('https://example.com/test', {
+              body: { result: 'ok' },
+            }).as('testRequest')
+
+            cy.window().then(() => {
+              let xhr = new XMLHttpRequest()
+
+              xhr.open('POST', 'https://example.com/test')
+              xhr.setRequestHeader('Content-Type', 'application/json')
+              xhr.send(str)
+            })
+
+            cy.wait('@testRequest')
+            .its('request')
+            .then((req) => {
+              expect(req.body).to.eq(str)
+            })
+          })
+        })
+      })
     })
 
     context('logging', function () {

packages/net-stubbing/lib/external-types.ts

@@ -79,7 +79,7 @@ export namespace CyHttpMessages {
     /**
      * The headers of the HTTP message.
      */
-    headers: { [key: string]: string }
+    headers: { [key: string]: string | string[] }
   }
 
   export type IncomingResponse = BaseMessage & {

packages/net-stubbing/lib/server/util.ts

@@ -241,7 +241,8 @@ export function getBodyEncoding (req: CyHttpMessages.IncomingRequest): BodyEncod
 
   // a simple heuristic for detecting UTF8 encoded requests
   if (req.headers && req.headers['content-type']) {
-    const contentType = req.headers['content-type'].toLowerCase()
+    const contentTypeHeader = req.headers['content-type'] as string
+    const contentType = contentTypeHeader.toLowerCase()
 
     if (contentType.includes('charset=utf-8') || contentType.includes('charset="utf-8"')) {
       return 'utf8'

packages/proxy/lib/http/index.ts

@@ -3,10 +3,12 @@ import CyServer from '@packages/server'
 import {
   CypressIncomingRequest,
   CypressOutgoingResponse,
+  BrowserPreRequest,
 } from '@packages/proxy'
-import debugModule from 'debug'
+import Debug from 'debug'
 import ErrorMiddleware from './error-middleware'
 import { HttpBuffers } from './util/buffers'
+import { GetPreRequestCb, PreRequests } from './util/prerequests'
 import { IncomingMessage } from 'http'
 import { NetStubbingState } from '@packages/net-stubbing'
 import Bluebird from 'bluebird'
@@ -16,7 +18,7 @@ import RequestMiddleware from './request-middleware'
 import ResponseMiddleware from './response-middleware'
 import { DeferredSourceMapCache } from '@packages/rewriter'
 
-const debug = debugModule('cypress:proxy:http')
+const debugRequests = Debug('cypress-verbose:proxy:http')
 
 export enum HttpStages {
   IncomingRequest,
@@ -35,9 +37,12 @@ export type HttpMiddlewareStacks = {
 type HttpMiddlewareCtx<T> = {
   req: CypressIncomingRequest
   res: CypressOutgoingResponse
-
+  shouldCorrelatePreRequests: () => boolean
+  stage: HttpStages
+  debug: Debug.Debugger
   middleware: HttpMiddlewareStacks
   deferSourceMapRewrite: (opts: { js: string, url: string }) => string
+  getPreRequest: (cb: GetPreRequestCb) => void
 } & T
 
 export const defaultMiddleware = {
@@ -48,6 +53,7 @@ export const defaultMiddleware = {
 
 export type ServerCtx = Readonly<{
   config: CyServer.Config
+  shouldCorrelatePreRequests?: () => boolean
   getFileServerToken: () => string
   getRemoteState: CyServer.getRemoteState
   netStubbingState: NetStubbingState
@@ -82,10 +88,8 @@ type HttpMiddlewareThis<T> = HttpMiddlewareCtx<T> & ServerCtx & Readonly<{
   skipMiddleware: (name: string) => void
 }>
 
-export function _runStage (type: HttpStages, ctx: any) {
-  const stage = HttpStages[type]
-
-  debug('Entering stage %o', { stage })
+export function _runStage (type: HttpStages, ctx: any, onError) {
+  ctx.stage = HttpStages[type]
 
   const runMiddlewareStack = () => {
     const middlewares = ctx.middleware[type]
@@ -131,8 +135,6 @@ export function _runStage (type: HttpStages, ctx: any) {
         return resolve()
       }
 
-      debug('Running middleware %o', { stage, middlewareName })
-
       const fullCtx = {
         next: () => {
           copyChangedCtx()
@@ -147,21 +149,19 @@ export function _runStage (type: HttpStages, ctx: any) {
           _end()
         },
         onError: (error: Error) => {
-          debug('Error in middleware %o', { stage, middlewareName, error })
+          ctx.debug('Error in middleware %o', { middlewareName, error })
 
           if (type === HttpStages.Error) {
             return
           }
 
           ctx.error = error
-
-          _end(_runStage(HttpStages.Error, ctx))
+          onError(error)
+          _end(_runStage(HttpStages.Error, ctx, onError))
         },
-
         skipMiddleware: (name) => {
           ctx.middleware[type] = _.omit(ctx.middleware[type], name)
         },
-
         ...ctx,
       }
 
@@ -174,19 +174,18 @@ export function _runStage (type: HttpStages, ctx: any) {
   }
 
   return runMiddlewareStack()
-  .then(() => {
-    debug('Leaving stage %o', { stage })
-  })
 }
 
 export class Http {
   buffers: HttpBuffers
   config: CyServer.Config
+  shouldCorrelatePreRequests: () => boolean
   deferredSourceMapCache: DeferredSourceMapCache
   getFileServerToken: () => string
   getRemoteState: () => any
   middleware: HttpMiddlewareStacks
   netStubbingState: NetStubbingState
+  preRequests: PreRequests = new PreRequests()
   request: any
   socket: CyServer.Socket
 
@@ -195,6 +194,7 @@ export class Http {
     this.deferredSourceMapCache = new DeferredSourceMapCache(opts.request)
 
     this.config = opts.config
+    this.shouldCorrelatePreRequests = opts.shouldCorrelatePreRequests || (() => false)
     this.getFileServerToken = opts.getFileServerToken
     this.getRemoteState = opts.getRemoteState
     this.middleware = opts.middleware
@@ -213,27 +213,43 @@ export class Http {
       res,
       buffers: this.buffers,
       config: this.config,
+      shouldCorrelatePreRequests: this.shouldCorrelatePreRequests,
       getFileServerToken: this.getFileServerToken,
       getRemoteState: this.getRemoteState,
       request: this.request,
       middleware: _.cloneDeep(this.middleware),
       netStubbingState: this.netStubbingState,
       socket: this.socket,
+      debug: (formatter, ...args) => {
+        debugRequests(`%s %s %s ${formatter}`, ctx.req.method, ctx.req.proxiedUrl, ctx.stage, ...args)
+      },
       deferSourceMapRewrite: (opts) => {
         this.deferredSourceMapCache.defer({
           resHeaders: ctx.incomingRes.headers,
           ...opts,
         })
       },
+      getPreRequest: (cb) => {
+        this.preRequests.get(ctx.req, ctx.debug, cb)
+      },
+    }
+
+    const onError = () => {
+      if (ctx.req.browserPreRequest) {
+        // browsers will retry requests in the event of network errors, but they will not send pre-requests,
+        // so try to re-use the current browserPreRequest for the next retry
+        ctx.debug('Re-using pre-request data %o', ctx.req.browserPreRequest)
+        this.addPendingBrowserPreRequest(ctx.req.browserPreRequest)
+      }
     }
 
-    return _runStage(HttpStages.IncomingRequest, ctx)
+    return _runStage(HttpStages.IncomingRequest, ctx, onError)
     .then(() => {
       if (ctx.incomingRes) {
-        return _runStage(HttpStages.IncomingResponse, ctx)
+        return _runStage(HttpStages.IncomingResponse, ctx, onError)
       }
 
-      return debug('warning: Request was not fulfilled with a response.')
+      return ctx.debug('Warning: Request was not fulfilled with a response.')
     })
   }
 
@@ -253,9 +269,14 @@ export class Http {
 
   reset () {
     this.buffers.reset()
+    this.preRequests = new PreRequests()
   }
 
   setBuffer (buffer) {
     return this.buffers.set(buffer)
   }
+
+  addPendingBrowserPreRequest (browserPreRequest: BrowserPreRequest) {
+    this.preRequests.addPending(browserPreRequest)
+  }
 }

packages/proxy/lib/http/request-middleware.ts

@@ -19,11 +19,40 @@ const LogRequest: RequestMiddleware = function () {
   this.next()
 }
 
+const CorrelateBrowserPreRequest: RequestMiddleware = async function () {
+  if (!this.shouldCorrelatePreRequests()) {
+    return this.next()
+  }
+
+  if (this.req.headers['x-cypress-resolving-url']) {
+    this.debug('skipping prerequest for resolve:url')
+    delete this.req.headers['x-cypress-resolving-url']
+
+    return this.next()
+  }
+
+  this.debug('waiting for prerequest')
+  this.getPreRequest(((browserPreRequest) => {
+    this.req.browserPreRequest = browserPreRequest
+    this.next()
+  }))
+}
+
+const SendToDriver: RequestMiddleware = function () {
+  const { browserPreRequest } = this.req
+
+  if (browserPreRequest) {
+    this.socket.toDriver('proxy:incoming:request', browserPreRequest)
+  }
+
+  this.next()
+}
+
 const MaybeEndRequestWithBufferedResponse: RequestMiddleware = function () {
   const buffer = this.buffers.take(this.req.proxiedUrl)
 
   if (buffer) {
-    debug('got a buffer %o', _.pick(buffer, 'url'))
+    this.debug('ending request with buffered response')
     this.res.wantsInjection = 'full'
 
     return this.onResponse(buffer.response, buffer.stream)
@@ -52,10 +81,7 @@ const EndRequestsToBlockedHosts: RequestMiddleware = function () {
 
     if (matches) {
       this.res.set('x-cypress-matched-blocked-host', matches)
-      debug('blocking request %o', {
-        url: this.req.proxiedUrl,
-        matches,
-      })
+      this.debug('blocking request %o', { matches })
 
       this.res.status(503).end()
 
@@ -127,7 +153,7 @@ const SendRequestOutgoing: RequestMiddleware = function () {
   req.on('error', this.onError)
   req.on('response', (incomingRes) => this.onResponse(incomingRes, req))
   this.req.on('aborted', () => {
-    debug('request aborted')
+    this.debug('request aborted')
     req.abort()
   })
 
@@ -141,6 +167,8 @@ const SendRequestOutgoing: RequestMiddleware = function () {
 
 export default {
   LogRequest,
+  CorrelateBrowserPreRequest,
+  SendToDriver,
   MaybeEndRequestWithBufferedResponse,
   InterceptRequest,
   RedirectToClientRouteIfUnloaded,