feat: CT stack traces (#23916)

Co-authored-by: Zachary Williams <zachjw34@gmail.com>
Co-authored-by: astone123 <adams@cypress.io>
Co-authored-by: Lachlan Miller <lachlan.miller.1990@outlook.com>

.circleci/config.yml

@@ -63,7 +63,7 @@ windowsWorkflowFilters: &windows-workflow-filters
     or:
     - equal: [ develop, << pipeline.git.branch >> ]
     - equal: [ linux-arm64, << pipeline.git.branch >> ]
-    - equal: [ 'lmiller/fixing-windows-ci', << pipeline.git.branch >> ]
+    - equal: [ 'mikep/21720-ct-stack-traces', << pipeline.git.branch >> ]
     - matches:
           pattern: "-release$"
           value: << pipeline.git.branch >>

npm/vite-dev-server/client/initCypressTests.js

@@ -24,12 +24,22 @@ if (supportFile) {
 
   // We need a slash before /cypress/supportFile.js, this happens by default
   // with the current string replacement logic.
-  importsToLoad.push(() => import(`${devServerPublicPathRoute}${supportRelativeToProjectRoot}`))
+  importsToLoad.push({
+    load: () => import(`${devServerPublicPathRoute}${supportRelativeToProjectRoot}`),
+    absolute: supportFile,
+    relative: supportRelativeToProjectRoot,
+    relativeUrl: `${devServerPublicPathRoute}${supportRelativeToProjectRoot}`,
+  })
 }
 
 /* Spec file import logic */
 // We need a slash before /src/my-spec.js, this does not happen by default.
-importsToLoad.push(() => import(`${devServerPublicPathRoute}/${CypressInstance.spec.relative}`))
+importsToLoad.push({
+  load: () => import(`${devServerPublicPathRoute}/${CypressInstance.spec.relative}`),
+  absolute: CypressInstance.spec.absolute,
+  relative: CypressInstance.spec.relative,
+  relativeUrl: `${devServerPublicPathRoute}/${CypressInstance.spec.relative}`,
+})
 
 if (!CypressInstance) {
   throw new Error('Tests cannot run without a reference to Cypress!')

npm/vite-dev-server/cypress/e2e/react.cy.ts

@@ -57,6 +57,7 @@ for (const project of VITE_REACT) {
       cy.contains('MissingReactInSpec.cy.jsx').click()
       cy.waitForSpecToFinish()
       cy.get('.failed > .num').should('contain', 1)
+      cy.get('.test-err-code-frame').should('be.visible')
       cy.withCtx(async (ctx) => {
         await ctx.actions.file.writeFileInProject(`src/MissingReactInSpec.cy.jsx`,
           await ctx.file.readFileInProject('src/App.cy.jsx'))

npm/vite-dev-server/src/plugins/cypress.ts

@@ -105,6 +105,27 @@ export const Cypress = (
         return res.end(transformedIndexHtml)
       })
     },
+    transform (code, id, options?) {
+      try {
+        if (/\.js$/i.test(id) && !/\/\/# sourceMappingURL=/i.test(code)) {
+          // The Vite dev server and plugins automatically transpile TS and JSX files, which results in sourcemaps being generated
+          // and included in output. However, Vite serves up `esnext`-compliant Javascript which means many JS files won't be
+          // transpiled and won't supply a sourcemap - this prevents Cypress from providing codeFrames in the event of an error.
+          //
+          // A sourcemap is generated by Vite for JS files (just not included) which is in effect an "identity" sourcemap mapping
+          // 1-to-1 to the output file. We can grab this and pass it along as a sourcemap we want Vite to embed into the output,
+          // giving Cypress a sourcemap to use for codeFrame lookups.
+          // @see https://rollupjs.org/guide/en/#thisgetcombinedsourcemap
+
+          return {
+            code,
+            map: this.getCombinedSourcemap(),
+          }
+        }
+      } catch (_err) {
+        debug('Failed to propagate sourcemap for %s: %o', id, _err)
+      }
+    },
     handleHotUpdate: ({ server, file }) => {
       debug('handleHotUpdate - file', file)
 

npm/webpack-dev-server/__snapshots__/makeWebpackConfig.spec.ts.js

@@ -10,17 +10,18 @@ exports['makeWebpackConfig ignores userland webpack `output.publicPath` and `dev
       "overlay": false
     }
   },
-  "mode": "development",
   "optimization": {
     "emitOnErrors": true,
     "splitChunks": {
       "chunks": "all"
     }
   },
+  "mode": "development",
   "plugins": [
     "HtmlWebpackPlugin",
     "CypressCTWebpackPlugin"
-  ]
+  ],
+  "devtool": "inline-source-map"
 }
 
 exports['makeWebpackConfig ignores userland webpack `output.publicPath` and `devServer.overlay` with webpack-dev-server v3 1'] = {
@@ -32,15 +33,16 @@ exports['makeWebpackConfig ignores userland webpack `output.publicPath` and `dev
     "progress": true,
     "overlay": false
   },
-  "mode": "development",
   "optimization": {
     "noEmitOnErrors": false,
     "splitChunks": {
       "chunks": "all"
     }
   },
+  "mode": "development",
   "plugins": [
     "HtmlWebpackPlugin",
     "CypressCTWebpackPlugin"
-  ]
+  ],
+  "devtool": "inline-source-map"
 }

npm/webpack-dev-server/cypress/e2e/angular.cy.ts

@@ -73,6 +73,7 @@ for (const project of WEBPACK_REACT) {
       // The test should fail and the stack trace should appear in the command log
       cy.waitForSpecToFinish({ failCount: 1 })
       cy.contains('The following error originated from your test code, not from Cypress.').should('exist')
+      cy.get('.test-err-code-frame').should('be.visible')
     })
 
     // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23455

npm/webpack-dev-server/cypress/e2e/create-react-app.cy.ts

@@ -38,6 +38,7 @@ for (const project of WEBPACK_REACT) {
       })
 
       cy.waitForSpecToFinish({ failCount: 1 })
+      cy.get('.test-err-code-frame').should('be.visible')
 
       cy.withCtx(async (ctx) => {
         await ctx.actions.file.writeFileInProject(

npm/webpack-dev-server/cypress/e2e/next.cy.ts

@@ -40,6 +40,7 @@ for (const project of WEBPACK_REACT) {
       })
 
       cy.waitForSpecToFinish({ failCount: 1 })
+      cy.get('.test-err-code-frame').should('be.visible')
 
       cy.withCtx(async (ctx) => {
         const indexTestPath = ctx.path.join('pages', 'index.cy.js')

npm/webpack-dev-server/cypress/e2e/nuxt.cy.ts

@@ -35,6 +35,7 @@ for (const project of PROJECTS) {
       })
 
       cy.waitForSpecToFinish({ failCount: 1 })
+      cy.get('.test-err-code-frame').should('be.visible')
 
       cy.withCtx(async (ctx) => {
         const tutorialCyPath = ctx.path.join('components', 'Tutorial.cy.js')

npm/webpack-dev-server/cypress/e2e/react.cy.ts

@@ -24,8 +24,7 @@ for (const project of WEBPACK_REACT) {
     it('should mount a passing test', () => {
       cy.visitApp()
       cy.contains('App.cy.jsx').click()
-      cy.waitForSpecToFinish()
-      cy.get('.passed > .num').should('contain', 1)
+      cy.waitForSpecToFinish({ passCount: 1 })
     })
 
     it('MissingReact: should fail, rerun, succeed', () => {
@@ -36,15 +35,15 @@ for (const project of WEBPACK_REACT) {
 
       cy.visitApp()
       cy.contains('MissingReact.cy.jsx').click()
-      cy.waitForSpecToFinish()
-      cy.get('.failed > .num').should('contain', 1)
+      cy.waitForSpecToFinish({ failCount: 1 })
+      cy.get('.test-err-code-frame').should('be.visible')
       cy.withCtx(async (ctx) => {
         await ctx.actions.file.writeFileInProject(`src/MissingReact.jsx`,
         `import React from 'react';
         ${await ctx.file.readFileInProject('src/MissingReact.jsx')}`)
       })
 
-      cy.get('.passed > .num').should('contain', 1)
+      cy.waitForSpecToFinish({ passCount: 1 })
     })
 
     it('MissingReactInSpec: should fail, rerun, succeed', () => {
@@ -55,14 +54,14 @@ for (const project of WEBPACK_REACT) {
 
       cy.visitApp()
       cy.contains('MissingReactInSpec.cy.jsx').click()
-      cy.waitForSpecToFinish()
-      cy.get('.failed > .num').should('contain', 1)
+      cy.waitForSpecToFinish({ failCount: 1 })
+      cy.get('.test-err-code-frame').should('be.visible')
       cy.withCtx(async (ctx) => {
         await ctx.actions.file.writeFileInProject(`src/MissingReactInSpec.cy.jsx`,
           await ctx.file.readFileInProject('src/App.cy.jsx'))
       })
 
-      cy.get('.passed > .num').should('contain', 1)
+      cy.waitForSpecToFinish({ passCount: 1 })
     })
 
     it('AppCompilationError: should fail with uncaught exception error', () => {
@@ -73,8 +72,7 @@ for (const project of WEBPACK_REACT) {
 
       cy.visitApp()
       cy.contains('AppCompilationError.cy.jsx').click()
-      cy.waitForSpecToFinish()
-      cy.get('.failed > .num').should('contain', 1)
+      cy.waitForSpecToFinish({ failCount: 1 })
       cy.contains('An uncaught error was detected outside of a test')
       cy.contains('The following error originated from your test code, not from Cypress.')
 
@@ -86,8 +84,7 @@ for (const project of WEBPACK_REACT) {
         )
       })
 
-      cy.waitForSpecToFinish()
-      cy.get('.passed > .num').should('contain', 1)
+      cy.waitForSpecToFinish({ passCount: 1 })
 
       const appCompilationErrorSpec = dedent`
         import React from 'react'
@@ -109,8 +106,7 @@ for (const project of WEBPACK_REACT) {
         )
       }, { appCompilationErrorSpec })
 
-      cy.waitForSpecToFinish()
-      cy.get('.failed > .num').should('contain', 1)
+      cy.waitForSpecToFinish({ failCount: 1 })
       cy.contains('An uncaught error was detected outside of a test')
       cy.contains('The following error originated from your test code, not from Cypress.')
     })
@@ -121,7 +117,7 @@ for (const project of WEBPACK_REACT) {
 
       // 1. assert spec executes successfully
       cy.contains('App.cy.jsx').click()
-      cy.get('.passed > .num').should('contain', 1)
+      cy.waitForSpecToFinish({ passCount: 1 })
 
       // 2. remove file from file system
       cy.withCtx(async (ctx) => {
@@ -143,7 +139,7 @@ for (const project of WEBPACK_REACT) {
       })
 
       // 5. assert recreated spec executes successfully
-      cy.get('.passed > .num').should('contain', 1)
+      cy.waitForSpecToFinish({ passCount: 1 })
     })
   })
 }

npm/webpack-dev-server/cypress/e2e/vue-cli.cy.ts

@@ -28,6 +28,36 @@ for (const project of PROJECTS) {
       })
     })
 
+    it('should live-reload on src changes', () => {
+      cy.visitApp()
+
+      cy.contains('HelloWorld.cy.js').click()
+      cy.waitForSpecToFinish({ passCount: 1 })
+
+      cy.withCtx(async (ctx) => {
+        const helloWorldVuePath = ctx.path.join('src', 'components', 'HelloWorld.vue')
+
+        await ctx.actions.file.writeFileInProject(
+          helloWorldVuePath,
+          (await ctx.file.readFileInProject(helloWorldVuePath)).replace('{{ msg }}', ''),
+        )
+      })
+
+      cy.waitForSpecToFinish({ failCount: 1 })
+      cy.get('.test-err-code-frame').should('be.visible')
+
+      cy.withCtx(async (ctx) => {
+        const helloWorldVuePath = ctx.path.join('src', 'components', 'HelloWorld.vue')
+
+        await ctx.actions.file.writeFileInProject(
+          helloWorldVuePath,
+          (await ctx.file.readFileInProject(helloWorldVuePath)).replace('<h1></h1>', '<h1>{{ msg }}</h1>'),
+        )
+      })
+
+      cy.waitForSpecToFinish({ passCount: 1 })
+    })
+
     it('should show compilation errors on src changes', () => {
       cy.visitApp()
 

npm/webpack-dev-server/src/helpers/angularHandler.ts

@@ -2,6 +2,7 @@ import * as fs from 'fs-extra'
 import { tmpdir } from 'os'
 import * as path from 'path'
 import { pathToFileURL } from 'url'
+import type { Configuration } from 'webpack'
 import type { PresetHandlerResult, WebpackDevServerConfig } from '../devServer'
 import { sourceDefaultWebpackDependencies } from './sourceRelativeWebpackModules'
 
@@ -253,8 +254,16 @@ async function getAngularCliWebpackConfig (devServerConfig: AngularWebpackDevSer
   return config
 }
 
+function removeSourceMapPlugin (config: Configuration) {
+  config.plugins = config.plugins?.filter((plugin) => {
+    return plugin?.constructor?.name !== 'SourceMapDevToolPlugin'
+  })
+}
+
 export async function angularHandler (devServerConfig: AngularWebpackDevServerConfig): Promise<PresetHandlerResult> {
   const webpackConfig = await getAngularCliWebpackConfig(devServerConfig)
 
+  removeSourceMapPlugin(webpackConfig)
+
   return { frameworkConfig: webpackConfig, sourceWebpackModulesResult: sourceDefaultWebpackDependencies(devServerConfig) }
 }

npm/webpack-dev-server/src/loader.ts

@@ -20,7 +20,9 @@ const makeImport = (file: Cypress.Cypress['spec'], filename: string, chunkName:
   return `"${filename}": {
     shouldLoad: () => document.location.pathname.includes("${encodeURI(file.absolute)}"),
     load: () => import("${file.absolute}" ${magicComments}),
-    chunkName: "${chunkName}",
+    absolute: "${file.absolute.split(path.sep).join(path.posix.sep)}",
+    relative: "${file.relative.split(path.sep).join(path.posix.sep)}",
+    relativeUrl: "/__cypress/src/${chunkName}.js",
   }`
 }
 
@@ -60,21 +62,32 @@ export default function loader (this: unknown) {
   const { files, projectRoot, supportFile } = ctx._cypress
 
   const supportFileAbsolutePath = supportFile ? JSON.stringify(path.resolve(projectRoot, supportFile)) : undefined
-
-  return `
-  var loadSupportFile = ${supportFile ? `() => import(${supportFileAbsolutePath})` : `() => Promise.resolve()`}
+  const supportFileRelativePath = supportFile ? JSON.stringify(path.relative(projectRoot, supportFileAbsolutePath || '')) : undefined
+  const result = `
   var allTheSpecs = ${buildSpecs(projectRoot, files)};
 
   var { init } = require(${JSON.stringify(require.resolve('./aut-runner'))})
 
   var scriptLoaders = Object.values(allTheSpecs).reduce(
     (accSpecLoaders, specLoader) => {
       if (specLoader.shouldLoad()) {
-        accSpecLoaders.push(specLoader.load)
+        accSpecLoaders.push(specLoader)
       }
       return accSpecLoaders
-  }, [loadSupportFile])
+  }, [])
+
+  if (${!!supportFile}) {
+    var supportFile = {
+      absolute: ${supportFileAbsolutePath},
+      relative: ${supportFileRelativePath},
+      relativeUrl: "/__cypress/src/cypress-support-file.js",
+      load: () => import(${supportFileAbsolutePath} /* webpackChunkName: "cypress-support-file" */),
+    }
+    scriptLoaders.unshift(supportFile)
+  }
 
   init(scriptLoaders)
   `
+
+  return result
 }

npm/webpack-dev-server/src/makeDefaultWebpackConfig.ts

@@ -51,6 +51,7 @@ export function makeDefaultWebpackConfig (
         ...(config.devServerConfig.framework === 'angular' ? { scriptLoading: 'module' } : {}),
       }),
     ],
+    devtool: 'inline-source-map',
   } as any
 
   if (config.sourceWebpackModulesResult.webpackDevServer.majorVersion === 4) {

npm/webpack-dev-server/src/makeWebpackConfig.ts

@@ -166,6 +166,11 @@ export async function makeWebpackConfig (
     dynamicWebpackConfig,
   )
 
+  // Some frameworks (like Next.js) change this value which changes the path we would need to use to fetch our spec.
+  // (eg, http://localhost:xxxx/<dev-server-public-path>/static/chunks/spec-<x>.js). Deleting this key to normalize
+  // the spec URL to `*/spec-<x>.js` which we need to know up-front so we can fetch the sourcemaps.
+  delete mergedConfig.output?.chunkFilename
+
   // Angular loads global styles and polyfills via script injection in the index.html
   if (framework === 'angular') {
     mergedConfig.entry = {

npm/webpack-dev-server/test/makeWebpackConfig.spec.ts

@@ -31,6 +31,7 @@ describe('makeWebpackConfig', () => {
         optimization: {
           noEmitOnErrors: true, // This will be overridden by makeWebpackConfig.ts
         },
+        devtool: 'eval', // This will be overridden by makeWebpackConfig.ts
       },
       devServerEvents: new EventEmitter(),
     }
@@ -77,6 +78,7 @@ describe('makeWebpackConfig', () => {
         optimization: {
           emitOnErrors: false, // This will be overridden by makeWebpackConfig.ts
         },
+        devtool: 'eval', // This will be overridden by makeWebpackConfig.ts
       },
       devServerEvents: new EventEmitter(),
     }

packages/app/cypress.config.ts

@@ -48,6 +48,7 @@ export default defineConfig({
       // Delete this as we only want to honor it on parent Cypress when doing E2E Cypress in Cypress testing
       delete process.env.HTTP_PROXY_TARGET_FOR_ORIGIN_REQUESTS
       process.env.CYPRESS_INTERNAL_E2E_TESTING_SELF = 'true'
+      process.env.CYPRESS_INTERNAL_VITE_OPEN_MODE_TESTING = 'true'
       // process.env.DEBUG = '*'
       const { e2ePluginSetup } = require('@packages/frontend-shared/cypress/e2e/e2ePluginSetup')