From 405e7f7ccba1831aa08da5915e7c45ca2ab9e6a3 Mon Sep 17 00:00:00 2001 From: Tyler Biethman Date: Mon, 12 Sep 2022 15:40:12 -0500 Subject: [PATCH] chore(webkit): update error stack parsing and related system tests (#23730) * chore(webkit): update error stack parsing and related system tests * Adding better comment * Putting column back. Indexing at 1. * Let's wait for WebKit fixes to land for this * Using default name/location values already used in stack enhancing logic * Incorrect bracket in regex * Trying without location, as the fake location causes more problems downstream. * Loosening regex around locations in stack replacement * Defaulting location sans row/column * Parsing stack lines for reporter with unique regex * D'oh * Making the validation against '' more specific * Don't want a capture group here * Updating spec_isolation system tests * Consolidating regex pattern to errors package * Can just keep this global now * Simplifying regex. Removing lineAndColumn numbers from unknown locations. * Updating system test stack regex * Getting better baseline * Revert "Updating system test stack regex" This reverts commit 2b91eff369b98ab3658fbc8825179adb9fd61221. * Forking normalization for webkit to track down diffs * Ensure line or column are set before rendering in enhanced stack * Need to be a little more flexible * Tweaking leading newline detection * Trying out new composed regex * Few more tweaks for multiple leading newlines and file locations without function name * Updating remainderOfStack pattern with proper escaping * Cleaning up comments * Filtering native code from absolute path logic * Rebuild CI after outage --- packages/driver/src/cypress/stack_utils.ts | 52 +- packages/errors/src/stackUtils.ts | 2 +- packages/reporter/src/errors/error-stack.tsx | 4 +- .../__snapshots__/spec_isolation_spec.js | 931 ++++++++++++++++++ system-tests/lib/normalizeStdout.ts | 42 +- .../cypress/e2e/uncaught_during_test.cy.js | 15 +- system-tests/test/async_timeouts_spec.js | 1 - .../test/caught_uncaught_hook_errors_spec.js | 4 - .../test/commands_outside_of_test_spec.js | 2 - system-tests/test/form_submissions_spec.js | 1 - system-tests/test/issue_173_spec.ts | 1 - system-tests/test/issue_674_spec.js | 1 - system-tests/test/js_error_handling_spec.js | 1 - system-tests/test/promises_spec.js | 1 - system-tests/test/runnable_execution_spec.ts | 2 - system-tests/test/spec_isolation_spec.js | 2 - system-tests/test/stdout_spec.js | 1 - system-tests/test/testConfigOverrides_spec.ts | 1 - .../test/uncaught_spec_errors_spec.js | 5 - system-tests/test/visit_spec.js | 4 - 20 files changed, 1023 insertions(+), 50 deletions(-) diff --git a/packages/driver/src/cypress/stack_utils.ts b/packages/driver/src/cypress/stack_utils.ts index 37c386a91b17..53307d0d02f3 100644 --- a/packages/driver/src/cypress/stack_utils.ts +++ b/packages/driver/src/cypress/stack_utils.ts @@ -8,12 +8,13 @@ import $utils from './utils' import $sourceMapUtils from './source_map_utils' // Intentionally deep-importing from @packages/errors so as to not bundle the entire @packages/errors in the client unnecessarily -import { getStackLines, replacedStack, stackWithoutMessage, splitStack, unsplitStack } from '@packages/errors/src/stackUtils' +import { getStackLines, replacedStack, stackWithoutMessage, splitStack, unsplitStack, stackLineRegex } from '@packages/errors/src/stackUtils' const whitespaceRegex = /^(\s*)*/ -const stackLineRegex = /^\s*(at )?.*@?\(?.*\:\d+\:\d+\)?$/ const customProtocolRegex = /^[^:\/]+:\/{1,3}/ const percentNotEncodedRegex = /%(?![0-9A-F][0-9A-F])/g +const webkitStackLineRegex = /(.*)@(.*)(\n?)/g + const STACK_REPLACEMENT_MARKER = '__stackReplacementMarker' const hasCrossFrameStacks = (specWindow) => { @@ -244,9 +245,7 @@ const cleanFunctionName = (functionName) => { } const parseLine = (line) => { - const isStackLine = stackLineRegex.test(line) - - if (!isStackLine) return + if (!stackLineRegex.test(line)) return const parsed = errorStackParser.parse({ stack: line } as any)[0] @@ -318,7 +317,14 @@ const getSourceDetailsForLine = (projectRoot, line): LineDetail => { let absoluteFile - if (relativeFile && projectRoot) { + // WebKit stacks may include an `` or `[native code]` location that is not navigable. + // We ensure that the absolute path is not set in this case. + + const canBuildAbsolutePath = relativeFile && projectRoot && ( + !Cypress.isBrowser('webkit') || (relativeFile !== '' && relativeFile !== '[native code]') + ) + + if (canBuildAbsolutePath) { absoluteFile = path.resolve(projectRoot, relativeFile) // rollup-plugin-node-builtins/src/es6/path.js only support POSIX, we have @@ -356,7 +362,9 @@ const reconstructStack = (parsedStack) => { const { whitespace, originalFile, function: fn, line, column } = parsedLine - return `${whitespace}at ${fn} (${originalFile || ''}:${line}:${column})` + const lineAndColumn = (Number.isInteger(line) || Number.isInteger(column)) ? `:${line}:${column}` : '' + + return `${whitespace}at ${fn} (${originalFile || ''}${lineAndColumn})` }).join('\n').trimEnd() } @@ -392,7 +400,35 @@ const normalizedStack = (err) => { // Chromium-based errors do, so we normalize them so that the stack // always includes the name/message const errString = err.toString() - const errStack = err.stack || '' + let errStack = err.stack || '' + + if (Cypress.isBrowser('webkit')) { + // WebKit will not determine the proper stack trace for an error, with stack entries + // missing function names, call locations, or both. This is due to a number of documented + // issues with WebKit: + // https://bugs.webkit.org/show_bug.cgi?id=86493 + // https://bugs.webkit.org/show_bug.cgi?id=243668 + // https://bugs.webkit.org/show_bug.cgi?id=174380 + // + // We update these stack entries with placeholder names/locations to more closely align + // the output with other browsers, minimizing the visual impact to the stack traces we render + // within the command log and console and ensuring that the stacks can be identified within + // and parsed out of test snapshots that include them. + errStack = errStack.replaceAll(webkitStackLineRegex, (match, ...parts: string[]) => { + // We patch WebKit's Error within the AUT as CyWebKitError, causing it to + // be presented within the stack. If we detect it within the stack, we remove it. + if (parts[0] === '__CyWebKitError') { + return '' + } + + return [ + parts[0] || '', + '@', + parts[1] || '', + parts[2], + ].join('') + }) + } // the stack has already been normalized and normalizing the indentation // again could mess up the whitespace diff --git a/packages/errors/src/stackUtils.ts b/packages/errors/src/stackUtils.ts index 5d1d4f3b37b8..4f4d5b0ab75e 100644 --- a/packages/errors/src/stackUtils.ts +++ b/packages/errors/src/stackUtils.ts @@ -1,7 +1,7 @@ import _ from 'lodash' import type { ErrorLike } from './errorTypes' -const stackLineRegex = /^\s*(at )?.*@?\(?.*\:\d+\:\d+\)?$/ +export const stackLineRegex = /^\s*(at )?.*@?(?:\(?.*(?::\d+:\d+||\[native code\])+\)?)$/ type MessageLines = [string[], string[]] & {messageEnded?: boolean} diff --git a/packages/reporter/src/errors/error-stack.tsx b/packages/reporter/src/errors/error-stack.tsx index 14c93a20e440..bf3b65499d17 100644 --- a/packages/reporter/src/errors/error-stack.tsx +++ b/packages/reporter/src/errors/error-stack.tsx @@ -76,7 +76,9 @@ const ErrorStack = observer(({ err }: Props) => { ) if (dontLink) { - return makeLine(key, [whitespace, `at ${fn} (${originalFile}:${line}:${column})`]) + const lineAndColumn = (Number.isInteger(line) || Number.isInteger(column)) ? `:${line}:${column}` : '' + + return makeLine(key, [whitespace, `at ${fn} (${originalFile}${lineAndColumn})`]) } const link = ( diff --git a/system-tests/__snapshots__/spec_isolation_spec.js b/system-tests/__snapshots__/spec_isolation_spec.js index 3e2384231853..964ba207b200 100644 --- a/system-tests/__snapshots__/spec_isolation_spec.js +++ b/system-tests/__snapshots__/spec_isolation_spec.js @@ -3271,3 +3271,934 @@ exports['e2e spec_isolation failing with retries enabled [firefox] 1'] = { "config": {}, "status": "finished" } + +exports['e2e spec_isolation failing with retries enabled [webkit] 1'] = { + "status": "finished", + "startedTestsAt": "2018-02-01T20:14:19.323Z", + "endedTestsAt": "2018-02-01T20:14:19.323Z", + "totalDuration": 5555, + "totalSuites": 6, + "totalTests": 8, + "totalPassed": 2, + "totalPending": 1, + "totalFailed": 4, + "totalSkipped": 1, + "runs": [ + { + "stats": { + "suites": 5, + "tests": 6, + "passes": 1, + "pending": 1, + "skipped": 1, + "failures": 3, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "endedAt": "2018-02-01T20:14:19.323Z" + }, + "reporter": "spec", + "reporterStats": { + "suites": 5, + "tests": 5, + "passes": 1, + "pending": 1, + "failures": 3, + "start": "2018-02-01T20:14:19.323Z", + "end": "2018-02-01T20:14:19.323Z", + "duration": 1234 + }, + "hooks": [ + { + "hookName": "before each", + "title": [ + "\"before each\" hook" + ], + "body": "() => {\n throw new Error('fail1');\n }" + }, + { + "hookName": "after each", + "title": [ + "\"after each\" hook" + ], + "body": "() => {\n throw new Error('fail2');\n }" + }, + { + "hookName": "after all", + "title": [ + "\"after all\" hook" + ], + "body": "() => {\n throw new Error('fail3');\n }" + } + ], + "tests": [ + { + "title": [ + "simple failing hook spec", + "beforeEach hooks", + "never gets here" + ], + "state": "failed", + "body": "() => {}", + "displayError": "Error: fail1\n\nBecause this error occurred during a `before each` hook we are skipping the remaining tests in the current suite: `beforeEach hooks`\n [stack trace lines]", + "attempts": [ + { + "state": "failed", + "error": { + "name": "Error", + "message": "fail1", + "stack": "[stack trace lines]" + }, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [ + { + "name": null, + "takenAt": "2018-02-01T20:14:19.323Z", + "path": "/foo/bar/.projects/e2e/cypress/screenshots/simple_failing_hook.cy.js/simple failing hook spec -- beforeEach hooks -- never gets here (failed).png", + "height": 720, + "width": 1280 + } + ] + }, + { + "state": "failed", + "error": { + "name": "Error", + "message": "fail1\n\nBecause this error occurred during a `before each` hook we are skipping the remaining tests in the current suite: `beforeEach hooks`", + "stack": "[stack trace lines]" + }, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [ + { + "name": null, + "takenAt": "2018-02-01T20:14:19.323Z", + "path": "/foo/bar/.projects/e2e/cypress/screenshots/simple_failing_hook.cy.js/simple failing hook spec -- beforeEach hooks -- never gets here -- before each hook (failed) (attempt 2).png", + "height": 720, + "width": 1280 + } + ] + } + ] + }, + { + "title": [ + "simple failing hook spec", + "pending", + "is pending" + ], + "state": "pending", + "body": "", + "displayError": null, + "attempts": [ + { + "state": "pending", + "error": null, + "videoTimestamp": null, + "duration": null, + "startedAt": null, + "screenshots": [] + } + ] + }, + { + "title": [ + "simple failing hook spec", + "afterEach hooks", + "runs this" + ], + "state": "failed", + "body": "() => {}", + "displayError": "Error: fail2\n\nBecause this error occurred during a `after each` hook we are skipping the remaining tests in the current suite: `afterEach hooks`\n [stack trace lines]", + "attempts": [ + { + "state": "failed", + "error": { + "name": "Error", + "message": "fail2", + "stack": "[stack trace lines]" + }, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [ + { + "name": null, + "takenAt": "2018-02-01T20:14:19.323Z", + "path": "/foo/bar/.projects/e2e/cypress/screenshots/simple_failing_hook.cy.js/simple failing hook spec -- afterEach hooks -- runs this -- after each hook (failed).png", + "height": 720, + "width": 1280 + } + ] + }, + { + "state": "failed", + "error": { + "name": "Error", + "message": "fail2\n\nBecause this error occurred during a `after each` hook we are skipping the remaining tests in the current suite: `afterEach hooks`", + "stack": "[stack trace lines]" + }, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [ + { + "name": null, + "takenAt": "2018-02-01T20:14:19.323Z", + "path": "/foo/bar/.projects/e2e/cypress/screenshots/simple_failing_hook.cy.js/simple failing hook spec -- afterEach hooks -- runs this -- after each hook (failed) (attempt 2).png", + "height": 720, + "width": 1280 + } + ] + } + ] + }, + { + "title": [ + "simple failing hook spec", + "afterEach hooks", + "does not run this" + ], + "state": "skipped", + "body": "() => {}", + "displayError": null, + "attempts": [ + { + "state": "skipped", + "error": null, + "videoTimestamp": null, + "duration": null, + "startedAt": null, + "screenshots": [] + } + ] + }, + { + "title": [ + "simple failing hook spec", + "after hooks", + "runs this" + ], + "state": "passed", + "body": "() => {}", + "displayError": null, + "attempts": [ + { + "state": "passed", + "error": null, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [] + } + ] + }, + { + "title": [ + "simple failing hook spec", + "after hooks", + "fails on this" + ], + "state": "failed", + "body": "() => {}", + "displayError": "Error: fail3\n\nBecause this error occurred during a `after all` hook we are skipping the remaining tests in the current suite: `after hooks`\n\nAlthough you have test retries enabled, we do not retry tests when `before all` or `after all` hooks fail\n [stack trace lines]", + "attempts": [ + { + "state": "failed", + "error": { + "name": "Error", + "message": "fail3\n\nBecause this error occurred during a `after all` hook we are skipping the remaining tests in the current suite: `after hooks`\n\nAlthough you have test retries enabled, we do not retry tests when `before all` or `after all` hooks fail", + "stack": "[stack trace lines]" + }, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [ + { + "name": null, + "takenAt": "2018-02-01T20:14:19.323Z", + "path": "/foo/bar/.projects/e2e/cypress/screenshots/simple_failing_hook.cy.js/simple failing hook spec -- after hooks -- fails on this -- after all hook (failed).png", + "height": 720, + "width": 1280 + } + ] + } + ] + } + ], + "error": null, + "video": null, + "spec": { + "fileExtension": ".js", + "baseName": "simple_failing_hook.cy.js", + "fileName": "simple_failing_hook", + "specFileExtension": ".cy.js", + "relativeToCommonRoot": "simple_failing_hook.cy.js", + "specType": "integration", + "name": "cypress/e2e/simple_failing_hook.cy.js", + "relative": "cypress/e2e/simple_failing_hook.cy.js", + "absolute": "/foo/bar/.projects/e2e/cypress/e2e/simple_failing_hook.cy.js" + }, + "shouldUploadVideo": true + }, + { + "stats": { + "suites": 1, + "tests": 2, + "passes": 1, + "pending": 0, + "skipped": 0, + "failures": 1, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "endedAt": "2018-02-01T20:14:19.323Z" + }, + "reporter": "spec", + "reporterStats": { + "suites": 1, + "tests": 2, + "passes": 1, + "pending": 0, + "failures": 1, + "start": "2018-02-01T20:14:19.323Z", + "end": "2018-02-01T20:14:19.323Z", + "duration": 1234 + }, + "hooks": [], + "tests": [ + { + "title": [ + "simple retrying spec", + "t1" + ], + "state": "failed", + "body": "() => {\n const test = cy.state('test');\n throw new Error(`${test.title} attempt #${cy.state('test').currentRetry()}`);\n }", + "displayError": "Error: t1 attempt #1\n [stack trace lines]", + "attempts": [ + { + "state": "failed", + "error": { + "name": "Error", + "message": "t1 attempt #0", + "stack": "[stack trace lines]" + }, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [ + { + "name": null, + "takenAt": "2018-02-01T20:14:19.323Z", + "path": "/foo/bar/.projects/e2e/cypress/screenshots/simple_retrying.cy.js/simple retrying spec -- t1 (failed).png", + "height": 720, + "width": 1280 + } + ] + }, + { + "state": "failed", + "error": { + "name": "Error", + "message": "t1 attempt #1", + "stack": "[stack trace lines]" + }, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [ + { + "name": null, + "takenAt": "2018-02-01T20:14:19.323Z", + "path": "/foo/bar/.projects/e2e/cypress/screenshots/simple_retrying.cy.js/simple retrying spec -- t1 (failed) (attempt 2).png", + "height": 720, + "width": 1280 + } + ] + } + ] + }, + { + "title": [ + "simple retrying spec", + "t2" + ], + "state": "passed", + "body": "() => {// pass\n }", + "displayError": null, + "attempts": [ + { + "state": "passed", + "error": null, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [] + } + ] + } + ], + "error": null, + "video": null, + "spec": { + "fileExtension": ".js", + "baseName": "simple_retrying.cy.js", + "fileName": "simple_retrying", + "specFileExtension": ".cy.js", + "relativeToCommonRoot": "simple_retrying.cy.js", + "specType": "integration", + "name": "cypress/e2e/simple_retrying.cy.js", + "relative": "cypress/e2e/simple_retrying.cy.js", + "absolute": "/foo/bar/.projects/e2e/cypress/e2e/simple_retrying.cy.js" + }, + "shouldUploadVideo": true + } + ], + "browserPath": "path/to/browser", + "browserName": "FooBrowser", + "browserVersion": "88", + "osName": "FooOS", + "osVersion": "1234", + "cypressVersion": "9.9.9", + "config": {} +} + +exports['e2e spec_isolation fails [webkit] 1'] = { + "status": "finished", + "startedTestsAt": "2018-02-01T20:14:19.323Z", + "endedTestsAt": "2018-02-01T20:14:19.323Z", + "totalDuration": 5555, + "totalSuites": 8, + "totalTests": 12, + "totalPassed": 5, + "totalPending": 1, + "totalFailed": 5, + "totalSkipped": 1, + "runs": [ + { + "stats": { + "suites": 1, + "tests": 1, + "passes": 1, + "pending": 0, + "skipped": 0, + "failures": 0, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "endedAt": "2018-02-01T20:14:19.323Z" + }, + "reporter": "spec", + "reporterStats": { + "suites": 1, + "tests": 1, + "passes": 1, + "pending": 0, + "failures": 0, + "start": "2018-02-01T20:14:19.323Z", + "end": "2018-02-01T20:14:19.323Z", + "duration": 1234 + }, + "hooks": [ + { + "hookName": "before each", + "title": [ + "\"before each\" hook" + ], + "body": "() => {\n cy.wait(1000);\n }" + } + ], + "tests": [ + { + "title": [ + "simple passing spec", + "passes" + ], + "state": "passed", + "body": "() => {\n cy.wrap(true).should('be.true');\n }", + "displayError": null, + "attempts": [ + { + "state": "passed", + "error": null, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [] + } + ] + } + ], + "error": null, + "video": null, + "spec": { + "fileExtension": ".js", + "baseName": "simple_passing.cy.js", + "fileName": "simple_passing", + "specFileExtension": ".cy.js", + "relativeToCommonRoot": "simple_passing.cy.js", + "specType": "integration", + "name": "cypress/e2e/simple_passing.cy.js", + "relative": "cypress/e2e/simple_passing.cy.js", + "absolute": "/foo/bar/.projects/e2e/cypress/e2e/simple_passing.cy.js" + }, + "shouldUploadVideo": true + }, + { + "stats": { + "suites": 1, + "tests": 3, + "passes": 3, + "pending": 0, + "skipped": 0, + "failures": 0, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "endedAt": "2018-02-01T20:14:19.323Z" + }, + "reporter": "spec", + "reporterStats": { + "suites": 1, + "tests": 3, + "passes": 3, + "pending": 0, + "failures": 0, + "start": "2018-02-01T20:14:19.323Z", + "end": "2018-02-01T20:14:19.323Z", + "duration": 1234 + }, + "hooks": [ + { + "hookName": "before all", + "title": [ + "\"before all\" hook" + ], + "body": "() => {\n cy.wait(100);\n }" + }, + { + "hookName": "before each", + "title": [ + "\"before each\" hook" + ], + "body": "() => {\n cy.wait(200);\n }" + }, + { + "hookName": "after each", + "title": [ + "\"after each\" hook" + ], + "body": "() => {\n cy.wait(200);\n }" + }, + { + "hookName": "after all", + "title": [ + "\"after all\" hook" + ], + "body": "() => {\n cy.wait(100);\n }" + } + ], + "tests": [ + { + "title": [ + "simple hooks spec", + "t1" + ], + "state": "passed", + "body": "() => {\n cy.wrap('t1').should('eq', 't1');\n }", + "displayError": null, + "attempts": [ + { + "state": "passed", + "error": null, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [] + } + ] + }, + { + "title": [ + "simple hooks spec", + "t2" + ], + "state": "passed", + "body": "() => {\n cy.wrap('t2').should('eq', 't2');\n }", + "displayError": null, + "attempts": [ + { + "state": "passed", + "error": null, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [] + } + ] + }, + { + "title": [ + "simple hooks spec", + "t3" + ], + "state": "passed", + "body": "() => {\n cy.wrap('t3').should('eq', 't3');\n }", + "displayError": null, + "attempts": [ + { + "state": "passed", + "error": null, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [] + } + ] + } + ], + "error": null, + "video": null, + "spec": { + "fileExtension": ".js", + "baseName": "simple_hooks.cy.js", + "fileName": "simple_hooks", + "specFileExtension": ".cy.js", + "relativeToCommonRoot": "simple_hooks.cy.js", + "specType": "integration", + "name": "cypress/e2e/simple_hooks.cy.js", + "relative": "cypress/e2e/simple_hooks.cy.js", + "absolute": "/foo/bar/.projects/e2e/cypress/e2e/simple_hooks.cy.js" + }, + "shouldUploadVideo": true + }, + { + "stats": { + "suites": 1, + "tests": 2, + "passes": 0, + "pending": 0, + "skipped": 0, + "failures": 2, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "endedAt": "2018-02-01T20:14:19.323Z" + }, + "reporter": "spec", + "reporterStats": { + "suites": 1, + "tests": 2, + "passes": 0, + "pending": 0, + "failures": 2, + "start": "2018-02-01T20:14:19.323Z", + "end": "2018-02-01T20:14:19.323Z", + "duration": 1234 + }, + "hooks": [], + "tests": [ + { + "title": [ + "simple failing spec", + "fails1" + ], + "state": "failed", + "body": "() => {\n cy.wrap(true, {\n timeout: 100\n }).should('be.false');\n }", + "displayError": "AssertionError: Timed out retrying after 100ms: expected true to be false\n [stack trace lines]", + "attempts": [ + { + "state": "failed", + "error": { + "name": "AssertionError", + "message": "Timed out retrying after 100ms: expected true to be false", + "stack": "[stack trace lines]" + }, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [ + { + "name": null, + "takenAt": "2018-02-01T20:14:19.323Z", + "path": "/foo/bar/.projects/e2e/cypress/screenshots/simple_failing.cy.js/simple failing spec -- fails1 (failed).png", + "height": 720, + "width": 1280 + } + ] + } + ] + }, + { + "title": [ + "simple failing spec", + "fails2" + ], + "state": "failed", + "body": "() => {\n throw new Error('fails2');\n }", + "displayError": "Error: fails2\n [stack trace lines]", + "attempts": [ + { + "state": "failed", + "error": { + "name": "Error", + "message": "fails2", + "stack": "[stack trace lines]" + }, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [ + { + "name": null, + "takenAt": "2018-02-01T20:14:19.323Z", + "path": "/foo/bar/.projects/e2e/cypress/screenshots/simple_failing.cy.js/simple failing spec -- fails2 (failed).png", + "height": 720, + "width": 1280 + } + ] + } + ] + } + ], + "error": null, + "video": null, + "spec": { + "fileExtension": ".js", + "baseName": "simple_failing.cy.js", + "fileName": "simple_failing", + "specFileExtension": ".cy.js", + "relativeToCommonRoot": "simple_failing.cy.js", + "specType": "integration", + "name": "cypress/e2e/simple_failing.cy.js", + "relative": "cypress/e2e/simple_failing.cy.js", + "absolute": "/foo/bar/.projects/e2e/cypress/e2e/simple_failing.cy.js" + }, + "shouldUploadVideo": true + }, + { + "stats": { + "suites": 5, + "tests": 6, + "passes": 1, + "pending": 1, + "skipped": 1, + "failures": 3, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "endedAt": "2018-02-01T20:14:19.323Z" + }, + "reporter": "spec", + "reporterStats": { + "suites": 5, + "tests": 5, + "passes": 1, + "pending": 1, + "failures": 3, + "start": "2018-02-01T20:14:19.323Z", + "end": "2018-02-01T20:14:19.323Z", + "duration": 1234 + }, + "hooks": [ + { + "hookName": "before each", + "title": [ + "\"before each\" hook" + ], + "body": "() => {\n throw new Error('fail1');\n }" + }, + { + "hookName": "after each", + "title": [ + "\"after each\" hook" + ], + "body": "() => {\n throw new Error('fail2');\n }" + }, + { + "hookName": "after all", + "title": [ + "\"after all\" hook" + ], + "body": "() => {\n throw new Error('fail3');\n }" + } + ], + "tests": [ + { + "title": [ + "simple failing hook spec", + "beforeEach hooks", + "never gets here" + ], + "state": "failed", + "body": "() => {}", + "displayError": "Error: fail1\n\nBecause this error occurred during a `before each` hook we are skipping the remaining tests in the current suite: `beforeEach hooks`\n [stack trace lines]", + "attempts": [ + { + "state": "failed", + "error": { + "name": "Error", + "message": "fail1\n\nBecause this error occurred during a `before each` hook we are skipping the remaining tests in the current suite: `beforeEach hooks`", + "stack": "[stack trace lines]" + }, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [ + { + "name": null, + "takenAt": "2018-02-01T20:14:19.323Z", + "path": "/foo/bar/.projects/e2e/cypress/screenshots/simple_failing_hook.cy.js/simple failing hook spec -- beforeEach hooks -- never gets here -- before each hook (failed).png", + "height": 720, + "width": 1280 + } + ] + } + ] + }, + { + "title": [ + "simple failing hook spec", + "pending", + "is pending" + ], + "state": "pending", + "body": "", + "displayError": null, + "attempts": [ + { + "state": "pending", + "error": null, + "videoTimestamp": null, + "duration": null, + "startedAt": null, + "screenshots": [] + } + ] + }, + { + "title": [ + "simple failing hook spec", + "afterEach hooks", + "runs this" + ], + "state": "failed", + "body": "() => {}", + "displayError": "Error: fail2\n\nBecause this error occurred during a `after each` hook we are skipping the remaining tests in the current suite: `afterEach hooks`\n [stack trace lines]", + "attempts": [ + { + "state": "failed", + "error": { + "name": "Error", + "message": "fail2\n\nBecause this error occurred during a `after each` hook we are skipping the remaining tests in the current suite: `afterEach hooks`", + "stack": "[stack trace lines]" + }, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [ + { + "name": null, + "takenAt": "2018-02-01T20:14:19.323Z", + "path": "/foo/bar/.projects/e2e/cypress/screenshots/simple_failing_hook.cy.js/simple failing hook spec -- afterEach hooks -- runs this -- after each hook (failed).png", + "height": 720, + "width": 1280 + } + ] + } + ] + }, + { + "title": [ + "simple failing hook spec", + "afterEach hooks", + "does not run this" + ], + "state": "skipped", + "body": "() => {}", + "displayError": null, + "attempts": [ + { + "state": "skipped", + "error": null, + "videoTimestamp": null, + "duration": null, + "startedAt": null, + "screenshots": [] + } + ] + }, + { + "title": [ + "simple failing hook spec", + "after hooks", + "runs this" + ], + "state": "passed", + "body": "() => {}", + "displayError": null, + "attempts": [ + { + "state": "passed", + "error": null, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [] + } + ] + }, + { + "title": [ + "simple failing hook spec", + "after hooks", + "fails on this" + ], + "state": "failed", + "body": "() => {}", + "displayError": "Error: fail3\n\nBecause this error occurred during a `after all` hook we are skipping the remaining tests in the current suite: `after hooks`\n [stack trace lines]", + "attempts": [ + { + "state": "failed", + "error": { + "name": "Error", + "message": "fail3\n\nBecause this error occurred during a `after all` hook we are skipping the remaining tests in the current suite: `after hooks`", + "stack": "[stack trace lines]" + }, + "videoTimestamp": null, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [ + { + "name": null, + "takenAt": "2018-02-01T20:14:19.323Z", + "path": "/foo/bar/.projects/e2e/cypress/screenshots/simple_failing_hook.cy.js/simple failing hook spec -- after hooks -- fails on this -- after all hook (failed).png", + "height": 720, + "width": 1280 + } + ] + } + ] + } + ], + "error": null, + "video": null, + "spec": { + "fileExtension": ".js", + "baseName": "simple_failing_hook.cy.js", + "fileName": "simple_failing_hook", + "specFileExtension": ".cy.js", + "relativeToCommonRoot": "simple_failing_hook.cy.js", + "specType": "integration", + "name": "cypress/e2e/simple_failing_hook.cy.js", + "relative": "cypress/e2e/simple_failing_hook.cy.js", + "absolute": "/foo/bar/.projects/e2e/cypress/e2e/simple_failing_hook.cy.js" + }, + "shouldUploadVideo": true + } + ], + "browserPath": "path/to/browser", + "browserName": "FooBrowser", + "browserVersion": "88", + "osName": "FooOS", + "osVersion": "1234", + "cypressVersion": "9.9.9", + "config": {} +} diff --git a/system-tests/lib/normalizeStdout.ts b/system-tests/lib/normalizeStdout.ts index 1bac6aa44d2d..b56e84e0ecc6 100644 --- a/system-tests/lib/normalizeStdout.ts +++ b/system-tests/lib/normalizeStdout.ts @@ -9,7 +9,6 @@ export const pathUpToProjectName = Fixtures.projectPath('') export const browserNameVersionRe = /(Browser\:\s+)(Custom |)(Electron|Chrome|Canary|Chromium|Firefox|WebKit)(\s\d+)(\s\(\w+\))?(\s+)/ -const stackTraceLinesRe = /(\n?[^\S\n\r]*).*?(@|\bat\b)(?:.*node:.*|.*\.(js|coffee|ts|html|jsx|tsx))\??(-\d+)?:\d+:\d+[\n\S\s]*?(\n\s*?\n|$)/g const availableBrowsersRe = /(Available browsers found on your system are:)([\s\S]+)/g const crossOriginErrorRe = /(Blocked a frame .* from accessing a cross-origin frame.*|Permission denied.*cross-origin object.*)/gm const whiteSpaceBetweenNewlines = /\n\s+\n/ @@ -77,14 +76,31 @@ const replaceUploadingResults = function (orig: string, ...rest: string[]) { // this captures an entire stack trace and replaces it with [stack trace lines] // so that the stdout can contain stack traces of different lengths -// '@' will be present in firefox stack trace lines -// 'at' will be present in chrome stack trace lines -export const replaceStackTraceLines = (str: string) => { - return str.replace(stackTraceLinesRe, (match: string, ...parts: string[]) => { - const isFirefoxStack = parts[1] === '@' - let post = parts[4] - - if (isFirefoxStack) { +export const replaceStackTraceLines = (str: string, browserName: 'electron' | 'firefox' | 'chrome' | 'webkit') => { + // matches the newline preceding the stack and any leading whitespace + const leadingNewLinesAndWhitespace = `(?:\\n?[^\\S\\n\\r]*)` + // matches against the potential file location patterns, including: + // foo.js:1:2 - file locations including line/column numbers + // - rendered when location cannot be determined + // [native code] - rendered in some cases by WebKit browser + const location = `(?:.*:\\d+:\\d+||\\[native code\\])` + // matches stack lines with Chrome-style rendering: + // ' at foobar (foo.js:1:2)' + // ' at foo.js:1:2' + const verboseStyleLine = `at\\s.*(?::\\d+:\\d+|\\s\\(${location}\\))` + // matches stack lines with Firefox/WebKit style rendering: + // ' foobar@foo.js:1:2' + const condensedStyleLine = `.*@${location}` + // matches against remainder of stack trace until blank lines found. + // includes group to normalize whitespace between newlines in Firefox + const remainderOfStack = `[\\n\\S\\s]*?(\\n\\s*?\\n|$)` + + const stackTraceRegex = new RegExp(`${leadingNewLinesAndWhitespace}(?:${verboseStyleLine}|${condensedStyleLine})${remainderOfStack}`, 'g') + + return str.replace(stackTraceRegex, (match: string, ...parts: string[]) => { + let post = parts[0] + + if (browserName === 'firefox') { post = post.replace(whiteSpaceBetweenNewlines, '\n') } @@ -142,6 +158,12 @@ export const normalizeStdout = function (str: string, options: any = {}) { // Replaces connection warning since Chrome or Firefox sometimes take longer to connect .replace(/Still waiting to connect to .+, retrying in 1 second \(attempt .+\/.+\)\n/g, '') + if (options.browser === 'webkit') { + // WebKit throws for lookups on undefined refs with "Can't find variable: " + // This message is replaced with Chrome/Firefox's exception text for consistent diffs + str = str.replace(/(ReferenceError:|>) Can\'t find variable: (\S+)/g, '$1 $2 is not defined') + } + // avoid race condition when webpack prints this at a non-deterministic timing const wdsFailedMsg = 'ℹ ｢wdm｣: Failed to compile.' @@ -154,5 +176,5 @@ export const normalizeStdout = function (str: string, options: any = {}) { str = str.replace(/(\(\d+x\d+\))/g, replaceScreenshotDims) } - return replaceStackTraceLines(str) + return replaceStackTraceLines(str, options.browser) } diff --git a/system-tests/projects/e2e/cypress/e2e/uncaught_during_test.cy.js b/system-tests/projects/e2e/cypress/e2e/uncaught_during_test.cy.js index bfdccd940d18..83f593bb930d 100644 --- a/system-tests/projects/e2e/cypress/e2e/uncaught_during_test.cy.js +++ b/system-tests/projects/e2e/cypress/e2e/uncaught_during_test.cy.js @@ -20,7 +20,11 @@ describe('foo', () => { it('passes with fail handler after failing with setTimeout', (done) => { cy.on('fail', (err) => { - expect(err.message).to.include('foo is not defined') + if (Cypress.isBrowser('webkit')) { + expect(err.message).to.include('Can\'t find variable: foo') + } else { + expect(err.message).to.include('foo is not defined') + } setTimeout(() => { return done() @@ -43,8 +47,13 @@ describe('foo', () => { it('passes with fail handler after failing with async app code error', (done) => { cy.on('fail', (err) => { - expect(err.message).to.include('qax is not defined') - expect(err.stack).to.include('qax is not defined') + if (Cypress.isBrowser('webkit')) { + expect(err.message).to.include('Can\'t find variable: qax') + expect(err.stack).to.include('Can\'t find variable: qax') + } else { + expect(err.message).to.include('qax is not defined') + expect(err.stack).to.include('qax is not defined') + } setTimeout(() => { return done() diff --git a/system-tests/test/async_timeouts_spec.js b/system-tests/test/async_timeouts_spec.js index cc22f18bbe67..857d38cc30b9 100644 --- a/system-tests/test/async_timeouts_spec.js +++ b/system-tests/test/async_timeouts_spec.js @@ -4,7 +4,6 @@ describe('e2e async timeouts', () => { systemTests.setup() systemTests.it('failing1', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'async_timeouts.cy.js', snapshot: true, expectedExitCode: 2, diff --git a/system-tests/test/caught_uncaught_hook_errors_spec.js b/system-tests/test/caught_uncaught_hook_errors_spec.js index 934a4750467e..8831628bd0e7 100644 --- a/system-tests/test/caught_uncaught_hook_errors_spec.js +++ b/system-tests/test/caught_uncaught_hook_errors_spec.js @@ -9,28 +9,24 @@ describe('e2e caught and uncaught hooks errors', () => { }) systemTests.it('failing1', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'hook_caught_error_failing.cy.js', snapshot: true, expectedExitCode: 3, }) systemTests.it('failing2', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to different ReferenceError snapshot) spec: 'hook_uncaught_error_failing.cy.js', snapshot: true, expectedExitCode: 1, }) systemTests.it('failing3', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to different ReferenceError snapshot) spec: 'hook_uncaught_root_error_failing.cy.js', snapshot: true, expectedExitCode: 1, }) systemTests.it('failing4', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to different ReferenceError snapshot) spec: 'hook_uncaught_error_events_failing.cy.js', snapshot: true, expectedExitCode: 1, diff --git a/system-tests/test/commands_outside_of_test_spec.js b/system-tests/test/commands_outside_of_test_spec.js index 60ff96c9edde..b7c2f62ad21a 100644 --- a/system-tests/test/commands_outside_of_test_spec.js +++ b/system-tests/test/commands_outside_of_test_spec.js @@ -4,14 +4,12 @@ describe('e2e commands outside of test', () => { systemTests.setup() systemTests.it('fails on cy commands', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'commands_outside_of_test.cy.js', snapshot: true, expectedExitCode: 1, }) systemTests.it('fails on failing assertions', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'assertions_failing_outside_of_test.cy.js', snapshot: true, expectedExitCode: 1, diff --git a/system-tests/test/form_submissions_spec.js b/system-tests/test/form_submissions_spec.js index 25f6c990fa2b..701cc37b62b9 100644 --- a/system-tests/test/form_submissions_spec.js +++ b/system-tests/test/form_submissions_spec.js @@ -95,7 +95,6 @@ describe('e2e forms', () => { }) systemTests.it('failing', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'form_submission_failing.cy.js', snapshot: true, expectedExitCode: 1, diff --git a/system-tests/test/issue_173_spec.ts b/system-tests/test/issue_173_spec.ts index b7b3891f8a75..8973f636f4bd 100644 --- a/system-tests/test/issue_173_spec.ts +++ b/system-tests/test/issue_173_spec.ts @@ -5,7 +5,6 @@ describe('e2e issue 173', () => { // https://github.com/cypress-io/cypress/issues/173 systemTests.it('failing', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'issue_173.cy.js', snapshot: true, expectedExitCode: 1, diff --git a/system-tests/test/issue_674_spec.js b/system-tests/test/issue_674_spec.js index ab2155f9aa2f..3cb7be05f2b2 100644 --- a/system-tests/test/issue_674_spec.js +++ b/system-tests/test/issue_674_spec.js @@ -5,7 +5,6 @@ describe('e2e issue 674', () => { // https://github.com/cypress-io/cypress/issues/674 systemTests.it('fails', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'issue_674.cy.js', snapshot: true, expectedExitCode: 1, diff --git a/system-tests/test/js_error_handling_spec.js b/system-tests/test/js_error_handling_spec.js index 503ae72959b9..9807c776e8ce 100644 --- a/system-tests/test/js_error_handling_spec.js +++ b/system-tests/test/js_error_handling_spec.js @@ -46,7 +46,6 @@ describe('e2e js error handling', () => { }) systemTests.it('fails', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'js_error_handling_failing.cy.js', snapshot: true, expectedExitCode: 5, diff --git a/system-tests/test/promises_spec.js b/system-tests/test/promises_spec.js index f468b56e5428..da52bf0f143e 100644 --- a/system-tests/test/promises_spec.js +++ b/system-tests/test/promises_spec.js @@ -4,7 +4,6 @@ describe('e2e promises', () => { systemTests.setup() systemTests.it('failing1', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'promises.cy.js', snapshot: true, expectedExitCode: 2, diff --git a/system-tests/test/runnable_execution_spec.ts b/system-tests/test/runnable_execution_spec.ts index 62a22c67dd99..a30e14c7c82a 100644 --- a/system-tests/test/runnable_execution_spec.ts +++ b/system-tests/test/runnable_execution_spec.ts @@ -20,7 +20,6 @@ describe('e2e runnable execution', () => { // but throws correct error // https://github.com/cypress-io/cypress/issues/1987 systemTests.it('cannot navigate in before hook and test', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) project: 'hooks-after-rerun', spec: 'beforehook-and-test-navigation.cy.js', snapshot: true, @@ -34,7 +33,6 @@ describe('e2e runnable execution', () => { }) systemTests.it('runs correctly after top navigation with already ran suite', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'runnables_already_run_suite.cy.js', snapshot: true, expectedExitCode: 1, diff --git a/system-tests/test/spec_isolation_spec.js b/system-tests/test/spec_isolation_spec.js index 643da5056210..315952d6005c 100644 --- a/system-tests/test/spec_isolation_spec.js +++ b/system-tests/test/spec_isolation_spec.js @@ -21,7 +21,6 @@ describe('e2e spec_isolation', () => { systemTests.setup() it('fails', { - browser: '!webkit', // TODO(webkit): fix+unskip spec: specs, outputPath, snapshot: false, @@ -52,7 +51,6 @@ describe('e2e spec_isolation', () => { }) it('failing with retries enabled', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'simple_failing_hook.cy.js,simple_retrying.cy.js', outputPath, snapshot: true, diff --git a/system-tests/test/stdout_spec.js b/system-tests/test/stdout_spec.js index ce0ca859ac41..9c529fef2987 100644 --- a/system-tests/test/stdout_spec.js +++ b/system-tests/test/stdout_spec.js @@ -71,7 +71,6 @@ describe('e2e stdout', () => { }) systemTests.it('displays assertion errors', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'stdout_assertion_errors.cy.js', snapshot: true, expectedExitCode: 4, diff --git a/system-tests/test/testConfigOverrides_spec.ts b/system-tests/test/testConfigOverrides_spec.ts index c88c78a3f3ec..54ab83218477 100644 --- a/system-tests/test/testConfigOverrides_spec.ts +++ b/system-tests/test/testConfigOverrides_spec.ts @@ -21,7 +21,6 @@ describe('testConfigOverrides', () => { }) systemTests.it('fails when passing invalid config value browser', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'testConfigOverrides/invalid-browser.js', snapshot: true, expectedExitCode: 1, diff --git a/system-tests/test/uncaught_spec_errors_spec.js b/system-tests/test/uncaught_spec_errors_spec.js index 72199ce7c953..1117a537ff97 100644 --- a/system-tests/test/uncaught_spec_errors_spec.js +++ b/system-tests/test/uncaught_spec_errors_spec.js @@ -4,35 +4,30 @@ describe('e2e uncaught errors', () => { systemTests.setup() systemTests.it('failing1', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'uncaught_synchronous_before_tests_parsed.js', snapshot: true, expectedExitCode: 1, }) systemTests.it('failing2', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'uncaught_synchronous_during_hook.cy.js', snapshot: true, expectedExitCode: 1, }) systemTests.it('failing3', { - browser: '!webkit', // TODO(webkit): fix+unskip spec: 'uncaught_during_test.cy.js', snapshot: true, expectedExitCode: 3, }) systemTests.it('failing4', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'uncaught_during_hook.cy.js', snapshot: true, expectedExitCode: 1, }) systemTests.it('failing5', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'caught_async_sync_test.cy.js', snapshot: true, expectedExitCode: 4, diff --git a/system-tests/test/visit_spec.js b/system-tests/test/visit_spec.js index 145882546871..278553bc4769 100644 --- a/system-tests/test/visit_spec.js +++ b/system-tests/test/visit_spec.js @@ -171,7 +171,6 @@ describe('e2e visit', () => { }) systemTests.it('fails when server responds with 500', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'visit_http_500_response_failing.cy.js', snapshot: true, expectedExitCode: 1, @@ -185,7 +184,6 @@ describe('e2e visit', () => { }) systemTests.it('fails when content type isnt html', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'visit_non_html_content_type_failing.cy.js', snapshot: true, expectedExitCode: 1, @@ -211,7 +209,6 @@ describe('e2e visit', () => { }) systemTests.it('fails when response never ends', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'visit_response_never_ends_failing.cy.js', snapshot: true, expectedExitCode: 3, @@ -232,7 +229,6 @@ describe('e2e visit', () => { }) systemTests.it('fails when visit times out', { - browser: '!webkit', // TODO(webkit): fix+unskip (failing due to broken stack trace) spec: 'visit_http_timeout_failing.cy.js', snapshot: true, expectedExitCode: 2,