From 16a1f3a83a023f2e41dc72c878fa83093a85e505 Mon Sep 17 00:00:00 2001 From: Ben Kucera <14625260+Bkucera@users.noreply.github.com> Date: Mon, 17 Aug 2020 13:58:17 -0400 Subject: [PATCH 1/5] fix run result attempts array order --- packages/server/__snapshots__/5_spec_isolation_spec.js | 8 ++++---- packages/server/lib/reporter.js | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/server/__snapshots__/5_spec_isolation_spec.js b/packages/server/__snapshots__/5_spec_isolation_spec.js index 6f439cfdc402..699bc87f2884 100644 --- a/packages/server/__snapshots__/5_spec_isolation_spec.js +++ b/packages/server/__snapshots__/5_spec_isolation_spec.js @@ -706,7 +706,7 @@ exports['failing with retries enabled'] = { "state": "failed", "error": { "name": "Error", - "message": "fail1\n\nBecause this error occurred during a `before each` hook we are skipping the remaining tests in the current suite: `beforeEach hooks`", + "message": "fail1", "stack": "[stack trace lines]" }, "videoTimestamp": 9999, @@ -726,7 +726,7 @@ exports['failing with retries enabled'] = { "state": "failed", "error": { "name": "Error", - "message": "fail1", + "message": "fail1\n\nBecause this error occurred during a `before each` hook we are skipping the remaining tests in the current suite: `beforeEach hooks`", "stack": "[stack trace lines]" }, "videoTimestamp": 9999, @@ -778,7 +778,7 @@ exports['failing with retries enabled'] = { "state": "failed", "error": { "name": "Error", - "message": "fail2\n\nBecause this error occurred during a `after each` hook we are skipping the remaining tests in the current suite: `afterEach hooks`", + "message": "fail2", "stack": "[stack trace lines]" }, "videoTimestamp": 9999, @@ -798,7 +798,7 @@ exports['failing with retries enabled'] = { "state": "failed", "error": { "name": "Error", - "message": "fail2", + "message": "fail2\n\nBecause this error occurred during a `after each` hook we are skipping the remaining tests in the current suite: `afterEach hooks`", "stack": "[stack trace lines]" }, "videoTimestamp": 9999, diff --git a/packages/server/lib/reporter.js b/packages/server/lib/reporter.js index 84119a80fd73..092482913989 100644 --- a/packages/server/lib/reporter.js +++ b/packages/server/lib/reporter.js @@ -357,7 +357,7 @@ class Reporter { state: orNull(test.state), body: orNull(test.body), displayError: orNull(test.err && test.err.stack), - attempts: _.map([test].concat(test.prevAttempts || []), (attempt) => { + attempts: _.map((test.prevAttempts || []).concat([test]), (attempt) => { const err = attempt.err && { name: attempt.err.name, message: attempt.err.message, From b1cf4ad3becd3eb76fbcb01291e1bbe192c22969 Mon Sep 17 00:00:00 2001 From: Ben Kucera <14625260+Bkucera@users.noreply.github.com> Date: Mon, 17 Aug 2020 14:28:32 -0400 Subject: [PATCH 2/5] add better test: test with varying error message --- .../__snapshots__/5_spec_isolation_spec.js | 176 +++++++++++++++++- .../server/test/e2e/5_spec_isolation_spec.js | 8 +- .../integration/simple_retrying_spec.js | 11 ++ 3 files changed, 183 insertions(+), 12 deletions(-) create mode 100644 packages/server/test/support/fixtures/projects/e2e/cypress/integration/simple_retrying_spec.js diff --git a/packages/server/__snapshots__/5_spec_isolation_spec.js b/packages/server/__snapshots__/5_spec_isolation_spec.js index 699bc87f2884..8e365e10c154 100644 --- a/packages/server/__snapshots__/5_spec_isolation_spec.js +++ b/packages/server/__snapshots__/5_spec_isolation_spec.js @@ -525,14 +525,15 @@ exports['e2e spec_isolation / failing with retries enabled'] = ` ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ Cypress: 1.2.3 │ │ Browser: FooBrowser 88 │ - │ Specs: 1 found (simple_failing_hook_spec.coffee) │ - │ Searched: cypress/integration/simple_failing_hook_spec.coffee │ + │ Specs: 2 found (simple_failing_hook_spec.coffee, simple_retrying_spec.js) │ + │ Searched: cypress/integration/simple_failing_hook_spec.coffee, cypress/integration/simple_re │ + │ trying_spec.js │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ ──────────────────────────────────────────────────────────────────────────────────────────────────── - Running: simple_failing_hook_spec.coffee (1 of 1) + Running: simple_failing_hook_spec.coffee (1 of 2) simple failing hook spec @@ -620,6 +621,57 @@ Although you have test retries enabled, we do not retry tests when \`before all\ .mp4 +──────────────────────────────────────────────────────────────────────────────────────────────────── + + Running: simple_retrying_spec.js (2 of 2) + + + simple retrying spec + (Attempt 1 of 2) t1 + 1) t1 + ✓ t2 + + + 1 passing + 1 failing + + 1) simple retrying spec + t1: + Error: t1 attempt #1 + [stack trace lines] + + + + + (Results) + + ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ + │ Tests: 2 │ + │ Passing: 1 │ + │ Failing: 1 │ + │ Pending: 0 │ + │ Skipped: 0 │ + │ Screenshots: 2 │ + │ Video: true │ + │ Duration: X seconds │ + │ Spec Ran: simple_retrying_spec.js │ + └────────────────────────────────────────────────────────────────────────────────────────────────┘ + + + (Screenshots) + + - /XXX/XXX/XXX/cypress/screenshots/simple_retrying_spec.js/simple retrying spec -- (1280x720) + t1 (failed).png + - /XXX/XXX/XXX/cypress/screenshots/simple_retrying_spec.js/simple retrying spec -- (1280x720) + t1 (failed) (attempt 2).png + + + (Video) + + - Started processing: Compressing to 32 CRF + - Finished processing: /XXX/XXX/XXX/cypress/videos/simple_retrying_spec.js.mp4 (X second) + + ==================================================================================================== (Run Finished) @@ -628,8 +680,10 @@ Although you have test retries enabled, we do not retry tests when \`before all\ Spec Tests Passing Failing Pending Skipped ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ │ ✖ simple_failing_hook_spec.coffee XX:XX 6 1 3 1 1 │ + ├────────────────────────────────────────────────────────────────────────────────────────────────┤ + │ ✖ simple_retrying_spec.js XX:XX 2 1 1 - - │ └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✖ 1 of 1 failed (100%) XX:XX 6 1 3 1 1 + ✖ 2 of 2 failed (100%) XX:XX 8 2 4 1 1 ` @@ -638,10 +692,10 @@ exports['failing with retries enabled'] = { "startedTestsAt": "2018-02-01T20:14:19.323Z", "endedTestsAt": "2018-02-01T20:14:19.323Z", "totalDuration": 5555, - "totalSuites": 5, - "totalTests": 6, - "totalFailed": 3, - "totalPassed": 1, + "totalSuites": 6, + "totalTests": 8, + "totalFailed": 4, + "totalPassed": 2, "totalPending": 1, "totalSkipped": 1, "runs": [ @@ -898,6 +952,112 @@ exports['failing with retries enabled'] = { "specType": "integration" }, "shouldUploadVideo": true + }, + { + "stats": { + "suites": 1, + "tests": 2, + "passes": 1, + "pending": 0, + "skipped": 0, + "failures": 1, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "endedAt": "2018-02-01T20:14:19.323Z" + }, + "reporter": "spec", + "reporterStats": { + "suites": 1, + "tests": 2, + "passes": 1, + "pending": 0, + "failures": 1, + "start": "2018-02-01T20:14:19.323Z", + "end": "2018-02-01T20:14:19.323Z", + "duration": 1234 + }, + "hooks": [], + "tests": [ + { + "title": [ + "simple retrying spec", + "t1" + ], + "state": "failed", + "body": "function () {\n var test = cy.state('test');\n throw new Error(\"\".concat(test.title, \" attempt #\").concat(cy.state('test').currentRetry()));\n }", + "displayError": "Error: t1 attempt #1\n [stack trace lines]", + "attempts": [ + { + "state": "failed", + "error": { + "name": "Error", + "message": "t1 attempt #0", + "stack": "[stack trace lines]" + }, + "videoTimestamp": 9999, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [ + { + "name": null, + "takenAt": "2018-02-01T20:14:19.323Z", + "path": "/foo/bar/.projects/e2e/cypress/screenshots/simple_retrying_spec.js/simple retrying spec -- t1 (failed).png", + "height": 720, + "width": 1280 + } + ] + }, + { + "state": "failed", + "error": { + "name": "Error", + "message": "t1 attempt #1", + "stack": "[stack trace lines]" + }, + "videoTimestamp": 9999, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [ + { + "name": null, + "takenAt": "2018-02-01T20:14:19.323Z", + "path": "/foo/bar/.projects/e2e/cypress/screenshots/simple_retrying_spec.js/simple retrying spec -- t1 (failed) (attempt 2).png", + "height": 720, + "width": 1280 + } + ] + } + ] + }, + { + "title": [ + "simple retrying spec", + "t2" + ], + "state": "passed", + "body": "function () {}", + "displayError": null, + "attempts": [ + { + "state": "passed", + "error": null, + "videoTimestamp": 9999, + "duration": 1234, + "startedAt": "2018-02-01T20:14:19.323Z", + "screenshots": [] + } + ] + } + ], + "error": null, + "video": "/foo/bar/.projects/e2e/cypress/videos/simple_retrying_spec.js.mp4", + "spec": { + "name": "simple_retrying_spec.js", + "relative": "cypress/integration/simple_retrying_spec.js", + "absolute": "/foo/bar/.projects/e2e/cypress/integration/simple_retrying_spec.js", + "specType": "integration" + }, + "shouldUploadVideo": true } ], "browserPath": "path/to/browser", diff --git a/packages/server/test/e2e/5_spec_isolation_spec.js b/packages/server/test/e2e/5_spec_isolation_spec.js index 17f47704770c..7db428551043 100644 --- a/packages/server/test/e2e/5_spec_isolation_spec.js +++ b/packages/server/test/e2e/5_spec_isolation_spec.js @@ -42,11 +42,11 @@ describe('e2e spec_isolation', () => { }, }) - e2e.it('failing with retries enabled', { - spec: 'simple_failing_hook_spec.coffee', + e2e.it.only('failing with retries enabled', { + spec: 'simple_failing_hook_spec.coffee,simple_retrying_spec.js', outputPath, snapshot: true, - expectedExitCode: 3, + expectedExitCode: 4, config: { retries: 1, }, @@ -55,7 +55,7 @@ describe('e2e spec_isolation', () => { const json = await fs.readJsonAsync(outputPath) // also mutates into normalized obj ready for snapshot - expectCorrectModuleApiResult(json, { e2ePath, runs: 1 }) + expectCorrectModuleApiResult(json, { e2ePath, runs: 2 }) snapshot('failing with retries enabled', json) }, diff --git a/packages/server/test/support/fixtures/projects/e2e/cypress/integration/simple_retrying_spec.js b/packages/server/test/support/fixtures/projects/e2e/cypress/integration/simple_retrying_spec.js new file mode 100644 index 000000000000..50f0962a844d --- /dev/null +++ b/packages/server/test/support/fixtures/projects/e2e/cypress/integration/simple_retrying_spec.js @@ -0,0 +1,11 @@ +describe('simple retrying spec', () => { + it('t1', () => { + const test = cy.state('test') + + throw new Error(`${test.title} attempt #${cy.state('test').currentRetry()}`) + }) + + it('t2', () => { + // pass + }) +}) From 503c07504a182e204a855e273c2f403749090763 Mon Sep 17 00:00:00 2001 From: Ben Kucera <14625260+Bkucera@users.noreply.github.com> Date: Mon, 17 Aug 2020 14:29:17 -0400 Subject: [PATCH 3/5] fix .only --- packages/server/test/e2e/5_spec_isolation_spec.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/server/test/e2e/5_spec_isolation_spec.js b/packages/server/test/e2e/5_spec_isolation_spec.js index 7db428551043..79bb39c3413f 100644 --- a/packages/server/test/e2e/5_spec_isolation_spec.js +++ b/packages/server/test/e2e/5_spec_isolation_spec.js @@ -6,7 +6,7 @@ const fs = require('../../lib/util/fs') const e2e = require('../support/helpers/e2e').default const Fixtures = require('../support/helpers/fixtures') const { expectCorrectModuleApiResult } = require('../support/helpers/resultsUtils') - +const it = e2e.it const e2ePath = Fixtures.projectPath('e2e') const outputPath = path.join(e2ePath, 'output.json') @@ -21,7 +21,7 @@ const specs = [ describe('e2e spec_isolation', () => { e2e.setup() - e2e.it('fails', { + it('fails', { spec: specs, outputPath, snapshot: false, @@ -42,7 +42,7 @@ describe('e2e spec_isolation', () => { }, }) - e2e.it.only('failing with retries enabled', { + it('failing with retries enabled', { spec: 'simple_failing_hook_spec.coffee,simple_retrying_spec.js', outputPath, snapshot: true, From 4b624d9135e176f09754c3e1161e165a4787f073 Mon Sep 17 00:00:00 2001 From: Ben Kucera <14625260+Bkucera@users.noreply.github.com> Date: Mon, 17 Aug 2020 14:29:46 -0400 Subject: [PATCH 4/5] cleanup --- packages/server/test/e2e/5_spec_isolation_spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/test/e2e/5_spec_isolation_spec.js b/packages/server/test/e2e/5_spec_isolation_spec.js index 79bb39c3413f..2a5941ac5144 100644 --- a/packages/server/test/e2e/5_spec_isolation_spec.js +++ b/packages/server/test/e2e/5_spec_isolation_spec.js @@ -6,8 +6,8 @@ const fs = require('../../lib/util/fs') const e2e = require('../support/helpers/e2e').default const Fixtures = require('../support/helpers/fixtures') const { expectCorrectModuleApiResult } = require('../support/helpers/resultsUtils') -const it = e2e.it const e2ePath = Fixtures.projectPath('e2e') +const it = e2e.it const outputPath = path.join(e2ePath, 'output.json') From aacf76c190a80ec0ef025b8e171cb3d4fdeaac64 Mon Sep 17 00:00:00 2001 From: Ben Kucera <14625260+Bkucera@users.noreply.github.com> Date: Mon, 17 Aug 2020 14:58:39 -0400 Subject: [PATCH 5/5] update snapshot --- packages/server/__snapshots__/5_spec_isolation_spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/__snapshots__/5_spec_isolation_spec.js b/packages/server/__snapshots__/5_spec_isolation_spec.js index 8e365e10c154..eec49cf57178 100644 --- a/packages/server/__snapshots__/5_spec_isolation_spec.js +++ b/packages/server/__snapshots__/5_spec_isolation_spec.js @@ -1035,7 +1035,7 @@ exports['failing with retries enabled'] = { "t2" ], "state": "passed", - "body": "function () {}", + "body": "function () {// pass\n }", "displayError": null, "attempts": [ {