Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix(runner): fix run result attempts array order [develop] #8303

Merged
merged 5 commits into from Aug 18, 2020
Merged

fix(runner): fix run result attempts array order [develop] #8303

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
16a1f3a
fix run result attempts array order
kuceb Aug 17, 2020
b1cf4ad
add better test: test with varying error message
kuceb Aug 17, 2020
503c075
fix .only
kuceb Aug 17, 2020
4b624d9
cleanup
kuceb Aug 17, 2020
aacf76c
update snapshot
kuceb Aug 17, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
184 changes: 172 additions & 12 deletions packages/server/__snapshots__/5_spec_isolation_spec.js
View file
Open in desktop
Expand Up @@ -525,14 +525,15 @@ exports['e2e spec_isolation / failing with retries enabled'] = `
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Cypress: 1.2.3 │
│ Browser: FooBrowser 88 │
│ Specs: 1 found (simple_failing_hook_spec.coffee) │
│ Searched: cypress/integration/simple_failing_hook_spec.coffee │
│ Specs: 2 found (simple_failing_hook_spec.coffee, simple_retrying_spec.js) │
│ Searched: cypress/integration/simple_failing_hook_spec.coffee, cypress/integration/simple_re │
│ trying_spec.js │
└────────────────────────────────────────────────────────────────────────────────────────────────┘


────────────────────────────────────────────────────────────────────────────────────────────────────

Running: simple_failing_hook_spec.coffee (1 of 1)
Running: simple_failing_hook_spec.coffee (1 of 2)


simple failing hook spec
Expand Down Expand Up @@ -620,6 +621,57 @@ Although you have test retries enabled, we do not retry tests when \`before all\
.mp4


────────────────────────────────────────────────────────────────────────────────────────────────────

Running: simple_retrying_spec.js (2 of 2)


simple retrying spec
(Attempt 1 of 2) t1
1) t1
✓ t2


1 passing
1 failing

1) simple retrying spec
t1:
Error: t1 attempt #1
[stack trace lines]




(Results)

┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ Tests: 2 │
│ Passing: 1 │
│ Failing: 1 │
│ Pending: 0 │
│ Skipped: 0 │
│ Screenshots: 2 │
│ Video: true │
│ Duration: X seconds │
│ Spec Ran: simple_retrying_spec.js │
└────────────────────────────────────────────────────────────────────────────────────────────────┘


(Screenshots)

- /XXX/XXX/XXX/cypress/screenshots/simple_retrying_spec.js/simple retrying spec -- (1280x720)
t1 (failed).png
- /XXX/XXX/XXX/cypress/screenshots/simple_retrying_spec.js/simple retrying spec -- (1280x720)
t1 (failed) (attempt 2).png


(Video)

- Started processing: Compressing to 32 CRF
- Finished processing: /XXX/XXX/XXX/cypress/videos/simple_retrying_spec.js.mp4 (X second)


====================================================================================================

(Run Finished)
Expand All @@ -628,8 +680,10 @@ Although you have test retries enabled, we do not retry tests when \`before all\
Spec Tests Passing Failing Pending Skipped
┌────────────────────────────────────────────────────────────────────────────────────────────────┐
│ ✖ simple_failing_hook_spec.coffee XX:XX 6 1 3 1 1 │
├────────────────────────────────────────────────────────────────────────────────────────────────┤
│ ✖ simple_retrying_spec.js XX:XX 2 1 1 - - │
└────────────────────────────────────────────────────────────────────────────────────────────────┘
✖ 1 of 1 failed (100%) XX:XX 6 1 3 1 1
✖ 2 of 2 failed (100%) XX:XX 8 2 4 1 1


`
Expand All @@ -638,10 +692,10 @@ exports['failing with retries enabled'] = {
"startedTestsAt": "2018-02-01T20:14:19.323Z",
"endedTestsAt": "2018-02-01T20:14:19.323Z",
"totalDuration": 5555,
"totalSuites": 5,
"totalTests": 6,
"totalFailed": 3,
"totalPassed": 1,
"totalSuites": 6,
"totalTests": 8,
"totalFailed": 4,
"totalPassed": 2,
"totalPending": 1,
"totalSkipped": 1,
"runs": [
Expand Down Expand Up @@ -706,7 +760,7 @@ exports['failing with retries enabled'] = {
"state": "failed",
"error": {
"name": "Error",
"message": "fail1\n\nBecause this error occurred during a `before each` hook we are skipping the remaining tests in the current suite: `beforeEach hooks`",
"message": "fail1",
"stack": "[stack trace lines]"
},
"videoTimestamp": 9999,
Expand All @@ -726,7 +780,7 @@ exports['failing with retries enabled'] = {
"state": "failed",
"error": {
"name": "Error",
"message": "fail1",
"message": "fail1\n\nBecause this error occurred during a `before each` hook we are skipping the remaining tests in the current suite: `beforeEach hooks`",
"stack": "[stack trace lines]"
},
"videoTimestamp": 9999,
Expand Down Expand Up @@ -778,7 +832,7 @@ exports['failing with retries enabled'] = {
"state": "failed",
"error": {
"name": "Error",
"message": "fail2\n\nBecause this error occurred during a `after each` hook we are skipping the remaining tests in the current suite: `afterEach hooks`",
"message": "fail2",
"stack": "[stack trace lines]"
},
"videoTimestamp": 9999,
Expand All @@ -798,7 +852,7 @@ exports['failing with retries enabled'] = {
"state": "failed",
"error": {
"name": "Error",
"message": "fail2",
"message": "fail2\n\nBecause this error occurred during a `after each` hook we are skipping the remaining tests in the current suite: `afterEach hooks`",
"stack": "[stack trace lines]"
},
"videoTimestamp": 9999,
Expand Down Expand Up @@ -898,6 +952,112 @@ exports['failing with retries enabled'] = {
"specType": "integration"
},
"shouldUploadVideo": true
},
{
"stats": {
"suites": 1,
"tests": 2,
"passes": 1,
"pending": 0,
"skipped": 0,
"failures": 1,
"duration": 1234,
"startedAt": "2018-02-01T20:14:19.323Z",
"endedAt": "2018-02-01T20:14:19.323Z"
},
"reporter": "spec",
"reporterStats": {
"suites": 1,
"tests": 2,
"passes": 1,
"pending": 0,
"failures": 1,
"start": "2018-02-01T20:14:19.323Z",
"end": "2018-02-01T20:14:19.323Z",
"duration": 1234
},
"hooks": [],
"tests": [
{
"title": [
"simple retrying spec",
"t1"
],
"state": "failed",
"body": "function () {\n var test = cy.state('test');\n throw new Error(\"\".concat(test.title, \" attempt #\").concat(cy.state('test').currentRetry()));\n }",
"displayError": "Error: t1 attempt #1\n [stack trace lines]",
"attempts": [
{
"state": "failed",
"error": {
"name": "Error",
"message": "t1 attempt #0",
"stack": "[stack trace lines]"
},
"videoTimestamp": 9999,
"duration": 1234,
"startedAt": "2018-02-01T20:14:19.323Z",
"screenshots": [
{
"name": null,
"takenAt": "2018-02-01T20:14:19.323Z",
"path": "/foo/bar/.projects/e2e/cypress/screenshots/simple_retrying_spec.js/simple retrying spec -- t1 (failed).png",
"height": 720,
"width": 1280
}
]
},
{
"state": "failed",
"error": {
"name": "Error",
"message": "t1 attempt #1",
"stack": "[stack trace lines]"
},
"videoTimestamp": 9999,
"duration": 1234,
"startedAt": "2018-02-01T20:14:19.323Z",
"screenshots": [
{
"name": null,
"takenAt": "2018-02-01T20:14:19.323Z",
"path": "/foo/bar/.projects/e2e/cypress/screenshots/simple_retrying_spec.js/simple retrying spec -- t1 (failed) (attempt 2).png",
"height": 720,
"width": 1280
}
]
}
]
},
{
"title": [
"simple retrying spec",
"t2"
],
"state": "passed",
"body": "function () {// pass\n }",
"displayError": null,
"attempts": [
{
"state": "passed",
"error": null,
"videoTimestamp": 9999,
"duration": 1234,
"startedAt": "2018-02-01T20:14:19.323Z",
"screenshots": []
}
]
}
],
"error": null,
"video": "/foo/bar/.projects/e2e/cypress/videos/simple_retrying_spec.js.mp4",
"spec": {
"name": "simple_retrying_spec.js",
"relative": "cypress/integration/simple_retrying_spec.js",
"absolute": "/foo/bar/.projects/e2e/cypress/integration/simple_retrying_spec.js",
"specType": "integration"
},
"shouldUploadVideo": true
}
],
"browserPath": "path/to/browser",
Expand Down
2 changes: 1 addition & 1 deletion packages/server/lib/reporter.js
View file
Open in desktop
Expand Up @@ -357,7 +357,7 @@ class Reporter {
state: orNull(test.state),
body: orNull(test.body),
displayError: orNull(test.err && test.err.stack),
attempts: _.map([test].concat(test.prevAttempts || []), (attempt) => {
attempts: _.map((test.prevAttempts || []).concat([test]), (attempt) => {
const err = attempt.err && {
name: attempt.err.name,
message: attempt.err.message,
Expand Down
12 changes: 6 additions & 6 deletions packages/server/test/e2e/5_spec_isolation_spec.js
View file
Open in desktop
Expand Up @@ -6,8 +6,8 @@ const fs = require('../../lib/util/fs')
const e2e = require('../support/helpers/e2e').default
const Fixtures = require('../support/helpers/fixtures')
const { expectCorrectModuleApiResult } = require('../support/helpers/resultsUtils')

const e2ePath = Fixtures.projectPath('e2e')
const it = e2e.it

const outputPath = path.join(e2ePath, 'output.json')

Expand All @@ -21,7 +21,7 @@ const specs = [
describe('e2e spec_isolation', () => {
e2e.setup()

e2e.it('fails', {
it('fails', {
spec: specs,
outputPath,
snapshot: false,
Expand All @@ -42,11 +42,11 @@ describe('e2e spec_isolation', () => {
},
})

e2e.it('failing with retries enabled', {
spec: 'simple_failing_hook_spec.coffee',
it('failing with retries enabled', {
spec: 'simple_failing_hook_spec.coffee,simple_retrying_spec.js',
outputPath,
snapshot: true,
expectedExitCode: 3,
expectedExitCode: 4,
config: {
retries: 1,
},
Expand All @@ -55,7 +55,7 @@ describe('e2e spec_isolation', () => {
const json = await fs.readJsonAsync(outputPath)

// also mutates into normalized obj ready for snapshot
expectCorrectModuleApiResult(json, { e2ePath, runs: 1 })
expectCorrectModuleApiResult(json, { e2ePath, runs: 2 })

snapshot('failing with retries enabled', json)
},
Expand Down
11 changes: 11 additions & 0 deletions ...ges/server/test/support/fixtures/projects/e2e/cypress/integration/simple_retrying_spec.js
View file
Open in desktop
@@ -0,0 +1,11 @@
describe('simple retrying spec', () => {
it('t1', () => {
const test = cy.state('test')

throw new Error(`${test.title} attempt #${cy.state('test').currentRetry()}`)
})

it('t2', () => {
// pass
})
})