From 7154fc8dcd7abbef7feb0c0fdacaef31174bc672 Mon Sep 17 00:00:00 2001 From: Alejandro Estrada Date: Mon, 12 Dec 2022 12:47:57 -0500 Subject: [PATCH 1/2] feat: Improve CLOUD_PARALLEL_GROUP_PARAMS_MISMATCH error message (#24799) Co-authored-by: Emily Rohrbough --- ...OUP_PARAMS_MISMATCH - differentParams.html | 67 +++++++++++++++++++ packages/errors/src/errors.ts | 34 +++++++++- .../test/unit/visualSnapshotErrors_spec.ts | 34 ++++++++++ packages/server/__snapshots__/cypress_spec.js | 13 ++-- packages/server/lib/modes/record.js | 1 + .../server/test/integration/cypress_spec.js | 13 ++++ 6 files changed, 155 insertions(+), 7 deletions(-) create mode 100644 packages/errors/__snapshot-html__/CLOUD_PARALLEL_GROUP_PARAMS_MISMATCH - differentParams.html diff --git a/packages/errors/__snapshot-html__/CLOUD_PARALLEL_GROUP_PARAMS_MISMATCH - differentParams.html b/packages/errors/__snapshot-html__/CLOUD_PARALLEL_GROUP_PARAMS_MISMATCH - differentParams.html new file mode 100644 index 000000000000..26e6dbf56c4f --- /dev/null +++ b/packages/errors/__snapshot-html__/CLOUD_PARALLEL_GROUP_PARAMS_MISMATCH - differentParams.html @@ -0,0 +1,67 @@ + + + + + + + + + + + + 
You passed the --parallel flag, but we do not parallelize tests across different environments.
+
+This machine is sending different environment parameters than the first machine that started this parallel run.
+
+The existing run is: https://cloud.cypress.io/project/abcd/runs/1
+
+In order to run in parallel mode each machine must send identical environment parameters such as:
+
+ - specs
+ - osName
+ - osVersion
+ - browserName
+ - browserVersion (major)
+
+This machine sent the following parameters:
+
+{
+  "osName": "darwin",
+  "osVersion": "v1",
+  "browserName": "Electron.... (Expected: Electron)",
+  "browserVersion": "59.1.2.3.... (Expected: 64)",
+  "differentSpecs": {
+    "added": [
+      "cypress/integration/foo_spec.js"
+    ],
+    "missing": []
+  }
+}
+
+https://on.cypress.io/parallel-group-params-mismatch
+
\ No newline at end of file diff --git a/packages/errors/src/errors.ts b/packages/errors/src/errors.ts index 74be0d423f19..9d8d1c3c0f5b 100644 --- a/packages/errors/src/errors.ts +++ b/packages/errors/src/errors.ts @@ -279,7 +279,37 @@ export const AllCypressErrors = { https://on.cypress.io/parallel-disallowed` }, - CLOUD_PARALLEL_GROUP_PARAMS_MISMATCH: (arg1: {runUrl: string, parameters: any}) => { + CLOUD_PARALLEL_GROUP_PARAMS_MISMATCH: (arg1: {runUrl: string, parameters: any, payload: any }) => { + let params: any = arg1.parameters + + if (arg1.payload?.differentParams) { + params = {} + + _.map(arg1.parameters, (value, key) => { + if (key === 'specs' && arg1.payload.differentSpecs?.length) { + const addedSpecs: string[] = [] + const missingSpecs: string[] = [] + + _.forEach(arg1.payload.differentSpecs, (s) => { + if (value.includes(s)) { + addedSpecs.push(s) + } else { + missingSpecs.push(s) + } + }) + + params['differentSpecs'] = { + added: addedSpecs, + missing: missingSpecs, + } + } else if (arg1.payload.differentParams[key]?.expected) { + params[key] = `${value}.... (Expected: ${(arg1.payload.differentParams[key].expected)})` + } else { + params[key] = value + } + }) + } + return errTemplate`\ You passed the ${fmt.flag(`--parallel`)} flag, but we do not parallelize tests across different environments. @@ -299,7 +329,7 @@ export const AllCypressErrors = { This machine sent the following parameters: - ${fmt.meta(arg1.parameters)} + ${fmt.meta(params)} https://on.cypress.io/parallel-group-params-mismatch` }, diff --git a/packages/errors/test/unit/visualSnapshotErrors_spec.ts b/packages/errors/test/unit/visualSnapshotErrors_spec.ts index a570cd13aded..cbdfa67e1c1a 100644 --- a/packages/errors/test/unit/visualSnapshotErrors_spec.ts +++ b/packages/errors/test/unit/visualSnapshotErrors_spec.ts @@ -487,6 +487,40 @@ describe('visual error templates', () => { 'cypress/integration/app_spec.js', ], }, + payload: {}, + }, + ], + differentParams: [ + { + group: 'foo', + runUrl: 'https://cloud.cypress.io/project/abcd/runs/1', + ciBuildId: 'test-ciBuildId-123', + parameters: { + osName: 'darwin', + osVersion: 'v1', + browserName: 'Electron', + browserVersion: '59.1.2.3', + specs: [ + 'cypress/integration/app_spec.js', + 'cypress/integration/foo_spec.js', + 'cypress/integration/bar_spec.js', + ], + }, + payload: { + differentParams: { + browserName: { + detected: 'Chrome', + expected: 'Electron', + }, + browserVersion: { + detected: '65', + expected: '64', + }, + }, + differentSpecs: [ + 'cypress/integration/foo_spec.js', + ], + }, }, ], } diff --git a/packages/server/__snapshots__/cypress_spec.js b/packages/server/__snapshots__/cypress_spec.js index db7ef02924f8..609fd1d818a2 100644 --- a/packages/server/__snapshots__/cypress_spec.js +++ b/packages/server/__snapshots__/cypress_spec.js @@ -203,11 +203,14 @@ This machine sent the following parameters: { "osName": "darwin", "osVersion": "v1", - "browserName": "Electron", - "browserVersion": "59.1.2.3", - "specs": [ - "cypress/e2e/app.cy.js" - ] + "browserName": "Electron.... (Expected: Electron)", + "browserVersion": "59.1.2.3.... (Expected: 64)", + "differentSpecs": { + "added": [], + "missing": [ + "cypress/integration/foo_spec.js" + ] + } } https://on.cypress.io/parallel-group-params-mismatch diff --git a/packages/server/lib/modes/record.js b/packages/server/lib/modes/record.js index e1eba002ffd6..2caab1693782 100644 --- a/packages/server/lib/modes/record.js +++ b/packages/server/lib/modes/record.js @@ -466,6 +466,7 @@ const createRun = Promise.method((options = {}) => { browserVersion, specs, }, + payload, }) } case 'PARALLEL_DISALLOWED': diff --git a/packages/server/test/integration/cypress_spec.js b/packages/server/test/integration/cypress_spec.js index 129832968d66..1c6803fe6410 100644 --- a/packages/server/test/integration/cypress_spec.js +++ b/packages/server/test/integration/cypress_spec.js @@ -1491,6 +1491,19 @@ describe('lib/cypress', () => { code: 'PARALLEL_GROUP_PARAMS_MISMATCH', payload: { runUrl: 'https://cloud.cypress.io/runs/12345', + differentParams: { + browserName: { + detected: 'Chrome', + expected: 'Electron', + }, + browserVersion: { + detected: '65', + expected: '64', + }, + }, + differentSpecs: [ + 'cypress/integration/foo_spec.js', + ], }, } From 2062670f5c51011de09f679352eb34c7bf1406a9 Mon Sep 17 00:00:00 2001 From: Emily Rohrbough Date: Mon, 12 Dec 2022 13:07:03 -0600 Subject: [PATCH 2/2] fix: recollect session data after validation is successful (#25112) --- .../e2e/commands/sessions/sessions.cy.js | 56 ++++++++++++++----- .../driver/cypress/fixtures/auth/index.html | 18 ++++-- .../driver/src/cy/commands/sessions/index.ts | 16 ++++++ 3 files changed, 73 insertions(+), 17 deletions(-) diff --git a/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js b/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js index dda1626167d4..120b76b9224d 100644 --- a/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js +++ b/packages/driver/cypress/e2e/commands/sessions/sessions.cy.js @@ -256,7 +256,8 @@ describe('cy.session', { retries: 0 }, () => { const sessionStorageData = consoleProps.groups[0].groups[0] expect(sessionStorageData.name).to.contain('Session Storage - (1)') - expect(sessionStorageData.items).to.deep.eq({ cypressAuthToken: '{"body":{"username":"tester"}}' }) + expect(sessionStorageData.items).to.have.property('cypressAuthToken') + expect(sessionStorageData.items.cypressAuthToken).to.contains('"username":"tester"') }) }) @@ -839,29 +840,39 @@ describe('cy.session', { retries: 0 }, () => { let clearPageCount = 0 let sessionGroupId let setup + let slowSetup let validate - const handleSetup = () => { + const handleSetup = (slowLogin = false) => { // create session clears page before running cy.contains('Default blank page').should('not.exist') cy.visit('/fixtures/auth/index.html') cy.contains('You are not logged in') - cy.window().then((win) => { - win.sessionStorage.setItem('cypressAuthToken', JSON.stringify({ body: { username: 'tester' } })) - }) + cy.get('[data-cy=login-same-origin]').click() + cy.get('input').type('tester') + if (slowLogin) { + cy.get('[data-cy=slow-login]').click() + } else { + cy.get('[data-cy=login]').click() + } } + const handleSlowSetup = () => handleSetup(true) + const handleValidate = () => { - // both create & restore session clears page after running + // both create & restore session clears page after running cy.contains('Default blank page').should('not.exist') - cy.visit('/fixtures/auth/index.html') - cy.contains('Welcome tester') + cy.window() + .its('sessionStorage') + .its('cypressAuthToken', { timeout: 5000 }) + .should('contain', '"username":"tester"') } before(() => { setup = cy.stub().callsFake(handleSetup).as('setupSession') + slowSetup = cy.stub().callsFake(handleSlowSetup).as('setupSlowSession') validate = cy.stub().callsFake(handleValidate).as('validateSession') }) @@ -879,7 +890,7 @@ describe('cy.session', { retries: 0 }, () => { resetMocks() clearAllSavedSessions() cy.on('log:added', (attrs, log) => { - if (attrs.name === 'session' || attrs.name === 'sessions_manager' || attrs.name === 'page load' || attrs.alias?.includes('setupSession') || attrs.alias?.includes('validateSession')) { + if (attrs.name === 'session' || attrs.name === 'sessions_manager' || attrs.alias?.includes('setupSession') || attrs.alias?.includes('setupSlowSession') || attrs.alias?.includes('validateSession')) { logs.push(log) if (!sessionGroupId) { sessionGroupId = attrs.id @@ -974,7 +985,8 @@ describe('cy.session', { retries: 0 }, () => { const sessionStorageData = consoleProps.groups[0].groups[0] expect(sessionStorageData.name).to.contain('Session Storage - (1)') - expect(sessionStorageData.items).to.deep.eq({ cypressAuthToken: '{"body":{"username":"tester"}}' }) + expect(sessionStorageData.items).to.have.property('cypressAuthToken') + expect(sessionStorageData.items.cypressAuthToken).to.contains('"username":"tester"') }) }) @@ -985,7 +997,7 @@ describe('cy.session', { retries: 0 }, () => { setupTestContext() cy.log('Creating new session with validation to test against') sessionId = `session-${Cypress.state('test').id}` - cy.session(sessionId, setup, { validate }) + cy.session(sessionId, slowSetup, { validate }) }) it('does not clear the page after command', () => { @@ -993,7 +1005,7 @@ describe('cy.session', { retries: 0 }, () => { }) it('successfully creates new session and validates it', () => { - expect(setup).to.be.calledOnce + expect(slowSetup).to.be.calledOnce expect(validate).to.be.calledOnce expect(clearPageCount, 'total times session cleared the page').to.eq(0) }) @@ -1024,7 +1036,7 @@ describe('cy.session', { retries: 0 }, () => { }) expect(logs[3].get()).to.deep.contain({ - alias: ['setupSession'], + alias: ['setupSlowSession'], group: createNewSessionGroup.id, }) @@ -1040,6 +1052,24 @@ describe('cy.session', { retries: 0 }, () => { group: validateSessionGroup.id, }) }) + + it('has session details in the consoleProps', () => { + const consoleProps = logs[0].get('consoleProps')() + + expect(consoleProps.Command).to.eq('session') + expect(consoleProps.id).to.eq(sessionId) + expect(consoleProps.Domains).to.eq('This session captured data from localhost.') + + expect(consoleProps.groups).to.have.length(1) + expect(consoleProps.groups[0].name).to.eq('localhost data:') + expect(consoleProps.groups[0].groups).to.have.length(1) + + const sessionStorageData = consoleProps.groups[0].groups[0] + + expect(sessionStorageData.name).to.contain('Session Storage - (1)') + expect(sessionStorageData.items).to.have.property('cypressAuthToken') + expect(sessionStorageData.items.cypressAuthToken).to.contains('"username":"tester"') + }) }) describe('create session with failed validation flow', () => { diff --git a/packages/driver/cypress/fixtures/auth/index.html b/packages/driver/cypress/fixtures/auth/index.html index c2406435849a..02f6c680821a 100644 --- a/packages/driver/cypress/fixtures/auth/index.html +++ b/packages/driver/cypress/fixtures/auth/index.html @@ -13,7 +13,7 @@ if (newToken) { - sessionStorage.setItem('cypressAuthToken', decodeURIComponent(newToken)); + sessionStorage.setItem('cypressAuthToken', decodeURIComponent(newToken)) urlParams.delete('token') const newSearchParams = urlParams.toString() @@ -43,13 +43,23 @@ // Add Login button that redirects to the idp const loginBtn = document.createElement("button"); - loginBtn.innerHTML = "Login IDP" - loginBtn.dataset.cy = "login-idp" + loginBtn.innerHTML = "Login Same Origin" + loginBtn.dataset.cy = "login-same-origin" loginBtn.onclick = function () { - window.location.href = `http://www.idp.com:3500/fixtures/auth/idp.html?redirect=${encodeURIComponent(window.location.href)}` + window.location.href = `http://localhost:3500/fixtures/auth/idp.html?redirect=${encodeURIComponent(window.location.href)}` }; document.body.appendChild(loginBtn); + + // Add Login button that redirects to the idp + const loginIDPBtn = document.createElement("button"); + loginIDPBtn.innerHTML = "Login IDP" + loginIDPBtn.dataset.cy = "login-idp" + loginIDPBtn.onclick = function () { + window.location.href = `http://www.idp.com:3500/fixtures/auth/idp.html?redirect=${encodeURIComponent(window.location.href)}` + }; + document.body.appendChild(loginIDPBtn); + // Add Login button that redirects to the idp const loginFoobarBtn = document.createElement("button"); loginFoobarBtn.innerHTML = "Login Foobar" diff --git a/packages/driver/src/cy/commands/sessions/index.ts b/packages/driver/src/cy/commands/sessions/index.ts index 32c1fa680f9a..3d168ccd2a54 100644 --- a/packages/driver/src/cy/commands/sessions/index.ts +++ b/packages/driver/src/cy/commands/sessions/index.ts @@ -185,6 +185,8 @@ export default function (Commands, Cypress, cy) { consoleProps: () => { return { Step: statusMap.stepName(step), + Message: 'The following is the collected session data after the session was successfully setup:', + ...getConsoleProps(existingSession), } }, @@ -396,6 +398,20 @@ export default function (Commands, Cypress, cy) { } } + // collect all session data again that may have been updated during the validation check + const data = await sessions.getCurrentSessionData() + + _.extend(existingSession, data) + validateLog.set({ + consoleProps: () => { + return { + Step: 'Validate Session', + Message: 'The following is the collected session data after the session was successfully validated:', + ...getConsoleProps(existingSession), + } + }, + }) + return isValidSession })