From e3161a0b37c5895b060c9a2883906409395f59b5 Mon Sep 17 00:00:00 2001 From: Matt Henkes Date: Thu, 21 Apr 2022 09:32:47 -0500 Subject: [PATCH] chore: [Multi-domain] Break out separate CI tasks to test the driver with experimentalSessionAndOrigin on (#21148) * Attempt to run tests without the feature flag and the multi-domain folder * do it for real this time dummy * trying circleci changes * valid file??? * moar changes! * i am a master of bash scripting * try another exclude pattern * one more time with feeling * lets do it again * maybe this? * different parallel group * lets try this * updating tests phase 1 * updating tests phase 2 * Apply suggestions from code review Co-authored-by: Matt Schile * Update tests * Run more tests * Re-locate system test * Flex message for firefox Co-authored-by: Matt Schile --- circle.yml | 77 ++++++++++++- packages/driver/cypress.json | 3 +- .../integration/commands/navigation_spec.js | 104 ++++++++++++------ .../e2e/multi-domain/stability_spec.ts | 34 ++++++ .../cypress/integration/e2e/stability_spec.js | 32 ------ packages/driver/package.json | 4 +- .../__snapshots__/navigation_spec.ts.js | 95 ---------------- .../navigation_cross_origin_errors.ts | 10 -- system-tests/test/navigation_spec.ts | 48 -------- 9 files changed, 183 insertions(+), 224 deletions(-) create mode 100644 packages/driver/cypress/integration/e2e/multi-domain/stability_spec.ts delete mode 100644 packages/driver/cypress/integration/e2e/stability_spec.js delete mode 100644 system-tests/__snapshots__/navigation_spec.ts.js delete mode 100644 system-tests/projects/e2e/cypress/integration/navigation_cross_origin_errors.ts delete mode 100644 system-tests/test/navigation_spec.ts diff --git a/circle.yml b/circle.yml index 24c0fb2c3f9b..49a982791e36 100644 --- a/circle.yml +++ b/circle.yml @@ -406,6 +406,10 @@ commands: description: chrome channel to install type: string default: '' + experimentalSessionAndOrigin: + description: experimental flag to apply + type: boolean + default: false steps: - restore_cached_workspace - when: @@ -423,8 +427,13 @@ commands: if [[ -v MAIN_RECORD_KEY ]]; then # internal PR - CYPRESS_RECORD_KEY=$MAIN_RECORD_KEY \ - yarn cypress:run --record --parallel --group 5x-driver-<> --browser <> + if <>; then + CYPRESS_RECORD_KEY=$MAIN_RECORD_KEY \ + yarn cypress:run-experimentalSessionAndOrigin --record --parallel --group 5x-driver-<>-experimentalSessionAndOrigin --browser <> + else + CYPRESS_RECORD_KEY=$MAIN_RECORD_KEY \ + yarn cypress:run --record --parallel --group 5x-driver-<> --browser <> + fi else # external PR TESTFILES=$(circleci tests glob "cypress/integration/**/*spec.*" | circleci tests split --total=$CIRCLE_NODE_TOTAL) @@ -433,7 +442,11 @@ commands: if [[ -z "$TESTFILES" ]]; then echo "Empty list of test files" fi - yarn cypress:run --browser <> --spec $TESTFILES + if <>; then + yarn cypress:run-experimentalSessionAndOrigin --browser <> --spec $TESTFILES + else + yarn cypress:run --browser <> --spec $TESTFILES + fi fi working_directory: packages/driver - verify-mocha-results @@ -1294,6 +1307,44 @@ jobs: - run-driver-integration-tests: browser: electron + driver-integration-tests-chrome-experimentalSessionAndOrigin: + <<: *defaults + resource_class: medium + parallelism: 5 + steps: + - run-driver-integration-tests: + browser: chrome + install-chrome-channel: stable + experimentalSessionAndOrigin: true + + driver-integration-tests-chrome-beta-experimentalSessionAndOrigin: + <<: *defaults + resource_class: medium + parallelism: 5 + steps: + - run-driver-integration-tests: + browser: chrome:beta + install-chrome-channel: beta + experimentalSessionAndOrigin: true + + driver-integration-tests-firefox-experimentalSessionAndOrigin: + <<: *defaults + resource_class: medium + parallelism: 5 + steps: + - run-driver-integration-tests: + browser: firefox + experimentalSessionAndOrigin: true + + driver-integration-tests-electron-experimentalSessionAndOrigin: + <<: *defaults + resource_class: medium + parallelism: 5 + steps: + - run-driver-integration-tests: + browser: electron + experimentalSessionAndOrigin: true + desktop-gui-integration-tests-7x: <<: *defaults parallelism: 7 @@ -2084,6 +2135,22 @@ linux-workflow: &linux-workflow context: test-runner:cypress-record-key requires: - build + - driver-integration-tests-chrome-experimentalSessionAndOrigin: + context: test-runner:cypress-record-key + requires: + - build + - driver-integration-tests-chrome-beta-experimentalSessionAndOrigin: + context: test-runner:cypress-record-key + requires: + - build + - driver-integration-tests-firefox-experimentalSessionAndOrigin: + context: test-runner:cypress-record-key + requires: + - build + - driver-integration-tests-electron-experimentalSessionAndOrigin: + context: test-runner:cypress-record-key + requires: + - build - runner-integration-tests-chrome: context: [test-runner:cypress-record-key, test-runner:percy] requires: @@ -2180,6 +2247,10 @@ linux-workflow: &linux-workflow - driver-integration-tests-chrome - driver-integration-tests-chrome-beta - driver-integration-tests-electron + - driver-integration-tests-firefox-experimentalSessionAndOrigin + - driver-integration-tests-chrome-experimentalSessionAndOrigin + - driver-integration-tests-chrome-beta-experimentalSessionAndOrigin + - driver-integration-tests-electron-experimentalSessionAndOrigin - system-tests-non-root - system-tests-firefox - system-tests-electron diff --git a/packages/driver/cypress.json b/packages/driver/cypress.json index 911238c1181a..f6d381383ca1 100644 --- a/packages/driver/cypress.json +++ b/packages/driver/cypress.json @@ -13,6 +13,5 @@ "retries": { "runMode": 2, "openMode": 0 - }, - "experimentalSessionAndOrigin": true + } } diff --git a/packages/driver/cypress/integration/commands/navigation_spec.js b/packages/driver/cypress/integration/commands/navigation_spec.js index b6b4f0155e3b..cf3a9b5c94ef 100644 --- a/packages/driver/cypress/integration/commands/navigation_spec.js +++ b/packages/driver/cypress/integration/commands/navigation_spec.js @@ -1409,10 +1409,11 @@ describe('src/cy/commands/navigation', () => { it('throws when attempting to visit a 2nd domain on different port', function (done) { cy.on('fail', (err) => { const { lastLog } = this + const experimentalMessage = Cypress.config('experimentalSessionAndOrigin') ? `You likely forgot to use \`cy.origin()\`:\n` : `In order to visit a different origin, you can enable the \`experimentalSessionAndOrigin\` flag and use \`cy.origin()\`:\n` expect(err.message).to.equal(stripIndent`\ \`cy.visit()\` failed because you are attempting to visit a URL that is of a different origin.\n - You likely forgot to use \`cy.origin()\`:\n + ${experimentalMessage} \`cy.visit('http://localhost:3500/fixtures/generic.html')\` \`\`\n \`cy.origin('http://localhost:3501', () => {\` @@ -1441,10 +1442,11 @@ describe('src/cy/commands/navigation', () => { it('throws when attempting to visit a 2nd domain on different protocol', function (done) { cy.on('fail', (err) => { const { lastLog } = this + const experimentalMessage = Cypress.config('experimentalSessionAndOrigin') ? `You likely forgot to use \`cy.origin()\`:\n` : `In order to visit a different origin, you can enable the \`experimentalSessionAndOrigin\` flag and use \`cy.origin()\`:\n` expect(err.message).to.equal(stripIndent`\ \`cy.visit()\` failed because you are attempting to visit a URL that is of a different origin.\n - You likely forgot to use \`cy.origin()\`:\n + ${experimentalMessage} \`cy.visit('http://localhost:3500/fixtures/generic.html')\` \`\`\n \`cy.origin('https://localhost:3502', () => {\` @@ -1473,10 +1475,11 @@ describe('src/cy/commands/navigation', () => { it('throws when attempting to visit a 2nd domain on different superdomain', function (done) { cy.on('fail', (err) => { const { lastLog } = this + const experimentalMessage = Cypress.config('experimentalSessionAndOrigin') ? `You likely forgot to use \`cy.origin()\`:\n` : `In order to visit a different origin, you can enable the \`experimentalSessionAndOrigin\` flag and use \`cy.origin()\`:\n` expect(err.message).to.equal(stripIndent`\ \`cy.visit()\` failed because you are attempting to visit a URL that is of a different origin.\n - You likely forgot to use \`cy.origin()\`:\n + ${experimentalMessage} \`cy.visit('http://localhost:3500/fixtures/generic.html')\` \`\`\n \`cy.origin('http://foobar.com:3500', () => {\` @@ -1505,10 +1508,11 @@ describe('src/cy/commands/navigation', () => { it('throws attempting to visit 2 unique ip addresses', function (done) { cy.on('fail', (err) => { const { lastLog } = this + const experimentalMessage = Cypress.config('experimentalSessionAndOrigin') ? `You likely forgot to use \`cy.origin()\`:\n` : `In order to visit a different origin, you can enable the \`experimentalSessionAndOrigin\` flag and use \`cy.origin()\`:\n` expect(err.message).to.equal(stripIndent`\ \`cy.visit()\` failed because you are attempting to visit a URL that is of a different origin.\n - You likely forgot to use \`cy.origin()\`:\n + ${experimentalMessage} \`cy.visit('http://127.0.0.1:3500/fixtures/generic.html')\` \`\`\n \`cy.origin('http://0.0.0.0:3500', () => {\` @@ -2145,6 +2149,53 @@ describe('src/cy/commands/navigation', () => { .get('#does-not-exist', { timeout: 200 }).should('have.class', 'foo') }) + it('displays cross origin failures when navigating to a cross origin', { pageLoadTimeout: 3000 }, function (done) { + cy.on('fail', (err) => { + const { lastLog } = this + + if (Cypress.config('experimentalSessionAndOrigin')) { + // When the experimentalSessionAndOrigin feature is enabled, we will timeout and display this message. + expect(err.message).to.equal(stripIndent`\ + Timed out after waiting \`3000ms\` for your remote page to load on origin(s):\n + - \`http://localhost:3500\`\n + A cross-origin request for \`http://www.foobar.com:3500/fixtures/multi-domain-secondary.html\` was detected.\n + A command that triggers cross-origin navigation must be immediately followed by a \`cy.origin()\` command:\n + \`cy.origin(\'http://foobar.com:3500\', () => {\` + \` \` + \`})\`\n + If the cross-origin request was an intermediary state, you can try increasing the \`pageLoadTimeout\` value in \`cypress.json\` to wait longer.\n + Browsers will not fire the \`load\` event until all stylesheets and scripts are done downloading.\n + When this \`load\` event occurs, Cypress will continue running commands.`) + + expect(err.docsUrl).to.eq('https://on.cypress.io/origin') + } else { + const error = Cypress.isBrowser('firefox') ? 'Permission denied to access property "document" on cross-origin object' : 'Blocked a frame with origin "http://localhost:3500" from accessing a cross-origin frame.' + + // When the experimentalSessionAndOrigin feature is disabled, we will immediately and display this message. + expect(err.message).to.equal(stripIndent`\ + Cypress detected a cross origin error happened on page load:\n + > ${error}\n + Before the page load, you were bound to the origin policy:\n + > http://localhost:3500\n + A cross origin error happens when your application navigates to a new URL which does not match the origin policy above.\n + A new URL does not match the origin policy if the 'protocol', 'port' (if specified), and/or 'host' (unless of the same superdomain) are different.\n + Cypress does not allow you to navigate to a different origin URL within a single test.\n + You may need to restructure some of your test code to avoid this problem.\n + Alternatively you can also disable Chrome Web Security in Chromium-based browsers which will turn off this restriction by setting { chromeWebSecurity: false } in \`cypress.json\`.`) + + expect(err.docsUrl).to.eq('https://on.cypress.io/cross-origin-violation') + } + + assertLogLength(this.logs, 6) + expect(lastLog.get('error')).to.eq(err) + + done() + }) + + cy.visit('/fixtures/multi-domain.html') + cy.get('a[data-cy="cross-origin-secondary-link"]').click() + }) + return null }) }) @@ -2269,41 +2320,28 @@ describe('src/cy/commands/navigation', () => { }) }) - it('waits for stability at the end of the command queue when not stable', { experimentalSessionAndOrigin: false }, (done) => { + it('tests waiting on stability at the end of the command queue', (done) => { cy .visit('/fixtures/generic.html') .then((win) => { - cy.on('window:load', () => { + // We do not wait if the experimentalSessionAndOrigin feature is enabled + if (Cypress.config('experimentalSessionAndOrigin')) { + const onLoad = cy.spy() + + cy.on('window:load', onLoad) + cy.on('command:queue:end', () => { + expect(onLoad).not.have.been.called done() }) - }) - - cy.on('command:queue:before:end', () => { - // force us to become unstable immediately - // else the beforeunload event fires at the end - // of the tick which is too late - cy.isStable(false, 'testing') - - win.location.href = '/timeout?ms=100' - }) - - return null - }) - }) - - it('does not wait for stability at the end of the command queue when not stable with experimentalSessionAndOrigin', (done) => { - const onLoad = cy.spy() - - cy - .visit('/fixtures/generic.html') - .then((win) => { - cy.on('window:load', onLoad) - - cy.on('command:queue:end', () => { - expect(onLoad).not.have.been.called - done() - }) + } else { + // We do wait if the experimentalSessionAndOrigin feature is not enabled + cy.on('window:load', () => { + cy.on('command:queue:end', () => { + done() + }) + }) + } cy.on('command:queue:before:end', () => { // force us to become unstable immediately diff --git a/packages/driver/cypress/integration/e2e/multi-domain/stability_spec.ts b/packages/driver/cypress/integration/e2e/multi-domain/stability_spec.ts new file mode 100644 index 000000000000..26486e41c651 --- /dev/null +++ b/packages/driver/cypress/integration/e2e/multi-domain/stability_spec.ts @@ -0,0 +1,34 @@ +describe('stability', () => { + describe('before each transitions', () => { + describe('transitioning from a before block to an it block while unstable', () => { + beforeEach(() => { + cy.visit('/fixtures/auth/index.html') + cy.window().then((win) => { + win.location.href = 'http://localhost:3500/timeout?ms=1000' + }) + }) + + it('fails if the page does not load within the page load timeout', { defaultCommandTimeout: 50, pageLoadTimeout: 500 }, (done) => { + cy.on('fail', (err) => { + expect(err.message).to.include(`Timed out after waiting \`500ms\` for your remote page to load.`) + done() + }) + + cy.get('[data-cy="login-idp"]').click() // Takes you to idp.com + }) + + it('waits for the page to load before running the command', { defaultCommandTimeout: 50 }, () => { + cy.get('body').invoke('text').should('equal', 'timeout') + }) + + it('will retry and fail the command after the page loads', { defaultCommandTimeout: 50 }, (done) => { + cy.on('fail', (err) => { + expect(err.message).to.include(`Timed out retrying after 50ms: expected 'timeout' to equal 'not timeout'`) + done() + }) + + cy.get('body').invoke('text').should('equal', 'not timeout') + }) + }) + }) +}) diff --git a/packages/driver/cypress/integration/e2e/stability_spec.js b/packages/driver/cypress/integration/e2e/stability_spec.js deleted file mode 100644 index b6d541845f79..000000000000 --- a/packages/driver/cypress/integration/e2e/stability_spec.js +++ /dev/null @@ -1,32 +0,0 @@ -describe('stability', () => { - describe('transitioning from a before block to an it block while unstable', () => { - beforeEach(() => { - cy.visit('/fixtures/auth/index.html') - cy.window().then((win) => { - win.location.href = 'http://localhost:3500/timeout?ms=1000' - }) - }) - - it('fails if the page does not load within the page load timeout', { defaultCommandTimeout: 50, pageLoadTimeout: 500 }, (done) => { - cy.on('fail', (err) => { - expect(err.message).to.include(`Timed out after waiting \`500ms\` for your remote page to load.`) - done() - }) - - cy.get('[data-cy="login-idp"]').click() // Takes you to idp.com - }) - - it('waits for the page to load before running the command', { defaultCommandTimeout: 50 }, () => { - cy.get('body').invoke('text').should('equal', 'timeout') - }) - - it('will retry and fail the command after the page loads', { defaultCommandTimeout: 50 }, (done) => { - cy.on('fail', (err) => { - expect(err.message).to.include(`Timed out retrying after 50ms: expected 'timeout' to equal 'not timeout'`) - done() - }) - - cy.get('body').invoke('text').should('equal', 'not timeout') - }) - }) -}) diff --git a/packages/driver/package.json b/packages/driver/package.json index aca090231a17..6641bc4c0d00 100644 --- a/packages/driver/package.json +++ b/packages/driver/package.json @@ -5,7 +5,9 @@ "scripts": { "clean-deps": "rm -rf node_modules", "cypress:open": "node ../../scripts/cypress open", - "cypress:run": "node ../../scripts/cypress run", + "cypress:run": "node ../../scripts/cypress run --spec \"cypress/integration/*/*\",\"cypress/integration/*/!(multi-domain)/**/*\"", + "cypress:open-experimentalSessionAndOrigin": "node ../../scripts/cypress open --config experimentalSessionAndOrigin=true", + "cypress:run-experimentalSessionAndOrigin": "node ../../scripts/cypress run --config experimentalSessionAndOrigin=true", "postinstall": "patch-package", "start": "node -e 'console.log(require(`chalk`).red(`\nError:\n\tRunning \\`yarn start\\` is no longer needed for driver/cypress tests.\n\tWe now automatically spawn the server in the pluginsFile.\n\tChanges to the server will be watched and reloaded automatically.`))'" }, diff --git a/system-tests/__snapshots__/navigation_spec.ts.js b/system-tests/__snapshots__/navigation_spec.ts.js deleted file mode 100644 index 38df39d3a9c0..000000000000 --- a/system-tests/__snapshots__/navigation_spec.ts.js +++ /dev/null @@ -1,95 +0,0 @@ -exports['e2e cross origin navigation / captures cross origin failures when "experimentalSessionAndOrigin" config value is falsy'] = ` - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (navigation_cross_origin_errors.ts) │ - │ Searched: cypress/integration/navigation_cross_origin_errors.ts │ - │ Experiments: experimentalSessionAndOrigin=false │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: navigation_cross_origin_errors.ts (1 of 1) - - - navigation cross origin errors - 1) displays cross origin failures when "experimentalSessionAndOrigin" is turned off - - - 0 passing - 1 failing - - 1) navigation cross origin errors - displays cross origin failures when "experimentalSessionAndOrigin" is turned off: - CypressError: Cypress detected a cross origin error happened on page load: - - > [Cross origin error message] - -Before the page load, you were bound to the origin policy: - - > http://localhost:13370 - -A cross origin error happens when your application navigates to a new URL which does not match the origin policy above. - -A new URL does not match the origin policy if the 'protocol', 'port' (if specified), and/or 'host' (unless of the same superdomain) are different. - -Cypress does not allow you to navigate to a different origin URL within a single test. - -You may need to restructure some of your test code to avoid this problem. - -Alternatively you can also disable Chrome Web Security in Chromium-based browsers which will turn off this restriction by setting { chromeWebSecurity: false } in \`cypress.json\`. - -https://on.cypress.io/cross-origin-violation - [stack trace lines] - - - - - (Results) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 1 │ - │ Passing: 0 │ - │ Failing: 1 │ - │ Pending: 0 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: true │ - │ Duration: X seconds │ - │ Spec Ran: navigation_cross_origin_errors.ts │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/navigation_cross_origin_errors.ts/navigation cr (1280x720) - oss origin errors -- displays cross origin failures when experimentalSessionAndO - rigin is turned off (failed).png - - - (Video) - - - Started processing: Compressing to 32 CRF - - Finished processing: /XXX/XXX/XXX/cypress/videos/navigation_cross_origin_errors. (X second) - ts.mp4 - - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✖ navigation_cross_origin_errors.ts XX:XX 1 - 1 - - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✖ 1 of 1 failed (100%) XX:XX 1 - 1 - - - - -` diff --git a/system-tests/projects/e2e/cypress/integration/navigation_cross_origin_errors.ts b/system-tests/projects/e2e/cypress/integration/navigation_cross_origin_errors.ts deleted file mode 100644 index 7e1654adc89a..000000000000 --- a/system-tests/projects/e2e/cypress/integration/navigation_cross_origin_errors.ts +++ /dev/null @@ -1,10 +0,0 @@ -describe('navigation cross origin errors', () => { - it('displays cross origin failures when "experimentalSessionAndOrigin" is turned off', function () { - // @ts-ignore - cy.visit('/jquery.html').window().then((win) => { - const constructedCrossOriginAnchor = win.$(`cross origin`).appendTo(win.document.body) - - constructedCrossOriginAnchor.get(0).click() - }) - }) -}) diff --git a/system-tests/test/navigation_spec.ts b/system-tests/test/navigation_spec.ts deleted file mode 100644 index b4f83ade06e0..000000000000 --- a/system-tests/test/navigation_spec.ts +++ /dev/null @@ -1,48 +0,0 @@ -import systemTests, { expect } from '../lib/system-tests' - -const PORT = 13370 -const onServer = function (app) { - app.get('/cross_origin.html', (req, res) => { - res.send('

cross origin

') - }) -} - -describe('e2e cross origin navigation', () => { - systemTests.setup({ - servers: [{ - port: 4466, - onServer, - }], - settings: { - hosts: { - '*.foobar.com': '127.0.0.1', - }, - }, - }) - - // TODO: convert to cypress-in-cypress test if possible - // https://github.com/cypress-io/cypress/issues/20973 - systemTests.it('captures cross origin failures when "experimentalSessionAndOrigin" config value is falsy', { - // keep the port the same to prevent issues with the snapshot - port: PORT, - spec: 'navigation_cross_origin_errors.ts', - browser: ['chrome', 'electron'], - snapshot: true, - expectedExitCode: 1, - config: { - experimentalSessionAndOrigin: false, - }, - async onRun (exec) { - const res = await exec() - - expect(res.stdout).to.contain('Cypress detected a cross origin error happened on page load') - expect(res.stdout).to.contain('Before the page load, you were bound to the origin policy:') - expect(res.stdout).to.contain('A cross origin error happens when your application navigates to a new URL which does not match the origin policy above.') - expect(res.stdout).to.contain('A new URL does not match the origin policy if the \'protocol\', \'port\' (if specified), and/or \'host\' (unless of the same superdomain) are different.') - expect(res.stdout).to.contain('Cypress does not allow you to navigate to a different origin URL within a single test.') - expect(res.stdout).to.contain('You may need to restructure some of your test code to avoid this problem.') - expect(res.stdout).to.contain('Alternatively you can also disable Chrome Web Security in Chromium-based browsers which will turn off this restriction by setting { chromeWebSecurity: false } in `cypress.json`.') - expect(res.stdout).to.contain('https://on.cypress.io/cross-origin-violation') - }, - }) -})