Skip to content

Commit

Permalink
chore: [Multi-domain] Break out separate CI tasks to test the driver …
Browse files Browse the repository at this point in the history
…with experimentalSessionAndOrigin on (#21148)

* Attempt to run tests without the feature flag and the multi-domain folder

* do it for real this time dummy

* trying circleci changes

* valid file???

* moar changes!

* i am a master of bash scripting

* try another exclude pattern

* one more time with feeling

* lets do it again

* maybe this?

* different parallel group

* lets try this

* updating tests phase 1

* updating tests phase 2

* Apply suggestions from code review

Co-authored-by: Matt Schile <mschile@gmail.com>

* Update tests

* Run more tests

* Re-locate system test

* Flex message for firefox

Co-authored-by: Matt Schile <mschile@gmail.com>
  • Loading branch information
mjhenkes and mschile committed Apr 21, 2022
1 parent d7ce865 commit e3161a0
Show file tree
Hide file tree
Showing 9 changed files with 183 additions and 224 deletions.
77 changes: 74 additions & 3 deletions circle.yml
Expand Up @@ -406,6 +406,10 @@ commands:
description: chrome channel to install
type: string
default: ''
experimentalSessionAndOrigin:
description: experimental flag to apply
type: boolean
default: false
steps:
- restore_cached_workspace
- when:
Expand All @@ -423,8 +427,13 @@ commands:
if [[ -v MAIN_RECORD_KEY ]]; then
# internal PR
CYPRESS_RECORD_KEY=$MAIN_RECORD_KEY \
yarn cypress:run --record --parallel --group 5x-driver-<<parameters.browser>> --browser <<parameters.browser>>
if <<parameters.experimentalSessionAndOrigin>>; then
CYPRESS_RECORD_KEY=$MAIN_RECORD_KEY \
yarn cypress:run-experimentalSessionAndOrigin --record --parallel --group 5x-driver-<<parameters.browser>>-experimentalSessionAndOrigin --browser <<parameters.browser>>
else
CYPRESS_RECORD_KEY=$MAIN_RECORD_KEY \
yarn cypress:run --record --parallel --group 5x-driver-<<parameters.browser>> --browser <<parameters.browser>>
fi
else
# external PR
TESTFILES=$(circleci tests glob "cypress/integration/**/*spec.*" | circleci tests split --total=$CIRCLE_NODE_TOTAL)
Expand All @@ -433,7 +442,11 @@ commands:
if [[ -z "$TESTFILES" ]]; then
echo "Empty list of test files"
fi
yarn cypress:run --browser <<parameters.browser>> --spec $TESTFILES
if <<parameters.experimentalSessionAndOrigin>>; then
yarn cypress:run-experimentalSessionAndOrigin --browser <<parameters.browser>> --spec $TESTFILES
else
yarn cypress:run --browser <<parameters.browser>> --spec $TESTFILES
fi
fi
working_directory: packages/driver
- verify-mocha-results
Expand Down Expand Up @@ -1294,6 +1307,44 @@ jobs:
- run-driver-integration-tests:
browser: electron

driver-integration-tests-chrome-experimentalSessionAndOrigin:
<<: *defaults
resource_class: medium
parallelism: 5
steps:
- run-driver-integration-tests:
browser: chrome
install-chrome-channel: stable
experimentalSessionAndOrigin: true

driver-integration-tests-chrome-beta-experimentalSessionAndOrigin:
<<: *defaults
resource_class: medium
parallelism: 5
steps:
- run-driver-integration-tests:
browser: chrome:beta
install-chrome-channel: beta
experimentalSessionAndOrigin: true

driver-integration-tests-firefox-experimentalSessionAndOrigin:
<<: *defaults
resource_class: medium
parallelism: 5
steps:
- run-driver-integration-tests:
browser: firefox
experimentalSessionAndOrigin: true

driver-integration-tests-electron-experimentalSessionAndOrigin:
<<: *defaults
resource_class: medium
parallelism: 5
steps:
- run-driver-integration-tests:
browser: electron
experimentalSessionAndOrigin: true

desktop-gui-integration-tests-7x:
<<: *defaults
parallelism: 7
Expand Down Expand Up @@ -2084,6 +2135,22 @@ linux-workflow: &linux-workflow
context: test-runner:cypress-record-key
requires:
- build
- driver-integration-tests-chrome-experimentalSessionAndOrigin:
context: test-runner:cypress-record-key
requires:
- build
- driver-integration-tests-chrome-beta-experimentalSessionAndOrigin:
context: test-runner:cypress-record-key
requires:
- build
- driver-integration-tests-firefox-experimentalSessionAndOrigin:
context: test-runner:cypress-record-key
requires:
- build
- driver-integration-tests-electron-experimentalSessionAndOrigin:
context: test-runner:cypress-record-key
requires:
- build
- runner-integration-tests-chrome:
context: [test-runner:cypress-record-key, test-runner:percy]
requires:
Expand Down Expand Up @@ -2180,6 +2247,10 @@ linux-workflow: &linux-workflow
- driver-integration-tests-chrome
- driver-integration-tests-chrome-beta
- driver-integration-tests-electron
- driver-integration-tests-firefox-experimentalSessionAndOrigin
- driver-integration-tests-chrome-experimentalSessionAndOrigin
- driver-integration-tests-chrome-beta-experimentalSessionAndOrigin
- driver-integration-tests-electron-experimentalSessionAndOrigin
- system-tests-non-root
- system-tests-firefox
- system-tests-electron
Expand Down
3 changes: 1 addition & 2 deletions packages/driver/cypress.json
Expand Up @@ -13,6 +13,5 @@
"retries": {
"runMode": 2,
"openMode": 0
},
"experimentalSessionAndOrigin": true
}
}
104 changes: 71 additions & 33 deletions packages/driver/cypress/integration/commands/navigation_spec.js
Expand Up @@ -1409,10 +1409,11 @@ describe('src/cy/commands/navigation', () => {
it('throws when attempting to visit a 2nd domain on different port', function (done) {
cy.on('fail', (err) => {
const { lastLog } = this
const experimentalMessage = Cypress.config('experimentalSessionAndOrigin') ? `You likely forgot to use \`cy.origin()\`:\n` : `In order to visit a different origin, you can enable the \`experimentalSessionAndOrigin\` flag and use \`cy.origin()\`:\n`

expect(err.message).to.equal(stripIndent`\
\`cy.visit()\` failed because you are attempting to visit a URL that is of a different origin.\n
You likely forgot to use \`cy.origin()\`:\n
${experimentalMessage}
\`cy.visit('http://localhost:3500/fixtures/generic.html')\`
\`<commands targeting http://localhost:3500 go here>\`\n
\`cy.origin('http://localhost:3501', () => {\`
Expand Down Expand Up @@ -1441,10 +1442,11 @@ describe('src/cy/commands/navigation', () => {
it('throws when attempting to visit a 2nd domain on different protocol', function (done) {
cy.on('fail', (err) => {
const { lastLog } = this
const experimentalMessage = Cypress.config('experimentalSessionAndOrigin') ? `You likely forgot to use \`cy.origin()\`:\n` : `In order to visit a different origin, you can enable the \`experimentalSessionAndOrigin\` flag and use \`cy.origin()\`:\n`

expect(err.message).to.equal(stripIndent`\
\`cy.visit()\` failed because you are attempting to visit a URL that is of a different origin.\n
You likely forgot to use \`cy.origin()\`:\n
${experimentalMessage}
\`cy.visit('http://localhost:3500/fixtures/generic.html')\`
\`<commands targeting http://localhost:3500 go here>\`\n
\`cy.origin('https://localhost:3502', () => {\`
Expand Down Expand Up @@ -1473,10 +1475,11 @@ describe('src/cy/commands/navigation', () => {
it('throws when attempting to visit a 2nd domain on different superdomain', function (done) {
cy.on('fail', (err) => {
const { lastLog } = this
const experimentalMessage = Cypress.config('experimentalSessionAndOrigin') ? `You likely forgot to use \`cy.origin()\`:\n` : `In order to visit a different origin, you can enable the \`experimentalSessionAndOrigin\` flag and use \`cy.origin()\`:\n`

expect(err.message).to.equal(stripIndent`\
\`cy.visit()\` failed because you are attempting to visit a URL that is of a different origin.\n
You likely forgot to use \`cy.origin()\`:\n
${experimentalMessage}
\`cy.visit('http://localhost:3500/fixtures/generic.html')\`
\`<commands targeting http://localhost:3500 go here>\`\n
\`cy.origin('http://foobar.com:3500', () => {\`
Expand Down Expand Up @@ -1505,10 +1508,11 @@ describe('src/cy/commands/navigation', () => {
it('throws attempting to visit 2 unique ip addresses', function (done) {
cy.on('fail', (err) => {
const { lastLog } = this
const experimentalMessage = Cypress.config('experimentalSessionAndOrigin') ? `You likely forgot to use \`cy.origin()\`:\n` : `In order to visit a different origin, you can enable the \`experimentalSessionAndOrigin\` flag and use \`cy.origin()\`:\n`

expect(err.message).to.equal(stripIndent`\
\`cy.visit()\` failed because you are attempting to visit a URL that is of a different origin.\n
You likely forgot to use \`cy.origin()\`:\n
${experimentalMessage}
\`cy.visit('http://127.0.0.1:3500/fixtures/generic.html')\`
\`<commands targeting http://127.0.0.1:3500 go here>\`\n
\`cy.origin('http://0.0.0.0:3500', () => {\`
Expand Down Expand Up @@ -2145,6 +2149,53 @@ describe('src/cy/commands/navigation', () => {
.get('#does-not-exist', { timeout: 200 }).should('have.class', 'foo')
})

it('displays cross origin failures when navigating to a cross origin', { pageLoadTimeout: 3000 }, function (done) {
cy.on('fail', (err) => {
const { lastLog } = this

if (Cypress.config('experimentalSessionAndOrigin')) {
// When the experimentalSessionAndOrigin feature is enabled, we will timeout and display this message.
expect(err.message).to.equal(stripIndent`\
Timed out after waiting \`3000ms\` for your remote page to load on origin(s):\n
- \`http://localhost:3500\`\n
A cross-origin request for \`http://www.foobar.com:3500/fixtures/multi-domain-secondary.html\` was detected.\n
A command that triggers cross-origin navigation must be immediately followed by a \`cy.origin()\` command:\n
\`cy.origin(\'http://foobar.com:3500\', () => {\`
\` <commands targeting http://www.foobar.com:3500 go here>\`
\`})\`\n
If the cross-origin request was an intermediary state, you can try increasing the \`pageLoadTimeout\` value in \`cypress.json\` to wait longer.\n
Browsers will not fire the \`load\` event until all stylesheets and scripts are done downloading.\n
When this \`load\` event occurs, Cypress will continue running commands.`)

expect(err.docsUrl).to.eq('https://on.cypress.io/origin')
} else {
const error = Cypress.isBrowser('firefox') ? 'Permission denied to access property "document" on cross-origin object' : 'Blocked a frame with origin "http://localhost:3500" from accessing a cross-origin frame.'

// When the experimentalSessionAndOrigin feature is disabled, we will immediately and display this message.
expect(err.message).to.equal(stripIndent`\
Cypress detected a cross origin error happened on page load:\n
> ${error}\n
Before the page load, you were bound to the origin policy:\n
> http://localhost:3500\n
A cross origin error happens when your application navigates to a new URL which does not match the origin policy above.\n
A new URL does not match the origin policy if the 'protocol', 'port' (if specified), and/or 'host' (unless of the same superdomain) are different.\n
Cypress does not allow you to navigate to a different origin URL within a single test.\n
You may need to restructure some of your test code to avoid this problem.\n
Alternatively you can also disable Chrome Web Security in Chromium-based browsers which will turn off this restriction by setting { chromeWebSecurity: false } in \`cypress.json\`.`)

expect(err.docsUrl).to.eq('https://on.cypress.io/cross-origin-violation')
}

assertLogLength(this.logs, 6)
expect(lastLog.get('error')).to.eq(err)

done()
})

cy.visit('/fixtures/multi-domain.html')
cy.get('a[data-cy="cross-origin-secondary-link"]').click()
})

return null
})
})
Expand Down Expand Up @@ -2269,41 +2320,28 @@ describe('src/cy/commands/navigation', () => {
})
})

it('waits for stability at the end of the command queue when not stable', { experimentalSessionAndOrigin: false }, (done) => {
it('tests waiting on stability at the end of the command queue', (done) => {
cy
.visit('/fixtures/generic.html')
.then((win) => {
cy.on('window:load', () => {
// We do not wait if the experimentalSessionAndOrigin feature is enabled
if (Cypress.config('experimentalSessionAndOrigin')) {
const onLoad = cy.spy()

cy.on('window:load', onLoad)

cy.on('command:queue:end', () => {
expect(onLoad).not.have.been.called
done()
})
})

cy.on('command:queue:before:end', () => {
// force us to become unstable immediately
// else the beforeunload event fires at the end
// of the tick which is too late
cy.isStable(false, 'testing')

win.location.href = '/timeout?ms=100'
})

return null
})
})

it('does not wait for stability at the end of the command queue when not stable with experimentalSessionAndOrigin', (done) => {
const onLoad = cy.spy()

cy
.visit('/fixtures/generic.html')
.then((win) => {
cy.on('window:load', onLoad)

cy.on('command:queue:end', () => {
expect(onLoad).not.have.been.called
done()
})
} else {
// We do wait if the experimentalSessionAndOrigin feature is not enabled
cy.on('window:load', () => {
cy.on('command:queue:end', () => {
done()
})
})
}

cy.on('command:queue:before:end', () => {
// force us to become unstable immediately
Expand Down
@@ -0,0 +1,34 @@
describe('stability', () => {
describe('before each transitions', () => {
describe('transitioning from a before block to an it block while unstable', () => {
beforeEach(() => {
cy.visit('/fixtures/auth/index.html')
cy.window().then((win) => {
win.location.href = 'http://localhost:3500/timeout?ms=1000'
})
})

it('fails if the page does not load within the page load timeout', { defaultCommandTimeout: 50, pageLoadTimeout: 500 }, (done) => {
cy.on('fail', (err) => {
expect(err.message).to.include(`Timed out after waiting \`500ms\` for your remote page to load.`)
done()
})

cy.get('[data-cy="login-idp"]').click() // Takes you to idp.com
})

it('waits for the page to load before running the command', { defaultCommandTimeout: 50 }, () => {
cy.get('body').invoke('text').should('equal', 'timeout')
})

it('will retry and fail the command after the page loads', { defaultCommandTimeout: 50 }, (done) => {
cy.on('fail', (err) => {
expect(err.message).to.include(`Timed out retrying after 50ms: expected 'timeout' to equal 'not timeout'`)
done()
})

cy.get('body').invoke('text').should('equal', 'not timeout')
})
})
})
})
32 changes: 0 additions & 32 deletions packages/driver/cypress/integration/e2e/stability_spec.js

This file was deleted.

4 changes: 3 additions & 1 deletion packages/driver/package.json
Expand Up @@ -5,7 +5,9 @@
"scripts": {
"clean-deps": "rm -rf node_modules",
"cypress:open": "node ../../scripts/cypress open",
"cypress:run": "node ../../scripts/cypress run",
"cypress:run": "node ../../scripts/cypress run --spec \"cypress/integration/*/*\",\"cypress/integration/*/!(multi-domain)/**/*\"",
"cypress:open-experimentalSessionAndOrigin": "node ../../scripts/cypress open --config experimentalSessionAndOrigin=true",
"cypress:run-experimentalSessionAndOrigin": "node ../../scripts/cypress run --config experimentalSessionAndOrigin=true",
"postinstall": "patch-package",
"start": "node -e 'console.log(require(`chalk`).red(`\nError:\n\tRunning \\`yarn start\\` is no longer needed for driver/cypress tests.\n\tWe now automatically spawn the server in the pluginsFile.\n\tChanges to the server will be watched and reloaded automatically.`))'"
},
Expand Down

2 comments on commit e3161a0

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on e3161a0 Apr 21, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the linux x64 version of the Test Runner.

Learn more about this pre-release platform-specific build at https://on.cypress.io/installing-cypress#Install-pre-release-version.

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/9.6.0/linux-x64/feature-multidomain-e3161a0b37c5895b060c9a2883906409395f59b5/cypress.tgz

@cypress-bot
Copy link
Contributor

@cypress-bot cypress-bot bot commented on e3161a0 Apr 21, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Circle has built the win32 x64 version of the Test Runner.

Learn more about this pre-release platform-specific build at https://on.cypress.io/installing-cypress#Install-pre-release-version.

Run this command to install the pre-release locally:

npm install https://cdn.cypress.io/beta/npm/9.6.0/win32-x64/feature-multidomain-e3161a0b37c5895b060c9a2883906409395f59b5/cypress.tgz

Please sign in to comment.