Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

test: clean up skipped flaky tests #25671

Merged
merged 13 commits into from Feb 22, 2023
Expand Up @@ -46,12 +46,8 @@ describe('GitDataSource', () => {
sinon.restore()
})

// TODO: Very flaky on CI, even with a bunch of retries it still fails more often than nought
// see: https://github.com/cypress-io/cypress/issues/25824
// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23317
it.skip(`gets correct status for files on ${os.platform()}`, async function () {
// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23317
this.retries(15)

const onBranchChange = sinon.stub()
const onGitInfoChange = sinon.stub()
const onError = sinon.stub()
Expand Down
1 change: 0 additions & 1 deletion packages/driver/cypress/e2e/cypress/proxy-logging.cy.ts
Expand Up @@ -163,7 +163,6 @@ describe('Proxy Logging', () => {
.visit('/fixtures/empty.html')
})

// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23420
it('intercept log has consoleProps with intercept info', (done) => {
cy.intercept('/some-url', 'stubbed response').as('alias')
.then(() => {
Expand Down
4 changes: 2 additions & 2 deletions packages/driver/cypress/e2e/e2e/origin/snapshots.cy.ts
Expand Up @@ -74,7 +74,7 @@ describe('cy.origin - snapshots', { browser: '!webkit' }, () => {
cy.get('a[data-cy="xhr-fetch-requests-onload"]').click()
})

// TODO: the xhr event is showing up twice in the log, which is wrong and causing flake. skipping until: https://github.com/cypress-io/cypress/issues/23840 is addressed.
// TODO: fix failing test: https://github.com/cypress-io/cypress/issues/23840
it.skip('verifies XHR requests made while a secondary origin is active eventually update with snapshots of the secondary origin', () => {
cy.origin('http://www.foobar.com:3500', () => {
// need to set isInteractive in the spec bridge in order to take xhr snapshots in run mode, similar to how isInteractive is set within support/defaults.js
Expand All @@ -100,7 +100,7 @@ describe('cy.origin - snapshots', { browser: '!webkit' }, () => {
})
})

// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23437
// TODO: fix failing test: https://github.com/cypress-io/cypress/issues/23840
it.skip('verifies fetch requests made while a secondary origin is active eventually update with snapshots of the secondary origin', () => {
cy.origin('http://www.foobar.com:3500', () => {
// need to set isInteractive in the spec bridge in order to take xhr snapshots in run mode, similar to how isInteractive is set within support/defaults.js
Expand Down
2 changes: 1 addition & 1 deletion packages/driver/cypress/e2e/e2e/origin/validation.cy.ts
Expand Up @@ -211,7 +211,7 @@ describe('cy.origin', { browser: '!webkit' }, () => {
})
})

// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23451
// TODO: fix failing test https://github.com/cypress-io/cypress/issues/23451
it.skip('uses cy.origin twice', () => {
cy.visit('/fixtures/auth/index.html')
cy.get('[data-cy="login-idp"]').click()
Expand Down
9 changes: 5 additions & 4 deletions packages/launchpad/cypress/e2e/project-setup.cy.ts
Expand Up @@ -556,7 +556,8 @@ describe('Launchpad: Setup Project', () => {
})
})

describe('Command for package managers', () => {
// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23153
describe('Command for package managers', { retries: 15 }, () => {
it('makes the right command for yarn', () => {
scaffoldAndOpenProject('pristine-yarn')

Expand All @@ -578,7 +579,7 @@ describe('Launchpad: Setup Project', () => {
cy.get('[data-testid="select-framework"]').click()
cy.findByText('Create React App').click()
cy.findByText('Next step').click()
cy.findByTestId('terminal-prompt-input').should('have.value', 'pnpm install -D react-scripts react-dom react')
cy.findByDisplayValue('pnpm install -D react-scripts react-dom react')
})

it('makes the right command for npm', () => {
Expand All @@ -590,7 +591,7 @@ describe('Launchpad: Setup Project', () => {
cy.get('[data-testid="select-framework"]').click()
cy.findByText('Create React App').click()
cy.findByText('Next step').click()
cy.findByTestId('terminal-prompt-input').should('have.value', 'npm install -D react-scripts react-dom react')
cy.findByDisplayValue('npm install -D react-scripts react-dom react')
})
})

Expand Down Expand Up @@ -655,7 +656,7 @@ describe('Launchpad: Setup Project', () => {
verifyScaffoldedFiles('e2e')
})

// TODO: fix flaky tests https://github.com/cypress-io/cypress/issues/23418
// TODO: fix failing test https://github.com/cypress-io/cypress/issues/23418
it.skip('takes the user to first step of ct setup when switching from app', () => {
scaffoldAndOpenProject('pristine-with-e2e-testing')
cy.visitLaunchpad()
Expand Down
2 changes: 1 addition & 1 deletion packages/server/test/integration/cypress_spec.js
Expand Up @@ -1791,7 +1791,7 @@ describe('lib/cypress', () => {
})
})

// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23149
// TODO: fix failing test https://github.com/cypress-io/cypress/issues/23149
it.skip('passes filtered options to Project#open and sets cli config', async function () {
const open = sinon.stub(ServerE2E.prototype, 'open').resolves([])

Expand Down
6 changes: 4 additions & 2 deletions system-tests/test/record_spec.js
@@ -1,3 +1,4 @@
/* eslint-disable no-console */
const _ = require('lodash')
const path = require('path')
const Promise = require('bluebird')
Expand Down Expand Up @@ -342,7 +343,7 @@ describe('e2e record', () => {

setupStubbedServer(routes)

// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23152
// TODO: fix failing test https://github.com/cypress-io/cypress/issues/23152
it.skip('passes in parallel with group', function () {
this.retries(3)

Expand All @@ -366,6 +367,7 @@ describe('e2e record', () => {

// stagger the 2nd run
// starting up a bit
// NOTE: this is probably why this test flakes - despite waiting 3s, sometimes the second instance finishes first
Promise
.delay(3000)
.then(() => {
Expand All @@ -392,7 +394,7 @@ describe('e2e record', () => {
context('metadata', () => {
setupStubbedServer(createRoutes())

// TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23151
// TODO: fix failing test https://github.com/cypress-io/cypress/issues/23151
it.skip('sends Studio usage metadata', function () {
return systemTests.exec(this, {
key: 'f858a2bc-b469-4e48-be67-0876339ee7e1',
Expand Down