From 4a47081c68d5d5c3a252485ee54cdc09517f1c47 Mon Sep 17 00:00:00 2001 From: Zach Bloomquist Date: Wed, 22 Feb 2023 10:56:33 -0500 Subject: [PATCH] test: clean up skipped flaky tests (#25671) --- .../data-context/test/unit/sources/GitDataSource.spec.ts | 6 +----- packages/driver/cypress/e2e/cypress/proxy-logging.cy.ts | 1 - packages/driver/cypress/e2e/e2e/origin/snapshots.cy.ts | 4 ++-- packages/driver/cypress/e2e/e2e/origin/validation.cy.ts | 2 +- packages/launchpad/cypress/e2e/project-setup.cy.ts | 9 +++++---- packages/server/test/integration/cypress_spec.js | 2 +- system-tests/test/record_spec.js | 6 ++++-- 7 files changed, 14 insertions(+), 16 deletions(-) diff --git a/packages/data-context/test/unit/sources/GitDataSource.spec.ts b/packages/data-context/test/unit/sources/GitDataSource.spec.ts index 0f093c8586b4..4ee9f381f2d5 100644 --- a/packages/data-context/test/unit/sources/GitDataSource.spec.ts +++ b/packages/data-context/test/unit/sources/GitDataSource.spec.ts @@ -46,12 +46,8 @@ describe('GitDataSource', () => { sinon.restore() }) - // TODO: Very flaky on CI, even with a bunch of retries it still fails more often than nought - // see: https://github.com/cypress-io/cypress/issues/25824 + // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23317 it.skip(`gets correct status for files on ${os.platform()}`, async function () { - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23317 - this.retries(15) - const onBranchChange = sinon.stub() const onGitInfoChange = sinon.stub() const onError = sinon.stub() diff --git a/packages/driver/cypress/e2e/cypress/proxy-logging.cy.ts b/packages/driver/cypress/e2e/cypress/proxy-logging.cy.ts index 04c18c4a8e1a..fac6608beec1 100644 --- a/packages/driver/cypress/e2e/cypress/proxy-logging.cy.ts +++ b/packages/driver/cypress/e2e/cypress/proxy-logging.cy.ts @@ -163,7 +163,6 @@ describe('Proxy Logging', () => { .visit('/fixtures/empty.html') }) - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23420 it('intercept log has consoleProps with intercept info', (done) => { cy.intercept('/some-url', 'stubbed response').as('alias') .then(() => { diff --git a/packages/driver/cypress/e2e/e2e/origin/snapshots.cy.ts b/packages/driver/cypress/e2e/e2e/origin/snapshots.cy.ts index 1a7aa7070f32..4614269bd59f 100644 --- a/packages/driver/cypress/e2e/e2e/origin/snapshots.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/snapshots.cy.ts @@ -74,7 +74,7 @@ describe('cy.origin - snapshots', { browser: '!webkit' }, () => { cy.get('a[data-cy="xhr-fetch-requests-onload"]').click() }) - // TODO: the xhr event is showing up twice in the log, which is wrong and causing flake. skipping until: https://github.com/cypress-io/cypress/issues/23840 is addressed. + // TODO: fix failing test: https://github.com/cypress-io/cypress/issues/23840 it.skip('verifies XHR requests made while a secondary origin is active eventually update with snapshots of the secondary origin', () => { cy.origin('http://www.foobar.com:3500', () => { // need to set isInteractive in the spec bridge in order to take xhr snapshots in run mode, similar to how isInteractive is set within support/defaults.js @@ -100,7 +100,7 @@ describe('cy.origin - snapshots', { browser: '!webkit' }, () => { }) }) - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23437 + // TODO: fix failing test: https://github.com/cypress-io/cypress/issues/23840 it.skip('verifies fetch requests made while a secondary origin is active eventually update with snapshots of the secondary origin', () => { cy.origin('http://www.foobar.com:3500', () => { // need to set isInteractive in the spec bridge in order to take xhr snapshots in run mode, similar to how isInteractive is set within support/defaults.js diff --git a/packages/driver/cypress/e2e/e2e/origin/validation.cy.ts b/packages/driver/cypress/e2e/e2e/origin/validation.cy.ts index e9ccb571ef7c..d5f957330cc4 100644 --- a/packages/driver/cypress/e2e/e2e/origin/validation.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/validation.cy.ts @@ -211,7 +211,7 @@ describe('cy.origin', { browser: '!webkit' }, () => { }) }) - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23451 + // TODO: fix failing test https://github.com/cypress-io/cypress/issues/23451 it.skip('uses cy.origin twice', () => { cy.visit('/fixtures/auth/index.html') cy.get('[data-cy="login-idp"]').click() diff --git a/packages/launchpad/cypress/e2e/project-setup.cy.ts b/packages/launchpad/cypress/e2e/project-setup.cy.ts index 2ba23a94e0f2..278268b5a5bf 100644 --- a/packages/launchpad/cypress/e2e/project-setup.cy.ts +++ b/packages/launchpad/cypress/e2e/project-setup.cy.ts @@ -556,7 +556,8 @@ describe('Launchpad: Setup Project', () => { }) }) - describe('Command for package managers', () => { + // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23153 + describe('Command for package managers', { retries: 15 }, () => { it('makes the right command for yarn', () => { scaffoldAndOpenProject('pristine-yarn') @@ -578,7 +579,7 @@ describe('Launchpad: Setup Project', () => { cy.get('[data-testid="select-framework"]').click() cy.findByText('Create React App').click() cy.findByText('Next step').click() - cy.findByTestId('terminal-prompt-input').should('have.value', 'pnpm install -D react-scripts react-dom react') + cy.findByDisplayValue('pnpm install -D react-scripts react-dom react') }) it('makes the right command for npm', () => { @@ -590,7 +591,7 @@ describe('Launchpad: Setup Project', () => { cy.get('[data-testid="select-framework"]').click() cy.findByText('Create React App').click() cy.findByText('Next step').click() - cy.findByTestId('terminal-prompt-input').should('have.value', 'npm install -D react-scripts react-dom react') + cy.findByDisplayValue('npm install -D react-scripts react-dom react') }) }) @@ -655,7 +656,7 @@ describe('Launchpad: Setup Project', () => { verifyScaffoldedFiles('e2e') }) - // TODO: fix flaky tests https://github.com/cypress-io/cypress/issues/23418 + // TODO: fix failing test https://github.com/cypress-io/cypress/issues/23418 it.skip('takes the user to first step of ct setup when switching from app', () => { scaffoldAndOpenProject('pristine-with-e2e-testing') cy.visitLaunchpad() diff --git a/packages/server/test/integration/cypress_spec.js b/packages/server/test/integration/cypress_spec.js index ff060586b67f..360c5b960528 100644 --- a/packages/server/test/integration/cypress_spec.js +++ b/packages/server/test/integration/cypress_spec.js @@ -1791,7 +1791,7 @@ describe('lib/cypress', () => { }) }) - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23149 + // TODO: fix failing test https://github.com/cypress-io/cypress/issues/23149 it.skip('passes filtered options to Project#open and sets cli config', async function () { const open = sinon.stub(ServerE2E.prototype, 'open').resolves([]) diff --git a/system-tests/test/record_spec.js b/system-tests/test/record_spec.js index 197eae604057..5a5399f24d56 100644 --- a/system-tests/test/record_spec.js +++ b/system-tests/test/record_spec.js @@ -1,3 +1,4 @@ +/* eslint-disable no-console */ const _ = require('lodash') const path = require('path') const Promise = require('bluebird') @@ -342,7 +343,7 @@ describe('e2e record', () => { setupStubbedServer(routes) - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23152 + // TODO: fix failing test https://github.com/cypress-io/cypress/issues/23152 it.skip('passes in parallel with group', function () { this.retries(3) @@ -366,6 +367,7 @@ describe('e2e record', () => { // stagger the 2nd run // starting up a bit + // NOTE: this is probably why this test flakes - despite waiting 3s, sometimes the second instance finishes first Promise .delay(3000) .then(() => { @@ -392,7 +394,7 @@ describe('e2e record', () => { context('metadata', () => { setupStubbedServer(createRoutes()) - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23151 + // TODO: fix failing test https://github.com/cypress-io/cypress/issues/23151 it.skip('sends Studio usage metadata', function () { return systemTests.exec(this, { key: 'f858a2bc-b469-4e48-be67-0876339ee7e1',