From d88d24685e5a8fdd30c58a302317bdea1d9e336d Mon Sep 17 00:00:00 2001 From: Zach Bloomquist Date: Tue, 4 Oct 2022 19:21:55 -0400 Subject: [PATCH] test: change some skipped tests to use retries --- .circleci/config.yml | 13 ++++------ .../cypress/e2e/angular.cy.ts | 2 +- npm/webpack-dev-server/cypress/e2e/next.cy.ts | 4 +-- npm/webpack-dev-server/cypress/e2e/nuxt.cy.ts | 2 +- .../e2e/cypress-in-cypress-component.cy.ts | 2 +- .../e2e/cypress-in-cypress-run-mode.cy.ts | 2 +- .../app/cypress/e2e/cypress-in-cypress.cy.ts | 2 +- packages/app/cypress/e2e/specs_list_e2e.cy.ts | 2 +- .../cypress/e2e/specs_list_latest_runs.cy.ts | 2 +- .../cypress/e2e/specs_list_no_git_repo.cy.ts | 2 +- .../test/unit/sources/GitDataSource.spec.ts | 6 +++-- .../commands/actions/type_special_chars.cy.js | 16 ++++++------ .../driver/cypress/e2e/commands/cookies.cy.js | 15 ++++++----- .../cypress/e2e/commands/navigation.cy.js | 4 +-- .../cypress/e2e/commands/net_stubbing.cy.ts | 26 +++++++++---------- .../driver/cypress/e2e/commands/xhr.cy.js | 8 +++--- .../cypress/e2e/cypress/proxy-logging.cy.ts | 4 +-- .../driver/cypress/e2e/e2e/e2e_cookies.cy.js | 10 +++---- .../cypress/e2e/e2e/origin/basic_login.cy.ts | 2 +- .../e2e/e2e/origin/commands/actions.cy.ts | 2 +- .../e2e/e2e/origin/commands/assertions.cy.ts | 2 +- .../cypress/e2e/e2e/origin/logging.cy.ts | 4 +-- .../cypress/e2e/e2e/origin/navigation.cy.ts | 2 +- .../e2e/e2e/origin/uncaught_errors.cy.ts | 2 +- .../cypress/e2e/choose-a-browser.cy.ts | 6 ++--- .../cypress/e2e/config-warning.cy.ts | 2 +- .../launchpad/cypress/e2e/project-setup.cy.ts | 6 ++--- .../e2e/scaffold-component-testing.cy.ts | 2 +- .../launchpad/cypress/e2e/slow-network.cy.ts | 2 +- .../src/setup/OpenBrowserList.cy.tsx | 2 +- .../network/test/integration/connect_spec.ts | 4 +-- packages/server/lib/browsers/webkit.ts | 2 +- .../server/test/integration/cypress_spec.js | 6 +++-- .../performance/proxy_performance_spec.js | 2 +- packages/server/test/unit/fixture_spec.js | 6 +++-- .../server/test/unit/open_project_spec.js | 6 +++-- system-tests/test/plugins_spec.js | 6 +++-- 37 files changed, 98 insertions(+), 90 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7e4f7cca19e2..54357e7b979e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -27,8 +27,7 @@ mainBuildFilters: &mainBuildFilters branches: only: - develop - - fix-ci-deps - - issue-23843_electron_21_upgrade + - retry-flake # usually we don't build Mac app - it takes a long time # but sometimes we want to really confirm we are doing the right thing @@ -37,8 +36,7 @@ macWorkflowFilters: &darwin-workflow-filters when: or: - equal: [ develop, << pipeline.git.branch >> ] - - equal: [ 'correct-dashboard-results', << pipeline.git.branch >> ] - - equal: [ 'issue-23843_electron_21_upgrade', << pipeline.git.branch >> ] + - equal: [ 'retry-flake', << pipeline.git.branch >> ] - matches: pattern: "-release$" value: << pipeline.git.branch >> @@ -47,7 +45,7 @@ linuxArm64WorkflowFilters: &linux-arm64-workflow-filters when: or: - equal: [ develop, << pipeline.git.branch >> ] - - equal: [ 'issue-23843_electron_21_upgrade', << pipeline.git.branch >> ] + - equal: [ 'retry-flake', << pipeline.git.branch >> ] - matches: pattern: "-release$" value: << pipeline.git.branch >> @@ -65,8 +63,7 @@ windowsWorkflowFilters: &windows-workflow-filters when: or: - equal: [ develop, << pipeline.git.branch >> ] - - equal: [ linux-arm64, << pipeline.git.branch >> ] - - equal: [ 'issue-23843_electron_21_upgrade', << pipeline.git.branch >> ] + - equal: [ retry-flake, << pipeline.git.branch >> ] - matches: pattern: "-release$" value: << pipeline.git.branch >> @@ -131,7 +128,7 @@ commands: - run: name: Check current branch to persist artifacts command: | - if [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "issue-23843_electron_21_upgrade" ]]; then + if [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "retry-flake" ]]; then echo "Not uploading artifacts or posting install comment for this branch." circleci-agent step halt fi diff --git a/npm/webpack-dev-server/cypress/e2e/angular.cy.ts b/npm/webpack-dev-server/cypress/e2e/angular.cy.ts index 2875164ecd6c..f9bc78ce39c8 100644 --- a/npm/webpack-dev-server/cypress/e2e/angular.cy.ts +++ b/npm/webpack-dev-server/cypress/e2e/angular.cy.ts @@ -77,7 +77,7 @@ for (const project of WEBPACK_REACT) { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23455 - it.skip('should detect new spec', () => { + it('should detect new spec', { retries: 15 }, () => { cy.visitApp() cy.withCtx(async (ctx) => { diff --git a/npm/webpack-dev-server/cypress/e2e/next.cy.ts b/npm/webpack-dev-server/cypress/e2e/next.cy.ts index c0c7133425be..0c430073db6b 100644 --- a/npm/webpack-dev-server/cypress/e2e/next.cy.ts +++ b/npm/webpack-dev-server/cypress/e2e/next.cy.ts @@ -76,7 +76,7 @@ for (const project of WEBPACK_REACT) { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23417 - it.skip('should detect new spec', () => { + it('should detect new spec', { retries: 15 }, () => { cy.visitApp() cy.withCtx(async (ctx) => { @@ -94,7 +94,7 @@ for (const project of WEBPACK_REACT) { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23417 - it.skip('should allow import of global styles in support file', () => { + it('should allow import of global styles in support file', { retries: 15 }, () => { cy.visitApp() cy.contains('styles.cy.js').click() cy.waitForSpecToFinish({ passCount: 1 }) diff --git a/npm/webpack-dev-server/cypress/e2e/nuxt.cy.ts b/npm/webpack-dev-server/cypress/e2e/nuxt.cy.ts index c7797af925e2..0621d60c4339 100644 --- a/npm/webpack-dev-server/cypress/e2e/nuxt.cy.ts +++ b/npm/webpack-dev-server/cypress/e2e/nuxt.cy.ts @@ -71,7 +71,7 @@ for (const project of PROJECTS) { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23455 - it.skip('should detect new spec', () => { + it('should detect new spec', { retries: 15 }, () => { cy.visitApp() cy.withCtx(async (ctx) => { diff --git a/packages/app/cypress/e2e/cypress-in-cypress-component.cy.ts b/packages/app/cypress/e2e/cypress-in-cypress-component.cy.ts index 6bb825839dea..b4b906d248e0 100644 --- a/packages/app/cypress/e2e/cypress-in-cypress-component.cy.ts +++ b/packages/app/cypress/e2e/cypress-in-cypress-component.cy.ts @@ -79,7 +79,7 @@ describe('Cypress In Cypress CT', { viewportWidth: 1500, defaultCommandTimeout: }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23159 - it.skip('redirects to the specs list with error if a spec is not found', () => { + it('redirects to the specs list with error if a spec is not found', { retries: 15 }, () => { cy.visitApp() const { title, intro, explainer } = defaultMessages.specPage.noSpecError const badFilePath = 'src/DoesNotExist.spec.js' diff --git a/packages/app/cypress/e2e/cypress-in-cypress-run-mode.cy.ts b/packages/app/cypress/e2e/cypress-in-cypress-run-mode.cy.ts index b9f3a4130a00..75561b1090a0 100644 --- a/packages/app/cypress/e2e/cypress-in-cypress-run-mode.cy.ts +++ b/packages/app/cypress/e2e/cypress-in-cypress-run-mode.cy.ts @@ -34,7 +34,7 @@ describe('Cypress In Cypress - run mode', { viewportWidth: 1200 }, () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23306 - it.skip('component testing run mode spec runner header is correct', () => { + it('component testing run mode spec runner header is correct', { retries: 15 }, () => { cy.scaffoldProject('cypress-in-cypress') cy.findBrowsers() cy.openProject('cypress-in-cypress') diff --git a/packages/app/cypress/e2e/cypress-in-cypress.cy.ts b/packages/app/cypress/e2e/cypress-in-cypress.cy.ts index b958cd4785a4..a8f46938c63a 100644 --- a/packages/app/cypress/e2e/cypress-in-cypress.cy.ts +++ b/packages/app/cypress/e2e/cypress-in-cypress.cy.ts @@ -93,7 +93,7 @@ describe('Cypress in Cypress', { viewportWidth: 1500, defaultCommandTimeout: 100 }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23307 - it.skip(`scales the AUT correctly in ${testingType}`, () => { + it(`scales the AUT correctly in ${testingType}`, { retries: 15 }, () => { const assertNoScaleShown = () => { // check that no message about scale % is shown, // meaning the AUT is at 100% scale diff --git a/packages/app/cypress/e2e/specs_list_e2e.cy.ts b/packages/app/cypress/e2e/specs_list_e2e.cy.ts index 2c778a8b6a5b..52a4fac3c34a 100644 --- a/packages/app/cypress/e2e/specs_list_e2e.cy.ts +++ b/packages/app/cypress/e2e/specs_list_e2e.cy.ts @@ -246,7 +246,7 @@ describe('App: Spec List (E2E)', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23305 - it.skip('saves the filter when navigating to a spec and back', function () { + it('saves the filter when navigating to a spec and back', { retries: 15 }, function () { const targetSpecFile = 'accounts_list.spec.js' clearSearchAndType(targetSpecFile) diff --git a/packages/app/cypress/e2e/specs_list_latest_runs.cy.ts b/packages/app/cypress/e2e/specs_list_latest_runs.cy.ts index e8cef2166d81..4ea622f1842a 100644 --- a/packages/app/cypress/e2e/specs_list_latest_runs.cy.ts +++ b/packages/app/cypress/e2e/specs_list_latest_runs.cy.ts @@ -675,7 +675,7 @@ describe('App/Cloud Integration - Latest runs and Average duration', { viewportW }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23419 - it.skip('shows offline alert then hides it after coming online', () => { + it('shows offline alert then hides it after coming online', { retries: 15 }, () => { cy.findByTestId('offline-alert') .should('contain.text', defaultMessages.specPage.offlineWarning.title) .and('contain.text', defaultMessages.specPage.offlineWarning.explainer) diff --git a/packages/app/cypress/e2e/specs_list_no_git_repo.cy.ts b/packages/app/cypress/e2e/specs_list_no_git_repo.cy.ts index 24311a6ebcc4..fe5d93f26604 100644 --- a/packages/app/cypress/e2e/specs_list_no_git_repo.cy.ts +++ b/packages/app/cypress/e2e/specs_list_no_git_repo.cy.ts @@ -7,7 +7,7 @@ describe('Spec List - Last updated with no git info', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23474 - it.skip('shows no icon and file system timestamp for files', () => { + it('shows no icon and file system timestamp for files', { retries: 15 }, () => { cy.get('[data-cy-row="blank-contents.spec.js"] [data-cy="git-info-row"] svg') .should('not.exist') diff --git a/packages/data-context/test/unit/sources/GitDataSource.spec.ts b/packages/data-context/test/unit/sources/GitDataSource.spec.ts index f6fde302918c..3b4660adabdf 100644 --- a/packages/data-context/test/unit/sources/GitDataSource.spec.ts +++ b/packages/data-context/test/unit/sources/GitDataSource.spec.ts @@ -43,8 +43,10 @@ describe('GitDataSource', () => { sinon.restore() }) - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23317 - it.skip(`gets correct status for files on ${os.platform()}`, async () => { + it(`gets correct status for files on ${os.platform()}`, async function () { + // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23317 + this.retries(15) + const onBranchChange = sinon.stub() const onGitInfoChange = sinon.stub() const onError = sinon.stub() diff --git a/packages/driver/cypress/e2e/commands/actions/type_special_chars.cy.js b/packages/driver/cypress/e2e/commands/actions/type_special_chars.cy.js index 405857a55713..20dfb7167d8e 100644 --- a/packages/driver/cypress/e2e/commands/actions/type_special_chars.cy.js +++ b/packages/driver/cypress/e2e/commands/actions/type_special_chars.cy.js @@ -85,7 +85,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => { context('{{}', () => { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160 - it.skip('sets which and keyCode to 219', (done) => { + it('sets which and keyCode to 219', { retries: 15 }, (done) => { cy.$$(':text:first').on('keydown', (e) => { expect(e.which).to.eq(219) expect(e.keyCode).to.eq(219) @@ -238,7 +238,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160 - it.skip('sets which and keyCode to 8 and does not fire keypress events', (done) => { + it('sets which and keyCode to 8 and does not fire keypress events', { retries: 15 }, (done) => { cy.$$(':text:first').on('keypress', () => { done('should not have received keypress') }) @@ -255,7 +255,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160 - it.skip('does not fire textInput event', (done) => { + it('does not fire textInput event', { retries: 15 }, (done) => { cy.$$(':text:first').on('textInput', (e) => { done(new Error('textInput should not have fired')) }) @@ -266,7 +266,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160 - it.skip('can prevent default backspace movement', (done) => { + it('can prevent default backspace movement', { retries: 15 }, (done) => { cy.$$(':text:first').on('keydown', (e) => { if (e.keyCode === 8) { e.preventDefault() @@ -581,7 +581,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160 - it.skip('can prevent default del movement', (done) => { + it('can prevent default del movement', { retries: 15 }, (done) => { cy.$$(':text:first').on('keydown', (e) => { if (e.keyCode === 46) { e.preventDefault() @@ -1240,7 +1240,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => { // https://github.com/cypress-io/cypress/issues/3405 // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160 - it.skip('does fire input event when text inserted', (done) => { + it('does fire input event when text inserted', { retries: 15 }, (done) => { cy.$$('[contenteditable]:first').on('input', (e) => { done() }) @@ -1281,7 +1281,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => { context('1 input, no \'submit\' elements', () => { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160 - it.skip('triggers form submit', function (done) { + it('triggers form submit', { retries: 15 }, function (done) { this.foo = {} this.$forms.find('#single-input').submit((e) => { @@ -1470,7 +1470,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => { context('2 inputs, 1 \'submit\' element input[type=submit]', () => { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160 - it.skip('triggers form submit', function (done) { + it('triggers form submit', { retries: 15 }, function (done) { this.$forms.find('#multiple-inputs-and-input-submit').submit((e) => { e.preventDefault() diff --git a/packages/driver/cypress/e2e/commands/cookies.cy.js b/packages/driver/cypress/e2e/commands/cookies.cy.js index b739a8a7af8b..72d9ddfa7888 100644 --- a/packages/driver/cypress/e2e/commands/cookies.cy.js +++ b/packages/driver/cypress/e2e/commands/cookies.cy.js @@ -436,7 +436,7 @@ describe('src/cy/commands/cookies', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444 - context.skip('#setCookie', () => { + context('#setCookie', { retries: 15 }, () => { beforeEach(() => { cy.stub(Cypress.utils, 'addTwentyYears').returns(12345) }) @@ -510,8 +510,9 @@ describe('src/cy/commands/cookies', () => { }) describe('timeout', () => { - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444 - it.skip('sets timeout to Cypress.config(responseTimeout)', { + it('sets timeout to Cypress.config(responseTimeout)', { + // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444 + retries: 15, responseTimeout: 2500, }, () => { Cypress.automation.resolves(null) @@ -524,7 +525,7 @@ describe('src/cy/commands/cookies', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444 - it.skip('can override timeout', () => { + it('can override timeout', { retries: 15 }, () => { Cypress.automation.resolves(null) const timeout = cy.spy(Promise.prototype, 'timeout') @@ -535,7 +536,7 @@ describe('src/cy/commands/cookies', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444 - it.skip('clears the current timeout and restores after success', () => { + it('clears the current timeout and restores after success', { retries: 15 }, () => { Cypress.automation.resolves(null) cy.timeout(100) @@ -568,7 +569,7 @@ describe('src/cy/commands/cookies', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444 - it.skip('logs once on error', function (done) { + it('logs once on error', { retries: 15 }, function (done) { const error = new Error('some err message') error.name = 'foo' @@ -589,7 +590,7 @@ describe('src/cy/commands/cookies', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444 - it.skip('throws after timing out', function (done) { + it('throws after timing out', { retries: 15 }, function (done) { Cypress.automation.resolves(Promise.delay(1000)) cy.on('fail', (err) => { diff --git a/packages/driver/cypress/e2e/commands/navigation.cy.js b/packages/driver/cypress/e2e/commands/navigation.cy.js index cccfc66cda22..5418b0784b23 100644 --- a/packages/driver/cypress/e2e/commands/navigation.cy.js +++ b/packages/driver/cypress/e2e/commands/navigation.cy.js @@ -272,7 +272,7 @@ describe('src/cy/commands/navigation', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23308 - context.skip('#go', () => { + context('#go', { retries: 15 }, () => { // TODO: fix this it('sets timeout to Cypress.config(pageLoadTimeout)', { pageLoadTimeout: 4567, @@ -846,7 +846,7 @@ describe('src/cy/commands/navigation', () => { // https://github.com/cypress-io/cypress/issues/1311 // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23201 - it.skip('window immediately resolves and doesn\'t reload when visiting the same URL with hashes', () => { + it('window immediately resolves and doesn\'t reload when visiting the same URL with hashes', { retries: 15 }, () => { const onLoad = cy.stub() cy diff --git a/packages/driver/cypress/e2e/commands/net_stubbing.cy.ts b/packages/driver/cypress/e2e/commands/net_stubbing.cy.ts index ccea3125e0e2..f8589093070b 100644 --- a/packages/driver/cypress/e2e/commands/net_stubbing.cy.ts +++ b/packages/driver/cypress/e2e/commands/net_stubbing.cy.ts @@ -1638,7 +1638,7 @@ describe.skip('network stubbing', function () { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23422 - it.skip('can add a body to a request that does not have one', function () { + it('can add a body to a request that does not have one', { retries: 15 }, function () { const body = '{"foo":"bar"}' cy.intercept('/post-only', function (req) { @@ -1669,7 +1669,7 @@ describe.skip('network stubbing', function () { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23303 - it.skip('can delay and throttle a StaticResponse', function (done) { + it('can delay and throttle a StaticResponse', { retries: 15 }, function (done) { const payload = 'A'.repeat(10 * 1024) const throttleKbps = 10 const delay = 250 @@ -1695,7 +1695,7 @@ describe.skip('network stubbing', function () { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23404 - it.skip('can delay with deprecated delayMs param', function () { + it('can delay with deprecated delayMs param', { retries: 15 }, function () { const delayMs = 250 cy.intercept('/timeout*', (req) => { @@ -1713,7 +1713,7 @@ describe.skip('network stubbing', function () { // @see https://github.com/cypress-io/cypress/issues/14446 // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23406 - it.skip('should delay the same amount on every response', () => { + it('should delay the same amount on every response', { retries: 15 }, () => { const delay = 250 const testDelay = () => { @@ -1822,7 +1822,7 @@ describe.skip('network stubbing', function () { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23407 - it.skip('by doing both', () => { + it('by doing both', { retries: 15 }, () => { cy.intercept({ url: '/users*' }, (req) => { req.query = { a: 'b', @@ -1845,7 +1845,7 @@ describe.skip('network stubbing', function () { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23414 - it.skip('by deleting query member', () => { + it('by deleting query member', { retries: 15 }, () => { cy.intercept({ url: '/users*' }, (req) => { req.query = { a: 'b', @@ -1869,7 +1869,7 @@ describe.skip('network stubbing', function () { context('by setting new url', () => { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23415 - it.skip('absolute path', () => { + it('absolute path', { retries: 15 }, () => { cy.intercept({ url: '/users*' }, (req) => { req.url = 'http://localhost:3500/users?a=b' @@ -1887,7 +1887,7 @@ describe.skip('network stubbing', function () { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23433 - it.skip('relative path', () => { + it('relative path', { retries: 15 }, () => { cy.intercept({ url: '/users*' }, (req) => { req.url = '/users?a=b' @@ -1906,7 +1906,7 @@ describe.skip('network stubbing', function () { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23434 - it.skip('empty string', () => { + it('empty string', { retries: 15 }, () => { cy.intercept({ url: '/users*' }, (req) => { req.url = '' @@ -1926,7 +1926,7 @@ describe.skip('network stubbing', function () { context('throwing errors correctly', () => { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23423 - it.skip('defineproperty', (done) => { + it('defineproperty', { retries: 15 }, (done) => { cy.on('fail', (err) => { expect(err.message).to.eq('`defineProperty()` is not allowed.') @@ -1955,7 +1955,7 @@ describe.skip('network stubbing', function () { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23147 - it.skip('setPrototypeOf', (done) => { + it('setPrototypeOf', { retries: 15 }, (done) => { cy.on('fail', (err) => { expect(err.message).to.eq('`setPrototypeOf()` is not allowed.') @@ -1985,7 +1985,7 @@ describe.skip('network stubbing', function () { context('can end response', () => { for (const eventName of ['before:response', 'response']) { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23434 - it.skip(`in \`${eventName}\``, () => { + it(`in \`${eventName}\``, { retries: 15 }, () => { const url = uniqueRoute('/foo') const expectBeforeResponse = eventName === 'response' let beforeResponseCalled = false @@ -2196,7 +2196,7 @@ describe.skip('network stubbing', function () { context('with `times`', function () { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23434 - it.skip('only uses each handler N times', function () { + it('only uses each handler N times', { retries: 15 }, function () { const url = uniqueRoute('/foo') const third = sinon.stub() diff --git a/packages/driver/cypress/e2e/commands/xhr.cy.js b/packages/driver/cypress/e2e/commands/xhr.cy.js index 39efbbd72b08..0ac5ff54c7a8 100644 --- a/packages/driver/cypress/e2e/commands/xhr.cy.js +++ b/packages/driver/cypress/e2e/commands/xhr.cy.js @@ -2386,7 +2386,7 @@ describe('src/cy/commands/xhr', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23245 - describe.skip('{force404: false}', () => { + describe('{force404: false}', { retries: 15 }, () => { beforeEach(() => { cy .server() @@ -2400,7 +2400,7 @@ describe('src/cy/commands/xhr', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23245 - it.skip('logs request + response headers', () => { + it('logs request + response headers', { retries: 15 }, () => { cy.then(function () { cy.wrap(this).its('lastLog').invoke('invoke', 'consoleProps').should((consoleProps) => { expect(consoleProps['Request Headers']).to.be.an('object') @@ -2410,7 +2410,7 @@ describe('src/cy/commands/xhr', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23247 - it.skip('logs Method, Status, URL, and XHR', () => { + it('logs Method, Status, URL, and XHR', { retries: 15 }, () => { cy.then(function () { const { xhr } = cy.state('responses')[0] @@ -2428,7 +2428,7 @@ describe('src/cy/commands/xhr', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23246 - it.skip('logs response', () => { + it('logs response', { retries: 15 }, () => { cy.then(function () { cy.wrap(this).its('lastLog').invoke('invoke', 'consoleProps').should((consoleProps) => { expect(consoleProps['Response Body'].trim()).to.deep.eq(JSON.stringify({ diff --git a/packages/driver/cypress/e2e/cypress/proxy-logging.cy.ts b/packages/driver/cypress/e2e/cypress/proxy-logging.cy.ts index 7347de2433a8..f5df11ed0b47 100644 --- a/packages/driver/cypress/e2e/cypress/proxy-logging.cy.ts +++ b/packages/driver/cypress/e2e/cypress/proxy-logging.cy.ts @@ -47,7 +47,7 @@ describe('Proxy Logging', () => { context('request logging', () => { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23443 - it.skip('fetch log shows resource type, url, method, and status code and has expected snapshots and consoleProps', (done) => { + it('fetch log shows resource type, url, method, and status code and has expected snapshots and consoleProps', { retries: 15 }, (done) => { fetch('/some-url') // trigger: Cypress.Log() called @@ -264,7 +264,7 @@ describe('Proxy Logging', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23420 - it.skip('intercept log has consoleProps with intercept info', (done) => { + it('intercept log has consoleProps with intercept info', { retries: 15 }, (done) => { cy.intercept('/some-url', 'stubbed response').as('alias') .then(() => { fetch('/some-url') diff --git a/packages/driver/cypress/e2e/e2e/e2e_cookies.cy.js b/packages/driver/cypress/e2e/e2e/e2e_cookies.cy.js index 671691caab16..cd7512b0de65 100644 --- a/packages/driver/cypress/e2e/e2e/e2e_cookies.cy.js +++ b/packages/driver/cypress/e2e/e2e/e2e_cookies.cy.js @@ -20,7 +20,7 @@ describe('e2e cookies spec', () => { context('__Host- prefix', () => { // https://github.com/cypress-io/cypress/issues/8261 // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444 - it.skip('can set __Host- cookie', () => { + it('can set __Host- cookie', { retries: 15 }, () => { cy.visit('https://example.com') cy.setCookie('__Host-foobar', 'someval', { domain: 'example.com', @@ -37,7 +37,7 @@ describe('e2e cookies spec', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444 - it.skip('errors when __Host- cookie and secure:false', (done) => { + it('errors when __Host- cookie and secure:false', { retries: 15 }, (done) => { cy.visit('https://example.com') cy.setCookie('__Host-foobar', 'someval') @@ -51,7 +51,7 @@ describe('e2e cookies spec', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444 - it.skip('errors when __Host- cookie and path', (done) => { + it('errors when __Host- cookie and path', { retries: 15 }, (done) => { cy.visit('https://example.com') cy.setCookie('__Host-foobar', 'someval', { secure: true, @@ -67,7 +67,7 @@ describe('e2e cookies spec', () => { context('__Secure- prefix', () => { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444 - it.skip('can set __Secure- cookie', () => { + it('can set __Secure- cookie', { retries: 15 }, () => { cy.visit('https://example.com') cy.setCookie('__Secure-foobar', 'someval', { domain: 'example.com', @@ -84,7 +84,7 @@ describe('e2e cookies spec', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444 - it.skip('errors when __Secure- cookie secure:false', (done) => { + it('errors when __Secure- cookie secure:false', { retries: 15 }, (done) => { cy.visit('https://example.com') cy.setCookie('__Secure-foobar', 'someval', { domain: 'example.com', diff --git a/packages/driver/cypress/e2e/e2e/origin/basic_login.cy.ts b/packages/driver/cypress/e2e/e2e/origin/basic_login.cy.ts index f70894cf2c46..cd8a1d09941e 100644 --- a/packages/driver/cypress/e2e/e2e/origin/basic_login.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/basic_login.cy.ts @@ -169,7 +169,7 @@ describe('Multi-step Auth', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23481 - it.skip('final-auth redirects back to localhost - flat', () => { + it('final-auth redirects back to localhost - flat', { retries: 15 }, () => { cy.visit('/fixtures/auth/index.html') cy.get('[data-cy="login-with-approval"]').click() // takes you to foobar.com.../approval cy.origin('http://www.foobar.com:3500', () => { // Parent origin is localhost diff --git a/packages/driver/cypress/e2e/e2e/origin/commands/actions.cy.ts b/packages/driver/cypress/e2e/e2e/origin/commands/actions.cy.ts index ac5324b31d31..f27c567916de 100644 --- a/packages/driver/cypress/e2e/e2e/origin/commands/actions.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/commands/actions.cy.ts @@ -485,7 +485,7 @@ context('cy.origin actions', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23480 - it.skip('.submit()', () => { + it('.submit()', { retries: 15 }, () => { cy.get('a[data-cy="dom-link"]').click() cy.origin('http://www.foobar.com:3500', () => { cy.get('form#multiple-inputs-and-input-submit input[name="fname"]').type('foo') diff --git a/packages/driver/cypress/e2e/e2e/origin/commands/assertions.cy.ts b/packages/driver/cypress/e2e/e2e/origin/commands/assertions.cy.ts index 6e12063ae998..da08ac17c94d 100644 --- a/packages/driver/cypress/e2e/e2e/origin/commands/assertions.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/commands/assertions.cy.ts @@ -32,7 +32,7 @@ context('cy.origin assertions', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23148 - it.skip('.should() and .and()', () => { + it('.should() and .and()', { retries: 15 }, () => { cy.origin('http://www.foobar.com:3500', () => { cy.get(':checkbox[name="colors"][value="blue"]') .should('not.be.checked').and('not.be.disabled') diff --git a/packages/driver/cypress/e2e/e2e/origin/logging.cy.ts b/packages/driver/cypress/e2e/e2e/origin/logging.cy.ts index 7e0214dd563f..4fb3108dc813 100644 --- a/packages/driver/cypress/e2e/e2e/origin/logging.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/logging.cy.ts @@ -31,7 +31,7 @@ describe('cy.origin logging', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/21300 - it.skip('logs cy.origin as group when failing with validation failure', () => { + it('logs cy.origin as group when failing with validation failure', { retries: 15 }, () => { const logs: any[] = [] cy.on('log:added', (attrs) => { @@ -52,7 +52,7 @@ describe('cy.origin logging', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/21300 - it.skip('logs cy.origin as group when failing with serialization failure', () => { + it('logs cy.origin as group when failing with serialization failure', { retries: 15 }, () => { const logs: any[] = [] cy.on('log:added', (attrs) => { diff --git a/packages/driver/cypress/e2e/e2e/origin/navigation.cy.ts b/packages/driver/cypress/e2e/e2e/origin/navigation.cy.ts index 8d12f03066e3..840378a467bf 100644 --- a/packages/driver/cypress/e2e/e2e/origin/navigation.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/navigation.cy.ts @@ -384,7 +384,7 @@ describe('errors', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23481 - it.skip('fails in cy.origin when a command is run after we return to localhost', { defaultCommandTimeout: 50 }, (done) => { + it('fails in cy.origin when a command is run after we return to localhost', { defaultCommandTimeout: 50, retries: 15 }, (done) => { cy.on('fail', (err) => { expect(err.message).to.include(`Timed out retrying after 50ms:`) expect(err.message).to.include(`The command was expected to run against origin \`http://www.idp.com:3500\` but the application is at origin \`http://localhost:3500\`.`) diff --git a/packages/driver/cypress/e2e/e2e/origin/uncaught_errors.cy.ts b/packages/driver/cypress/e2e/e2e/origin/uncaught_errors.cy.ts index 6efefc99b559..9d7b60ab2ed7 100644 --- a/packages/driver/cypress/e2e/e2e/origin/uncaught_errors.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/uncaught_errors.cy.ts @@ -238,7 +238,7 @@ describe('cy.origin - uncaught errors', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23484 - it.skip('fails the current test/command if a promise is rejected from the cy.origin callback after it is finished running', (done) => { + it('fails the current test/command if a promise is rejected from the cy.origin callback after it is finished running', { retries: 15 }, (done) => { cy.on('fail', (err) => { expect(err.name).to.eq('Error') expect(err.message).to.include('rejected promise') diff --git a/packages/launchpad/cypress/e2e/choose-a-browser.cy.ts b/packages/launchpad/cypress/e2e/choose-a-browser.cy.ts index d2acd567841c..a4c5ed26c445 100644 --- a/packages/launchpad/cypress/e2e/choose-a-browser.cy.ts +++ b/packages/launchpad/cypress/e2e/choose-a-browser.cy.ts @@ -16,7 +16,7 @@ describe.skip('Choose a Browser Page', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23158 - it.skip('preselects browser that is provided through the command line', () => { + it('preselects browser that is provided through the command line', { retries: 15 }, () => { cy.withCtx((ctx, o) => { // stub launching project since we have `--browser --testingType --project` here o.sinon.stub(ctx._apis.projectApi, 'launchProject').resolves() @@ -218,7 +218,7 @@ describe.skip('Choose a Browser Page', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23220 - it.skip('performs mutation to focus open browser when focus button is pressed', () => { + it('performs mutation to focus open browser when focus button is pressed', { retries: 15 }, () => { cy.openProject('launchpad', ['--e2e']) cy.visitLaunchpad() @@ -271,7 +271,7 @@ describe.skip('Choose a Browser Page', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23158 - it.skip('subscribes to changes to browserStatus/activeBrowser through the browserStatusUpdated subscription', () => { + it('subscribes to changes to browserStatus/activeBrowser through the browserStatusUpdated subscription', { retries: 15 }, () => { cy.openProject('launchpad', ['--e2e']) cy.visitLaunchpad() diff --git a/packages/launchpad/cypress/e2e/config-warning.cy.ts b/packages/launchpad/cypress/e2e/config-warning.cy.ts index e1dc9914aabe..1e853d3d0434 100644 --- a/packages/launchpad/cypress/e2e/config-warning.cy.ts +++ b/packages/launchpad/cypress/e2e/config-warning.cy.ts @@ -164,7 +164,7 @@ describe('component testing dependency warnings', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23154 - it.skip('warns against outdated @vue/cli dependency', () => { + it('warns against outdated @vue/cli dependency', { retries: 15 }, () => { cy.scaffoldProject('outdated-deps-vuecli3') cy.addProject('outdated-deps-vuecli3') cy.openGlobalMode() diff --git a/packages/launchpad/cypress/e2e/project-setup.cy.ts b/packages/launchpad/cypress/e2e/project-setup.cy.ts index a2eb349ae681..030be6c8774e 100644 --- a/packages/launchpad/cypress/e2e/project-setup.cy.ts +++ b/packages/launchpad/cypress/e2e/project-setup.cy.ts @@ -585,7 +585,7 @@ describe('Launchpad: Setup Project', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23153 - it.skip('makes the right command for pnpm', () => { + it('makes the right command for pnpm', { retries: 15 }, () => { scaffoldAndOpenProject('pristine-pnpm') cy.visitLaunchpad() @@ -598,7 +598,7 @@ describe('Launchpad: Setup Project', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23153 - it.skip('makes the right command for npm', () => { + it('makes the right command for npm', { retries: 15 }, () => { scaffoldAndOpenProject('pristine-npm') cy.visitLaunchpad() @@ -673,7 +673,7 @@ describe('Launchpad: Setup Project', () => { }) // TODO: fix flaky tests https://github.com/cypress-io/cypress/issues/23418 - it.skip('takes the user to first step of ct setup when switching from app', () => { + it('takes the user to first step of ct setup when switching from app', { retries: 15 }, () => { scaffoldAndOpenProject('pristine-with-e2e-testing') cy.visitLaunchpad() verifyWelcomePage({ e2eIsConfigured: true, ctIsConfigured: false }) diff --git a/packages/launchpad/cypress/e2e/scaffold-component-testing.cy.ts b/packages/launchpad/cypress/e2e/scaffold-component-testing.cy.ts index 6a68d7cfa39d..5004ce2c4020 100644 --- a/packages/launchpad/cypress/e2e/scaffold-component-testing.cy.ts +++ b/packages/launchpad/cypress/e2e/scaffold-component-testing.cy.ts @@ -110,7 +110,7 @@ describe('scaffolding component testing', { context('angular-cli-unconfigured', () => { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23452 - it.skip('scaffolds component testing for Angular', () => { + it('scaffolds component testing for Angular', { retries: 15 }, () => { startSetupFor('angular-cli-unconfigured') // should detect correctly diff --git a/packages/launchpad/cypress/e2e/slow-network.cy.ts b/packages/launchpad/cypress/e2e/slow-network.cy.ts index ed709d17eaf3..65d52f2f3b3a 100644 --- a/packages/launchpad/cypress/e2e/slow-network.cy.ts +++ b/packages/launchpad/cypress/e2e/slow-network.cy.ts @@ -39,7 +39,7 @@ describe('slow network: launchpad', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/21897 - it.skip('shows the versions after they resolve', () => { + it('shows the versions after they resolve', { retries: 15 }, () => { cy.visitLaunchpad() cy.get('[data-cy=top-nav-cypress-version-current-link]').should('not.exist') cy.contains('Log In') diff --git a/packages/launchpad/src/setup/OpenBrowserList.cy.tsx b/packages/launchpad/src/setup/OpenBrowserList.cy.tsx index 9b7b20bcf3cd..968811749fda 100644 --- a/packages/launchpad/src/setup/OpenBrowserList.cy.tsx +++ b/packages/launchpad/src/setup/OpenBrowserList.cy.tsx @@ -135,7 +135,7 @@ describe('', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23099 - it.skip('throws when activeBrowser is null', (done) => { + it('throws when activeBrowser is null', { retries: 15 }, (done) => { cy.once('uncaught:exception', (err) => { expect(err.message).to.include('Missing activeBrowser in selectedBrowserId') done() diff --git a/packages/network/test/integration/connect_spec.ts b/packages/network/test/integration/connect_spec.ts index e49877577f61..a9a2d9f7cd6e 100644 --- a/packages/network/test/integration/connect_spec.ts +++ b/packages/network/test/integration/connect_spec.ts @@ -11,8 +11,8 @@ chai.use(sinonChai) describe('lib/connect', function () { context('.getAddress', function () { - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23155 - it.skip('resolves localhost on 127.0.0.1 immediately', function () { + it('resolves localhost on 127.0.0.1 immediately', function () { + this.retries(15) this.timeout(50) const server = net.createServer(_.partialRight(_.invoke, 'close')) diff --git a/packages/server/lib/browsers/webkit.ts b/packages/server/lib/browsers/webkit.ts index 534cff80347d..bb32c9e95fd1 100644 --- a/packages/server/lib/browsers/webkit.ts +++ b/packages/server/lib/browsers/webkit.ts @@ -121,7 +121,7 @@ export async function open (browser: Browser, url: string, options: BrowserLaunc */ private suppressUnhandledEconnreset () { unhandledExceptions.handle((err: NodeJS.ErrnoException) => { - return err.code === 'ECONNRESET' + return err.code !== 'ECONNRESET' }) // restore normal exception handling behavior diff --git a/packages/server/test/integration/cypress_spec.js b/packages/server/test/integration/cypress_spec.js index aed82d90d8a9..3e66a1ab4a46 100644 --- a/packages/server/test/integration/cypress_spec.js +++ b/packages/server/test/integration/cypress_spec.js @@ -1648,8 +1648,10 @@ describe('lib/cypress', () => { }) }) - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23149 - it.skip('passes filtered options to Project#open and sets cli config', async function () { + it('passes filtered options to Project#open and sets cli config', async function () { + // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23149 + this.retries(15) + const open = sinon.stub(ServerE2E.prototype, 'open').resolves([]) sinon.stub(interactiveMode, 'ready') diff --git a/packages/server/test/performance/proxy_performance_spec.js b/packages/server/test/performance/proxy_performance_spec.js index d032e28aa8c1..1ae459054ca4 100644 --- a/packages/server/test/performance/proxy_performance_spec.js +++ b/packages/server/test/performance/proxy_performance_spec.js @@ -376,7 +376,7 @@ describe('Proxy Performance', function () { URLS_UNDER_TEST.map((urlUnderTest) => { // TODO: fix flaky tests https://github.com/cypress-io/cypress/issues/23214 - describe.skip(urlUnderTest, function () { + describe(urlUnderTest, { retries: 15 }, function () { let baseline const testCases = _.cloneDeep(TEST_CASES) diff --git a/packages/server/test/unit/fixture_spec.js b/packages/server/test/unit/fixture_spec.js index 514545e194c3..e1867ac9e62e 100644 --- a/packages/server/test/unit/fixture_spec.js +++ b/packages/server/test/unit/fixture_spec.js @@ -153,8 +153,10 @@ Expecting 'EOF', '}', ':', ',', ']', got 'STRING'\ }) }) - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23457 - it.skip('does not reformat empty objects', function () { + it('does not reformat empty objects', function () { + // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23457 + this.retries(15) + const fn = () => { return fixture.get(this.fixturesFolder, 'empty_objects') } diff --git a/packages/server/test/unit/open_project_spec.js b/packages/server/test/unit/open_project_spec.js index f8a327eaa16b..9e1e8669a3f9 100644 --- a/packages/server/test/unit/open_project_spec.js +++ b/packages/server/test/unit/open_project_spec.js @@ -179,8 +179,10 @@ describe('lib/open_project', () => { }) }) - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23448 - it.skip('sends after:spec errors through onError option', function () { + it('sends after:spec errors through onError option', function () { + // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23448 + this.retries(15) + const err = new Error('thrown from after:spec handler') this.config.experimentalInteractiveRunEvents = true diff --git a/system-tests/test/plugins_spec.js b/system-tests/test/plugins_spec.js index 4dc5859a08cc..170ec8bcdcbe 100644 --- a/system-tests/test/plugins_spec.js +++ b/system-tests/test/plugins_spec.js @@ -26,8 +26,10 @@ describe('e2e plugins', function () { }, }) - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23493 - it.skip('fails when there is an async error inside an event handler', function () { + it('fails when there is an async error inside an event handler', function () { + // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23493 + this.retries(15) + return systemTests.exec(this, { spec: 'app.cy.js', project: 'plugins-async-error',