diff --git a/.circleci/workflows.yml b/.circleci/workflows.yml index ddccc7091b80..b5aa3d8b79e0 100644 --- a/.circleci/workflows.yml +++ b/.circleci/workflows.yml @@ -28,7 +28,7 @@ mainBuildFilters: &mainBuildFilters only: - develop - /^release\/\d+\.\d+\.\d+$/ - - 'feature/create-from-react-component' + - 'retry-flake' # usually we don't build Mac app - it takes a long time # but sometimes we want to really confirm we are doing the right thing @@ -37,7 +37,7 @@ macWorkflowFilters: &darwin-workflow-filters when: or: - equal: [ develop, << pipeline.git.branch >> ] - - equal: [ 'feature/create-from-react-component', << pipeline.git.branch >> ] + - equal: [ 'retry-flake', << pipeline.git.branch >> ] - matches: pattern: /^release\/\d+\.\d+\.\d+$/ value: << pipeline.git.branch >> @@ -45,7 +45,7 @@ linuxArm64WorkflowFilters: &linux-arm64-workflow-filters when: or: - equal: [ develop, << pipeline.git.branch >> ] - - equal: [ 'feature/create-from-react-component', << pipeline.git.branch >> ] + - equal: [ 'retry-flake', << pipeline.git.branch >> ] - matches: pattern: /^release\/\d+\.\d+\.\d+$/ value: << pipeline.git.branch >> @@ -63,7 +63,7 @@ windowsWorkflowFilters: &windows-workflow-filters when: or: - equal: [ develop, << pipeline.git.branch >> ] - - equal: [ 'feature/create-from-react-component', << pipeline.git.branch >> ] + - equal: [ 'retry-flake', << pipeline.git.branch >> ] - matches: pattern: /^release\/\d+\.\d+\.\d+$/ value: << pipeline.git.branch >> @@ -129,7 +129,7 @@ commands: - run: name: Check current branch to persist artifacts command: | - if [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "feature/create-from-react-component" ]]; then + if [[ "$CIRCLE_BRANCH" != "develop" && "$CIRCLE_BRANCH" != "release/"* && "$CIRCLE_BRANCH" != "retry-flake" ]]; then echo "Not uploading artifacts or posting install comment for this branch." circleci-agent step halt fi diff --git a/npm/webpack-dev-server/cypress/e2e/angular.cy.ts b/npm/webpack-dev-server/cypress/e2e/angular.cy.ts index d7b499e58fec..c0454de4dcf0 100644 --- a/npm/webpack-dev-server/cypress/e2e/angular.cy.ts +++ b/npm/webpack-dev-server/cypress/e2e/angular.cy.ts @@ -77,7 +77,7 @@ for (const project of WEBPACK_REACT) { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23455 - it.skip('should detect new spec', () => { + it('should detect new spec', { retries: 15 }, () => { cy.visitApp() cy.withCtx(async (ctx) => { diff --git a/npm/webpack-dev-server/cypress/e2e/next.cy.ts b/npm/webpack-dev-server/cypress/e2e/next.cy.ts index 89f1ed55e894..712e4c111490 100644 --- a/npm/webpack-dev-server/cypress/e2e/next.cy.ts +++ b/npm/webpack-dev-server/cypress/e2e/next.cy.ts @@ -76,7 +76,7 @@ for (const project of WEBPACK_REACT) { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23417 - it.skip('should detect new spec', () => { + it('should detect new spec', { retries: 15 }, () => { cy.visitApp() cy.withCtx(async (ctx) => { @@ -94,7 +94,7 @@ for (const project of WEBPACK_REACT) { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23417 - it.skip('should allow import of global styles in support file', () => { + it('should allow import of global styles in support file', { retries: 15 }, () => { cy.visitApp() cy.contains('styles.cy.js').click() cy.waitForSpecToFinish({ passCount: 1 }) diff --git a/npm/webpack-dev-server/cypress/e2e/nuxt.cy.ts b/npm/webpack-dev-server/cypress/e2e/nuxt.cy.ts index c7797af925e2..0621d60c4339 100644 --- a/npm/webpack-dev-server/cypress/e2e/nuxt.cy.ts +++ b/npm/webpack-dev-server/cypress/e2e/nuxt.cy.ts @@ -71,7 +71,7 @@ for (const project of PROJECTS) { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23455 - it.skip('should detect new spec', () => { + it('should detect new spec', { retries: 15 }, () => { cy.visitApp() cy.withCtx(async (ctx) => { diff --git a/packages/app/cypress/e2e/cypress-in-cypress-component.cy.ts b/packages/app/cypress/e2e/cypress-in-cypress-component.cy.ts index 46f3f72cff39..1971468fd7e9 100644 --- a/packages/app/cypress/e2e/cypress-in-cypress-component.cy.ts +++ b/packages/app/cypress/e2e/cypress-in-cypress-component.cy.ts @@ -82,7 +82,7 @@ describe('Cypress In Cypress CT', { viewportWidth: 1500, defaultCommandTimeout: }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23159 - it.skip('redirects to the specs list with error if a spec is not found', () => { + it('redirects to the specs list with error if a spec is not found', { retries: 15 }, () => { cy.visitApp() const { title, intro, explainer } = defaultMessages.specPage.noSpecError const badFilePath = 'src/DoesNotExist.spec.js' diff --git a/packages/app/cypress/e2e/cypress-in-cypress-run-mode.cy.ts b/packages/app/cypress/e2e/cypress-in-cypress-run-mode.cy.ts index b9f3a4130a00..75561b1090a0 100644 --- a/packages/app/cypress/e2e/cypress-in-cypress-run-mode.cy.ts +++ b/packages/app/cypress/e2e/cypress-in-cypress-run-mode.cy.ts @@ -34,7 +34,7 @@ describe('Cypress In Cypress - run mode', { viewportWidth: 1200 }, () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23306 - it.skip('component testing run mode spec runner header is correct', () => { + it('component testing run mode spec runner header is correct', { retries: 15 }, () => { cy.scaffoldProject('cypress-in-cypress') cy.findBrowsers() cy.openProject('cypress-in-cypress') diff --git a/packages/app/cypress/e2e/cypress-in-cypress.cy.ts b/packages/app/cypress/e2e/cypress-in-cypress.cy.ts index 2bba6f4c8c92..d49d3bf8aaba 100644 --- a/packages/app/cypress/e2e/cypress-in-cypress.cy.ts +++ b/packages/app/cypress/e2e/cypress-in-cypress.cy.ts @@ -93,7 +93,7 @@ describe('Cypress in Cypress', { viewportWidth: 1500, defaultCommandTimeout: 100 }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23307 - it.skip(`scales the AUT correctly in ${testingType}`, () => { + it(`scales the AUT correctly in ${testingType}`, { retries: 15 }, () => { const assertNoScaleShown = () => { // check that no message about scale % is shown, // meaning the AUT is at 100% scale diff --git a/packages/app/cypress/e2e/specs_list_latest_runs.cy.ts b/packages/app/cypress/e2e/specs_list_latest_runs.cy.ts index bff378126cfc..c47496a8a7c2 100644 --- a/packages/app/cypress/e2e/specs_list_latest_runs.cy.ts +++ b/packages/app/cypress/e2e/specs_list_latest_runs.cy.ts @@ -671,7 +671,7 @@ describe('App/Cloud Integration - Latest runs and Average duration', { viewportW }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23419 - it.skip('shows offline alert then hides it after coming online', () => { + it('shows offline alert then hides it after coming online', { retries: 15 }, () => { cy.findByTestId('offline-alert') .should('contain.text', defaultMessages.specPage.offlineWarning.title) .and('contain.text', defaultMessages.specPage.offlineWarning.explainer) diff --git a/packages/data-context/test/unit/sources/GitDataSource.spec.ts b/packages/data-context/test/unit/sources/GitDataSource.spec.ts index f6fde302918c..3b4660adabdf 100644 --- a/packages/data-context/test/unit/sources/GitDataSource.spec.ts +++ b/packages/data-context/test/unit/sources/GitDataSource.spec.ts @@ -43,8 +43,10 @@ describe('GitDataSource', () => { sinon.restore() }) - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23317 - it.skip(`gets correct status for files on ${os.platform()}`, async () => { + it(`gets correct status for files on ${os.platform()}`, async function () { + // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23317 + this.retries(15) + const onBranchChange = sinon.stub() const onGitInfoChange = sinon.stub() const onError = sinon.stub() diff --git a/packages/driver/cypress/e2e/commands/actions/type_special_chars.cy.js b/packages/driver/cypress/e2e/commands/actions/type_special_chars.cy.js index b93cea9658ea..3070b464bcc6 100644 --- a/packages/driver/cypress/e2e/commands/actions/type_special_chars.cy.js +++ b/packages/driver/cypress/e2e/commands/actions/type_special_chars.cy.js @@ -85,7 +85,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => { context('{{}', () => { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160 - it.skip('sets which and keyCode to 219', (done) => { + it('sets which and keyCode to 219', { retries: 15 }, (done) => { cy.$$(':text:first').on('keydown', (e) => { expect(e.which).to.eq(219) expect(e.keyCode).to.eq(219) @@ -238,7 +238,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160 - it.skip('sets which and keyCode to 8 and does not fire keypress events', (done) => { + it('sets which and keyCode to 8 and does not fire keypress events', { retries: 15 }, (done) => { cy.$$(':text:first').on('keypress', () => { done('should not have received keypress') }) @@ -255,7 +255,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160 - it.skip('does not fire textInput event', (done) => { + it('does not fire textInput event', { retries: 15 }, (done) => { cy.$$(':text:first').on('textInput', (e) => { done(new Error('textInput should not have fired')) }) @@ -266,7 +266,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160 - it.skip('can prevent default backspace movement', (done) => { + it('can prevent default backspace movement', { retries: 15 }, (done) => { cy.$$(':text:first').on('keydown', (e) => { if (e.keyCode === 8) { e.preventDefault() @@ -581,7 +581,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160 - it.skip('can prevent default del movement', (done) => { + it('can prevent default del movement', { retries: 15 }, (done) => { cy.$$(':text:first').on('keydown', (e) => { if (e.keyCode === 46) { e.preventDefault() @@ -1240,7 +1240,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => { // https://github.com/cypress-io/cypress/issues/3405 // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160 - it.skip('does fire input event when text inserted', (done) => { + it('does fire input event when text inserted', { retries: 15 }, (done) => { cy.$$('[contenteditable]:first').on('input', (e) => { done() }) @@ -1281,7 +1281,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => { context('1 input, no \'submit\' elements', () => { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160 - it.skip('triggers form submit', function (done) { + it('triggers form submit', { retries: 15 }, function (done) { this.foo = {} this.$forms.find('#single-input').submit((e) => { @@ -1481,7 +1481,7 @@ describe('src/cy/commands/actions/type - #type special chars', () => { context('2 inputs, 1 \'submit\' element input[type=submit]', () => { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23160 - it.skip('triggers form submit', function (done) { + it('triggers form submit', { retries: 15 }, function (done) { this.$forms.find('#multiple-inputs-and-input-submit').submit((e) => { e.preventDefault() diff --git a/packages/driver/cypress/e2e/commands/navigation.cy.js b/packages/driver/cypress/e2e/commands/navigation.cy.js index 1a9e2127ab1e..373acf1a9d64 100644 --- a/packages/driver/cypress/e2e/commands/navigation.cy.js +++ b/packages/driver/cypress/e2e/commands/navigation.cy.js @@ -273,7 +273,6 @@ describe('src/cy/commands/navigation', () => { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23308 context.skip('#go', () => { - // TODO: fix this it('sets timeout to Cypress.config(pageLoadTimeout)', { pageLoadTimeout: 4567, }, () => { @@ -816,7 +815,7 @@ describe('src/cy/commands/navigation', () => { // https://github.com/cypress-io/cypress/issues/1311 // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23201 - it.skip('window immediately resolves and doesn\'t reload when visiting the same URL with hashes', () => { + it('window immediately resolves and doesn\'t reload when visiting the same URL with hashes', { retries: 15 }, () => { const onLoad = cy.stub() cy @@ -1803,7 +1802,7 @@ describe('src/cy/commands/navigation', () => { }) }) - // TODO(webkit): fix+unskip for experimental webkit release + // TODO(webkit): fix+unskip for webkit release context('#page load', { browser: '!webkit' }, () => { it('sets initial=true and then removes', () => { Cookie.remove('__cypress.initial') @@ -2200,7 +2199,7 @@ describe('src/cy/commands/navigation', () => { }) }) - // TODO(webkit): fix+unskip for experimental webkit release + // TODO(webkit): fix+unskip for webkit release context('#url:changed', { browser: '!webkit' }, () => { beforeEach(function () { this.logs = [] diff --git a/packages/driver/cypress/e2e/commands/net_stubbing.cy.ts b/packages/driver/cypress/e2e/commands/net_stubbing.cy.ts index b1a97a0707a7..c812b42c9225 100644 --- a/packages/driver/cypress/e2e/commands/net_stubbing.cy.ts +++ b/packages/driver/cypress/e2e/commands/net_stubbing.cy.ts @@ -1639,7 +1639,7 @@ describe('network stubbing', { retries: 15 }, function () { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23422 - it.skip('can add a body to a request that does not have one', function () { + it('can add a body to a request that does not have one', { retries: 15 }, function () { const body = '{"foo":"bar"}' cy.intercept('/post-only', function (req) { @@ -1670,7 +1670,7 @@ describe('network stubbing', { retries: 15 }, function () { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23303 - it.skip('can delay and throttle a StaticResponse', function (done) { + it('can delay and throttle a StaticResponse', { retries: 15 }, function (done) { const payload = 'A'.repeat(10 * 1024) const throttleKbps = 10 const delay = 250 @@ -1696,7 +1696,7 @@ describe('network stubbing', { retries: 15 }, function () { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23404 - it.skip('can delay with deprecated delayMs param', function () { + it('can delay with deprecated delayMs param', { retries: 15 }, function () { const delayMs = 250 cy.intercept('/timeout*', (req) => { @@ -1714,7 +1714,7 @@ describe('network stubbing', { retries: 15 }, function () { // @see https://github.com/cypress-io/cypress/issues/14446 // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23406 - it.skip('should delay the same amount on every response', () => { + it('should delay the same amount on every response', { retries: 15 }, () => { const delay = 250 const testDelay = () => { @@ -1829,7 +1829,7 @@ describe('network stubbing', { retries: 15 }, function () { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23407 - it.skip('by doing both', () => { + it('by doing both', { retries: 15 }, () => { cy.intercept({ url: '/users*' }, (req) => { req.query = { a: 'b', @@ -1852,7 +1852,7 @@ describe('network stubbing', { retries: 15 }, function () { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23414 - it.skip('by deleting query member', () => { + it('by deleting query member', { retries: 15 }, () => { cy.intercept({ url: '/users*' }, (req) => { req.query = { a: 'b', @@ -1876,7 +1876,7 @@ describe('network stubbing', { retries: 15 }, function () { context('by setting new url', () => { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23415 - it.skip('absolute path', () => { + it('absolute path', { retries: 15 }, () => { cy.intercept({ url: '/users*' }, (req) => { req.url = 'http://localhost:3500/users?a=b' @@ -1894,7 +1894,7 @@ describe('network stubbing', { retries: 15 }, function () { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23433 - it.skip('relative path', () => { + it('relative path', { retries: 15 }, () => { cy.intercept({ url: '/users*' }, (req) => { req.url = '/users?a=b' @@ -1913,7 +1913,7 @@ describe('network stubbing', { retries: 15 }, function () { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23434 - it.skip('empty string', () => { + it('empty string', { retries: 15 }, () => { cy.intercept({ url: '/users*' }, (req) => { req.url = '' @@ -1933,7 +1933,7 @@ describe('network stubbing', { retries: 15 }, function () { context('throwing errors correctly', () => { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23423 - it.skip('defineproperty', (done) => { + it('defineproperty', { retries: 15 }, (done) => { cy.on('fail', (err) => { expect(err.message).to.eq('`defineProperty()` is not allowed.') @@ -1962,7 +1962,7 @@ describe('network stubbing', { retries: 15 }, function () { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23147 - it.skip('setPrototypeOf', (done) => { + it('setPrototypeOf', { retries: 15 }, (done) => { cy.on('fail', (err) => { expect(err.message).to.eq('`setPrototypeOf()` is not allowed.') @@ -1992,7 +1992,7 @@ describe('network stubbing', { retries: 15 }, function () { context('can end response', () => { for (const eventName of ['before:response', 'response']) { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23434 - it.skip(`in \`${eventName}\``, () => { + it(`in \`${eventName}\``, { retries: 15 }, () => { const url = uniqueRoute('/foo') const expectBeforeResponse = eventName === 'response' let beforeResponseCalled = false @@ -2203,7 +2203,7 @@ describe('network stubbing', { retries: 15 }, function () { context('with `times`', function () { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23434 - it.skip('only uses each handler N times', function () { + it('only uses each handler N times', { retries: 15 }, function () { const url = uniqueRoute('/foo') const third = sinon.stub() diff --git a/packages/driver/cypress/e2e/cypress/proxy-logging.cy.ts b/packages/driver/cypress/e2e/cypress/proxy-logging.cy.ts index aa3a617d33fd..04c18c4a8e1a 100644 --- a/packages/driver/cypress/e2e/cypress/proxy-logging.cy.ts +++ b/packages/driver/cypress/e2e/cypress/proxy-logging.cy.ts @@ -46,8 +46,10 @@ describe('Proxy Logging', () => { } context('request logging', () => { - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23443 - it.skip('fetch log shows resource type, url, method, and status code and has expected snapshots and consoleProps', (done) => { + it('fetch log shows resource type, url, method, and status code and has expected snapshots and consoleProps', { + // TODO(webkit): fix+unskip for webkit release + browser: '!webkit', + }, (done) => { fetch('/some-url') // trigger: Cypress.Log() called @@ -96,113 +98,11 @@ describe('Proxy Logging', () => { done() } catch (err) { - done(new Error(err)) - } - }) - }) - }) - - // @see https://github.com/cypress-io/cypress/issues/18757 and https://github.com/cypress-io/cypress/issues/17656 - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23250 - it.skip('xhr log has response body/status code when xhr response is logged first', { - // TODO: unskip in Electron: https://cypress-io.atlassian.net/browse/UNIFY-1753 - browser: '!electron', - }, (done) => { - cy.visit('/fixtures/empty.html') - - cy.window() - .then({ timeout: 10000 }, (win) => { - cy.on('log:changed', (log) => { - try { - expect(log.snapshots.map((v) => v.name)).to.deep.eq(['request', 'response']) - expect(log.consoleProps['Response Headers']).to.include({ - 'x-powered-by': 'Express', - }) - - expect(log.consoleProps['Response Body']).to.include('Cannot GET /some-url') - expect(log.consoleProps['Response Status Code']).to.eq(404) - - expect(log.renderProps).to.include({ - indicator: 'bad', - message: 'GET 404 /some-url', - }) - - expect(Object.keys(log.consoleProps)).to.deep.eq( - ['Event', 'Resource Type', 'Method', 'URL', 'Request went to origin?', 'XHR', 'groups', 'Request Headers', 'Response Status Code', 'Response Headers', 'Response Body'], - ) - - done() - } catch (err) { - done(new Error(err)) - } - }) - - const oldUpdateRequestWithResponse = Cypress.ProxyLogging.updateRequestWithResponse - - cy.stub(Cypress.ProxyLogging, 'updateRequestWithResponse').log(false).callsFake(function (...args) { - setTimeout(() => { - oldUpdateRequestWithResponse.call(this, ...args) - }, 500) - }) - - const xhr = new win.XMLHttpRequest() - - xhr.open('GET', '/some-url') - xhr.send() - }) - }) - - // @see https://github.com/cypress-io/cypress/issues/18757 and https://github.com/cypress-io/cypress/issues/17656 - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23203 - it.skip('xhr log has response body/status code when xhr response is logged second', (done) => { - cy.visit('/fixtures/empty.html') - - cy.window() - .then({ timeout: 10000 }, (win) => { - cy.on('log:changed', (log) => { - try { - expect(log.snapshots.map((v) => v.name)).to.deep.eq(['request', 'response']) - expect(log.consoleProps['Response Headers']).to.include({ - 'x-powered-by': 'Express', - }) - - expect(log.consoleProps['Response Body']).to.include('Cannot GET /some-url') - expect(log.consoleProps['Response Status Code']).to.eq(404) - - expect(log.renderProps).to.include({ - indicator: 'bad', - message: 'GET 404 /some-url', - }) - - expect(Object.keys(log.consoleProps)).to.deep.eq( - ['Event', 'Resource Type', 'Method', 'URL', 'Request went to origin?', 'XHR', 'groups', 'Request Headers', 'Response Status Code', 'Response Headers', 'Response Body'], - ) - - done() - } catch (err) { - done(new Error(err)) + // don't throw, eventually the log update will come in + // eslint-disable-next-line no-console + console.error('assertion error', err) } }) - - const xhr = new win.XMLHttpRequest() - - const logIncomingRequest = Cypress.ProxyLogging.logIncomingRequest - const updateRequestWithResponse = Cypress.ProxyLogging.updateRequestWithResponse - - // To simulate the xhr call landing second, we send updateRequestWithResponse immediately after - // the call is intercepted - cy.stub(Cypress.ProxyLogging, 'logIncomingRequest').log(false).callsFake(function (...args) { - logIncomingRequest.call(this, ...args) - updateRequestWithResponse.call(this, { - requestId: args[0].requestId, - status: 404, - }) - }) - - cy.stub(Cypress.ProxyLogging, 'updateRequestWithResponse').log(false).callsFake(function () {}) - - xhr.open('GET', '/some-url') - xhr.send() }) }) @@ -264,7 +164,7 @@ describe('Proxy Logging', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23420 - it.skip('intercept log has consoleProps with intercept info', (done) => { + it('intercept log has consoleProps with intercept info', (done) => { cy.intercept('/some-url', 'stubbed response').as('alias') .then(() => { fetch('/some-url') @@ -298,8 +198,10 @@ describe('Proxy Logging', () => { url: 'http://localhost:3500/some-url', body: '', httpVersion: '1.1', + query: {}, responseTimeout: Cypress.config('responseTimeout'), headers: interceptProps.Request.headers, + resourceType: 'fetch', }, Response: { body: 'stubbed response', @@ -316,7 +218,9 @@ describe('Proxy Logging', () => { done() } catch (error) { - done(new Error(error)) + // don't throw, eventually the log update will come in + // eslint-disable-next-line no-console + console.error('assertion error', error) } }) }) diff --git a/packages/driver/cypress/e2e/e2e/e2e_cookies.cy.js b/packages/driver/cypress/e2e/e2e/e2e_cookies.cy.js index 671691caab16..cd7512b0de65 100644 --- a/packages/driver/cypress/e2e/e2e/e2e_cookies.cy.js +++ b/packages/driver/cypress/e2e/e2e/e2e_cookies.cy.js @@ -20,7 +20,7 @@ describe('e2e cookies spec', () => { context('__Host- prefix', () => { // https://github.com/cypress-io/cypress/issues/8261 // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444 - it.skip('can set __Host- cookie', () => { + it('can set __Host- cookie', { retries: 15 }, () => { cy.visit('https://example.com') cy.setCookie('__Host-foobar', 'someval', { domain: 'example.com', @@ -37,7 +37,7 @@ describe('e2e cookies spec', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444 - it.skip('errors when __Host- cookie and secure:false', (done) => { + it('errors when __Host- cookie and secure:false', { retries: 15 }, (done) => { cy.visit('https://example.com') cy.setCookie('__Host-foobar', 'someval') @@ -51,7 +51,7 @@ describe('e2e cookies spec', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444 - it.skip('errors when __Host- cookie and path', (done) => { + it('errors when __Host- cookie and path', { retries: 15 }, (done) => { cy.visit('https://example.com') cy.setCookie('__Host-foobar', 'someval', { secure: true, @@ -67,7 +67,7 @@ describe('e2e cookies spec', () => { context('__Secure- prefix', () => { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444 - it.skip('can set __Secure- cookie', () => { + it('can set __Secure- cookie', { retries: 15 }, () => { cy.visit('https://example.com') cy.setCookie('__Secure-foobar', 'someval', { domain: 'example.com', @@ -84,7 +84,7 @@ describe('e2e cookies spec', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23444 - it.skip('errors when __Secure- cookie secure:false', (done) => { + it('errors when __Secure- cookie secure:false', { retries: 15 }, (done) => { cy.visit('https://example.com') cy.setCookie('__Secure-foobar', 'someval', { domain: 'example.com', diff --git a/packages/driver/cypress/e2e/e2e/focus_blur.cy.js b/packages/driver/cypress/e2e/e2e/focus_blur.cy.js index e80a5c1de06c..8a4f552d44bb 100644 --- a/packages/driver/cypress/e2e/e2e/focus_blur.cy.js +++ b/packages/driver/cypress/e2e/e2e/focus_blur.cy.js @@ -509,7 +509,7 @@ describe('polyfill programmatic blur events', () => { }) }) -// TODO(webkit): fix+unskip for experimental webkit release +// TODO(webkit): fix+unskip for webkit release describe('intercept blur methods correctly', { browser: '!webkit' }, () => { beforeEach(() => { cy.visit('http://localhost:3500/fixtures/active-elements.html').then(() => { diff --git a/packages/driver/cypress/e2e/e2e/origin/basic_login.cy.ts b/packages/driver/cypress/e2e/e2e/origin/basic_login.cy.ts index 9b6289a9bf74..2cd6df74e6c4 100644 --- a/packages/driver/cypress/e2e/e2e/origin/basic_login.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/basic_login.cy.ts @@ -167,7 +167,7 @@ describe('Multi-step Auth', { browser: '!webkit' }, () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23481 - it.skip('final-auth redirects back to localhost - flat', () => { + it('final-auth redirects back to localhost - flat', { retries: 15 }, () => { cy.visit('/fixtures/auth/index.html') cy.get('[data-cy="login-with-approval"]').click() // takes you to foobar.com.../approval cy.origin('http://www.foobar.com:3500', () => { // Parent origin is localhost diff --git a/packages/driver/cypress/e2e/e2e/origin/commands/actions.cy.ts b/packages/driver/cypress/e2e/e2e/origin/commands/actions.cy.ts index 69cb3af9d3b8..ad997e4182d9 100644 --- a/packages/driver/cypress/e2e/e2e/origin/commands/actions.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/commands/actions.cy.ts @@ -488,7 +488,7 @@ context('cy.origin actions', { browser: '!webkit' }, () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23480 - it.skip('.submit()', () => { + it('.submit()', { retries: 15 }, () => { cy.get('a[data-cy="dom-link"]').click() cy.origin('http://www.foobar.com:3500', () => { cy.get('form#multiple-inputs-and-input-submit input[name="fname"]').type('foo') diff --git a/packages/driver/cypress/e2e/e2e/origin/commands/assertions.cy.ts b/packages/driver/cypress/e2e/e2e/origin/commands/assertions.cy.ts index 5556787f8e79..d018b40a023b 100644 --- a/packages/driver/cypress/e2e/e2e/origin/commands/assertions.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/commands/assertions.cy.ts @@ -32,7 +32,7 @@ context('cy.origin assertions', { browser: '!webkit' }, () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23148 - it.skip('.should() and .and()', () => { + it('.should() and .and()', { retries: 15 }, () => { cy.origin('http://www.foobar.com:3500', () => { cy.get(':checkbox[name="colors"][value="blue"]') .should('not.be.checked').and('not.be.disabled') diff --git a/packages/driver/cypress/e2e/e2e/origin/navigation.cy.ts b/packages/driver/cypress/e2e/e2e/origin/navigation.cy.ts index 9b4538a0a3de..0890dd42414e 100644 --- a/packages/driver/cypress/e2e/e2e/origin/navigation.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/navigation.cy.ts @@ -396,7 +396,7 @@ describe('errors', { browser: '!webkit' }, () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23481 - it.skip('fails in cy.origin when a command is run after we return to localhost', { defaultCommandTimeout: 50 }, (done) => { + it('fails in cy.origin when a command is run after we return to localhost', { defaultCommandTimeout: 50, retries: 15 }, (done) => { cy.on('fail', (err) => { expect(err.message).to.include(`Timed out retrying after 50ms:`) expect(err.message).to.include(`The command was expected to run against origin \`http://www.idp.com:3500\` but the application is at origin \`http://localhost:3500\`.`) diff --git a/packages/driver/cypress/e2e/e2e/origin/uncaught_errors.cy.ts b/packages/driver/cypress/e2e/e2e/origin/uncaught_errors.cy.ts index fa5007da7b34..097c254430c4 100644 --- a/packages/driver/cypress/e2e/e2e/origin/uncaught_errors.cy.ts +++ b/packages/driver/cypress/e2e/e2e/origin/uncaught_errors.cy.ts @@ -238,7 +238,7 @@ describe('cy.origin - uncaught errors', { browser: '!webkit' }, () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23484 - it.skip('fails the current test/command if a promise is rejected from the cy.origin callback after it is finished running', (done) => { + it('fails the current test/command if a promise is rejected from the cy.origin callback after it is finished running', { retries: 15 }, (done) => { cy.on('fail', (err) => { expect(err.name).to.eq('Error') expect(err.message).to.include('rejected promise') diff --git a/packages/driver/cypress/e2e/e2e/react-15.cy.js b/packages/driver/cypress/e2e/e2e/react-15.cy.js index 16c8561e1468..504e81914c89 100644 --- a/packages/driver/cypress/e2e/e2e/react-15.cy.js +++ b/packages/driver/cypress/e2e/e2e/react-15.cy.js @@ -1,4 +1,4 @@ -// TODO(webkit): fix+unskip for experimental webkit release +// TODO(webkit): fix+unskip for webkit release describe('react v15.6.0', { browser: '!webkit' }, () => { context('fires onChange events', () => { beforeEach(() => { diff --git a/packages/driver/cypress/e2e/e2e/react-16.cy.js b/packages/driver/cypress/e2e/e2e/react-16.cy.js index 7a7c989ad256..9a81d3f8c4d4 100644 --- a/packages/driver/cypress/e2e/e2e/react-16.cy.js +++ b/packages/driver/cypress/e2e/e2e/react-16.cy.js @@ -1,4 +1,4 @@ -// TODO(webkit): fix+unskip for experimental webkit release +// TODO(webkit): fix+unskip for webkit release describe('react v16.0.0', { browser: '!webkit' }, () => { context('fires onChange events', () => { beforeEach(() => { diff --git a/packages/driver/cypress/e2e/e2e/text_mask.cy.js b/packages/driver/cypress/e2e/e2e/text_mask.cy.js index 18a4580a96a6..13a22bfcc322 100644 --- a/packages/driver/cypress/e2e/e2e/text_mask.cy.js +++ b/packages/driver/cypress/e2e/e2e/text_mask.cy.js @@ -1,6 +1,6 @@ let changed = 0 -// TODO(webkit): fix+unskip for experimental webkit release +// TODO(webkit): fix+unskip for webkit release describe('src/cy/commands/actions/type text_mask_spec', { browser: '!webkit' }, () => { beforeEach(() => { cy.visit('/fixtures/text-mask.html') diff --git a/packages/driver/cypress/e2e/issues/1244.cy.js b/packages/driver/cypress/e2e/issues/1244.cy.js index ea208886c68a..a1c2fd78d577 100644 --- a/packages/driver/cypress/e2e/issues/1244.cy.js +++ b/packages/driver/cypress/e2e/issues/1244.cy.js @@ -11,7 +11,7 @@ describe('issue 1244', () => { for (const [el, target, action] of [['button', 'form', 'submit'], ['a', 'a', 'click']]) { //
submit, click - // TODO(webkit): fix+unskip for experimental webkit release, or make skip more specific (only 4 of these generated tests fail in webkit) + // TODO(webkit): fix+unskip for webkit release, or make skip more specific (only 4 of these generated tests fail in webkit) describe(`<${target}> ${action}`, { browser: '!webkit' }, () => { it('correctly redirects when target=_top with target.target =', () => { cy.get(`${el}.setTarget`).click() diff --git a/packages/launchpad/cypress/e2e/choose-a-browser.cy.ts b/packages/launchpad/cypress/e2e/choose-a-browser.cy.ts index a0bd14f61602..377345cc2e57 100644 --- a/packages/launchpad/cypress/e2e/choose-a-browser.cy.ts +++ b/packages/launchpad/cypress/e2e/choose-a-browser.cy.ts @@ -16,7 +16,7 @@ describe.skip('Choose a browser page', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23158 - it.skip('preselects browser that is provided through the command line', () => { + it('preselects browser that is provided through the command line', { retries: 15 }, () => { cy.withCtx((ctx, o) => { // stub launching project since we have `--browser --testingType --project` here o.sinon.stub(ctx._apis.projectApi, 'launchProject').resolves() @@ -218,7 +218,7 @@ describe.skip('Choose a browser page', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23220 - it.skip('performs mutation to focus open browser when focus button is pressed', () => { + it('performs mutation to focus open browser when focus button is pressed', { retries: 15 }, () => { cy.openProject('launchpad', ['--e2e']) cy.visitLaunchpad() @@ -271,7 +271,7 @@ describe.skip('Choose a browser page', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23158 - it.skip('subscribes to changes to browserStatus/activeBrowser through the browserStatusUpdated subscription', () => { + it('subscribes to changes to browserStatus/activeBrowser through the browserStatusUpdated subscription', { retries: 15 }, () => { cy.openProject('launchpad', ['--e2e']) cy.visitLaunchpad() diff --git a/packages/launchpad/cypress/e2e/config-warning.cy.ts b/packages/launchpad/cypress/e2e/config-warning.cy.ts index 9ab466e5ba75..de894571c62c 100644 --- a/packages/launchpad/cypress/e2e/config-warning.cy.ts +++ b/packages/launchpad/cypress/e2e/config-warning.cy.ts @@ -244,7 +244,7 @@ describe('component testing dependency warnings', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23154 - it.skip('warns against outdated @vue/cli dependency', () => { + it('warns against outdated @vue/cli dependency', { retries: 15 }, () => { cy.scaffoldProject('outdated-deps-vuecli3') cy.addProject('outdated-deps-vuecli3') cy.openGlobalMode() diff --git a/packages/launchpad/cypress/e2e/scaffold-component-testing.cy.ts b/packages/launchpad/cypress/e2e/scaffold-component-testing.cy.ts index 1f3d9291035b..862998098536 100644 --- a/packages/launchpad/cypress/e2e/scaffold-component-testing.cy.ts +++ b/packages/launchpad/cypress/e2e/scaffold-component-testing.cy.ts @@ -112,7 +112,7 @@ describe('scaffolding component testing', { context('angular-cli-unconfigured', () => { // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23452 - it.skip('scaffolds component testing for Angular', () => { + it('scaffolds component testing for Angular', { retries: 15 }, () => { startSetupFor('angular-cli-unconfigured') // should detect correctly diff --git a/packages/launchpad/cypress/e2e/slow-network.cy.ts b/packages/launchpad/cypress/e2e/slow-network.cy.ts index eb9864fec644..849ea912553d 100644 --- a/packages/launchpad/cypress/e2e/slow-network.cy.ts +++ b/packages/launchpad/cypress/e2e/slow-network.cy.ts @@ -40,7 +40,7 @@ describe('slow network: launchpad', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/21897 - it.skip('shows the versions after they resolve', () => { + it('shows the versions after they resolve', { retries: 15 }, () => { cy.visitLaunchpad() cy.skipWelcome() cy.get('[data-cy=top-nav-cypress-version-current-link]').should('not.exist') diff --git a/packages/launchpad/src/setup/OpenBrowserList.cy.tsx b/packages/launchpad/src/setup/OpenBrowserList.cy.tsx index 9b7b20bcf3cd..968811749fda 100644 --- a/packages/launchpad/src/setup/OpenBrowserList.cy.tsx +++ b/packages/launchpad/src/setup/OpenBrowserList.cy.tsx @@ -135,7 +135,7 @@ describe('', () => { }) // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23099 - it.skip('throws when activeBrowser is null', (done) => { + it('throws when activeBrowser is null', { retries: 15 }, (done) => { cy.once('uncaught:exception', (err) => { expect(err.message).to.include('Missing activeBrowser in selectedBrowserId') done() diff --git a/packages/network/test/integration/connect_spec.ts b/packages/network/test/integration/connect_spec.ts index e49877577f61..6b026474dcc5 100644 --- a/packages/network/test/integration/connect_spec.ts +++ b/packages/network/test/integration/connect_spec.ts @@ -11,8 +11,9 @@ chai.use(sinonChai) describe('lib/connect', function () { context('.getAddress', function () { - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23155 - it.skip('resolves localhost on 127.0.0.1 immediately', function () { + it('resolves localhost on 127.0.0.1 immediately', function () { + // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23155 + this.retries(15) this.timeout(50) const server = net.createServer(_.partialRight(_.invoke, 'close')) diff --git a/packages/server/test/integration/cypress_spec.js b/packages/server/test/integration/cypress_spec.js index 1c6803fe6410..589df17cb2f9 100644 --- a/packages/server/test/integration/cypress_spec.js +++ b/packages/server/test/integration/cypress_spec.js @@ -1760,9 +1760,11 @@ describe('lib/cypress', () => { // this should be overridden by the env argument json.baseUrl = 'http://localhost:8080' - const { supportFile, specPattern, excludeSpecPattern, baseUrl, slowTestThreshold, testIsolation, ...rest } = json + // "pick" out the list of properties that cannot exist on the root level so we can re-add them on the "e2e" object + // TODO: refactor this part of the test, this is silly and a holdover from pre-split-config + const { experimentalRunAllSpecs, experimentalOriginDependencies, supportFile, specPattern, excludeSpecPattern, baseUrl, slowTestThreshold, testIsolation, ...rest } = json - return settings.writeForTesting(this.todosPath, { ...rest, e2e: { baseUrl, supportFile, specPattern, testIsolation, excludeSpecPattern } }) + return settings.writeForTesting(this.todosPath, { ...rest, e2e: { experimentalRunAllSpecs, experimentalOriginDependencies, baseUrl, supportFile, specPattern, testIsolation, excludeSpecPattern } }) }).then(async () => { await clearCtx() diff --git a/packages/server/test/performance/proxy_performance_spec.js b/packages/server/test/performance/proxy_performance_spec.js index d032e28aa8c1..1ae459054ca4 100644 --- a/packages/server/test/performance/proxy_performance_spec.js +++ b/packages/server/test/performance/proxy_performance_spec.js @@ -376,7 +376,7 @@ describe('Proxy Performance', function () { URLS_UNDER_TEST.map((urlUnderTest) => { // TODO: fix flaky tests https://github.com/cypress-io/cypress/issues/23214 - describe.skip(urlUnderTest, function () { + describe(urlUnderTest, { retries: 15 }, function () { let baseline const testCases = _.cloneDeep(TEST_CASES) diff --git a/packages/server/test/unit/fixture_spec.js b/packages/server/test/unit/fixture_spec.js index 514545e194c3..e1867ac9e62e 100644 --- a/packages/server/test/unit/fixture_spec.js +++ b/packages/server/test/unit/fixture_spec.js @@ -153,8 +153,10 @@ Expecting 'EOF', '}', ':', ',', ']', got 'STRING'\ }) }) - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23457 - it.skip('does not reformat empty objects', function () { + it('does not reformat empty objects', function () { + // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23457 + this.retries(15) + const fn = () => { return fixture.get(this.fixturesFolder, 'empty_objects') } diff --git a/packages/server/test/unit/open_project_spec.js b/packages/server/test/unit/open_project_spec.js index f8a327eaa16b..9e1e8669a3f9 100644 --- a/packages/server/test/unit/open_project_spec.js +++ b/packages/server/test/unit/open_project_spec.js @@ -179,8 +179,10 @@ describe('lib/open_project', () => { }) }) - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23448 - it.skip('sends after:spec errors through onError option', function () { + it('sends after:spec errors through onError option', function () { + // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23448 + this.retries(15) + const err = new Error('thrown from after:spec handler') this.config.experimentalInteractiveRunEvents = true diff --git a/system-tests/lib/system-tests.ts b/system-tests/lib/system-tests.ts index 0a7ced4f428e..fa5ae7162b34 100644 --- a/system-tests/lib/system-tests.ts +++ b/system-tests/lib/system-tests.ts @@ -55,6 +55,7 @@ export type ItOptions = ExecOptions & { * If a function is supplied, it will be executed instead of running the `systemTests.exec` function immediately. */ onRun?: ( + this: Mocha.Context, execFn: ExecFn, browser: BrowserName ) => Promise | any @@ -528,7 +529,8 @@ const localItFn = function (title: string, opts: ItOptions) { return systemTests.exec(ctx, _.extend({ originalTitle }, options, overrides, { browser })) } - return options.onRun(execFn, browser, ctx) + // pass Mocha's this context to onRun + return options.onRun.call(this, execFn, browser, ctx) }) } diff --git a/system-tests/test/plugins_spec.js b/system-tests/test/plugins_spec.js index 4dc5859a08cc..170ec8bcdcbe 100644 --- a/system-tests/test/plugins_spec.js +++ b/system-tests/test/plugins_spec.js @@ -26,8 +26,10 @@ describe('e2e plugins', function () { }, }) - // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23493 - it.skip('fails when there is an async error inside an event handler', function () { + it('fails when there is an async error inside an event handler', function () { + // TODO: fix flaky test https://github.com/cypress-io/cypress/issues/23493 + this.retries(15) + return systemTests.exec(this, { spec: 'app.cy.js', project: 'plugins-async-error', diff --git a/system-tests/test/video_compression_spec.js b/system-tests/test/video_compression_spec.js index 7421bf378c44..4f2d531a30ea 100644 --- a/system-tests/test/video_compression_spec.js +++ b/system-tests/test/video_compression_spec.js @@ -49,7 +49,12 @@ describe('e2e video compression', () => { MS_PER_TEST, }, }, - async onRun (exec) { + async onRun (exec, browserName) { + if (browserName === 'webkit') { + // TODO(webkit): fix video recording flake in webkit + this.retries(15) + } + process.env.VIDEO_COMPRESSION_THROTTLE = 10 const { stdout } = await exec()