fix: regression where failed events could cause a passing test to dis…

…play as failed (#15037)

packages/reporter/cypress/integration/unit/test_model_spec.ts

@@ -29,7 +29,6 @@ const createCommand = (props: Partial<CommandProps> = {}) => {
     testId: 'r3',
     timeout: 4000,
     wallClockStartedAt: new Date().toString(),
-
   } as CommandProps
 
   return _.defaults(props, defaults)
@@ -262,6 +261,19 @@ describe('Test model', () => {
       test.updateLog(createCommand({ timeout: 6000 }))
       expect(test.lastAttempt.commands[0].timeout).to.equal(6000)
     })
+
+    // https://github.com/cypress-io/cypress/issues/14978
+    it('does not change test state based on log state', () => {
+      const test = createTest()
+
+      test.addLog(createCommand({ state: 'active' }))
+      expect(test.lastAttempt.commands[0].state).to.equal('active')
+      expect(test.state).to.equal('processing')
+
+      test.updateLog(createCommand({ state: 'failed' }))
+      expect(test.lastAttempt.commands[0].state).to.equal('failed')
+      expect(test.state).to.equal('processing')
+    })
   })
 
   context('#removeLog', () => {

packages/reporter/src/attempts/attempt-model.ts

@@ -109,15 +109,11 @@ export default class Attempt {
     }
   }
 
-  @action updateLog (props: LogProps) {
+  updateLog (props: LogProps) {
     const log = this._logs[props.id]
 
     if (log) {
       log.update(props)
-
-      if (log.state === 'failed') {
-        this._state = 'failed'
-      }
     }
   }
 

packages/reporter/src/test/test-model.ts

@@ -168,7 +168,7 @@ export default class Test extends Runnable {
       }
     }
 
-    if (props.err) {
+    if (props.err || props.state) {
       this._withAttempt(this.currentRetry, (attempt: Attempt) => {
         attempt.update(props)
       })

packages/runner/cypress/fixtures/studio/error_hooks_spec.js

@@ -0,0 +1,13 @@
+describe('suite', () => {
+  beforeEach(() => {
+    cy.visit('the://url')
+
+    cy.get('body').then(() => {
+      throw new Error('Failing Test')
+    })
+  })
+
+  it('test', () => {
+    cy.get('body')
+  })
+})

packages/runner/cypress/fixtures/studio/error_test_spec.js

@@ -0,0 +1,11 @@
+describe('suite', () => {
+  beforeEach(() => {
+    cy.visit('the://url')
+  })
+
+  it('test', () => {
+    cy.get('body').then(() => {
+      throw new Error('Failing Test')
+    })
+  })
+})

packages/runner/cypress/integration/studio.ui.spec.js

@@ -42,6 +42,7 @@ describe('studio ui', () => {
       cy.get('.runner').find('.url').should('have.value', '')
 
       cy.get('.runner').find('.url-menu').should('be.visible')
+      cy.get('.runner').find('.url-menu').find('.btn-submit').should('be.disabled')
 
       cy.percySnapshot()
     })
@@ -63,6 +64,25 @@ describe('studio ui', () => {
     })
   })
 
+  // doesn't actually test visiting, just ui state
+  it('allows user to visit inputted url and prompts for interaction after visit', () => {
+    runIsolatedCypress('cypress/fixtures/studio/basic_spec.js', {
+      config: {
+        baseUrl: null,
+      },
+      state: {
+        studioTestId: 'r5',
+      },
+    })
+    .then(() => {
+      cy.get('.runner').find('.url').type('the://url')
+      cy.get('.runner').find('.url-menu').find('.btn-submit').click()
+
+      cy.get('.reporter').contains('the://url').closest('.command-wrapper-text').contains('visit')
+      cy.get('.reporter').contains('Interact with your site to add test commands.')
+    })
+  })
+
   it('displays modal when available commands is clicked', () => {
     runIsolatedCypress('cypress/fixtures/studio/basic_spec.js', {
       state: {
@@ -76,4 +96,59 @@ describe('studio ui', () => {
       cy.get('reach-portal').should('not.exist')
     })
   })
+
+  describe('error state', () => {
+    it('displays error state when extending a failed test', () => {
+      runIsolatedCypress('cypress/fixtures/studio/error_test_spec.js', {
+        state: {
+          studioTestId: 'r3',
+        },
+      })
+      .then(() => {
+        cy.get('.reporter').contains('test').closest('.runnable').should('have.class', 'runnable-failed')
+        cy.get('.reporter').contains('Studio cannot add commands to a failing test.').should('exist')
+
+        cy.get('.runner').find('.iframes-container').should('have.class', 'studio-is-failed')
+
+        cy.percySnapshot()
+      })
+    })
+
+    it('displays error state when a before hook fails', () => {
+      runIsolatedCypress('cypress/fixtures/studio/error_hooks_spec.js', {
+        state: {
+          studioTestId: 'r3',
+        },
+      })
+      .then(() => {
+        cy.get('.reporter').contains('test').closest('.runnable').should('have.class', 'runnable-failed')
+        cy.get('.reporter').contains('Studio cannot add commands to a failing test.').should('exist')
+
+        cy.get('.runner').find('.iframes-container').should('have.class', 'studio-is-failed')
+      })
+    })
+
+    it('displays error state when cy.visit() fails on user inputted url', () => {
+      runIsolatedCypress('cypress/fixtures/studio/basic_spec.js', {
+        config: {
+          baseUrl: null,
+        },
+        state: {
+          studioTestId: 'r5',
+        },
+        visitUrl: 'http://localhost:3500/foo',
+        visitSuccess: false,
+      })
+      .then(() => {
+        cy.get('.runner').find('.url').type('the://url')
+        cy.get('.runner').find('.url-menu').find('.btn-submit').click()
+
+        cy.get('.reporter').contains('test 3').closest('.runnable').should('have.class', 'runnable-failed')
+        cy.get('.reporter').contains('the://url').closest('.command-wrapper-text').contains('visit')
+        cy.get('.reporter').contains('Studio cannot add commands to a failing test.').should('exist')
+
+        cy.get('.runner').find('.iframes-container').should('have.class', 'studio-is-failed')
+      })
+    })
+  })
 })

packages/runner/cypress/support/helpers.js

@@ -106,6 +106,7 @@ function createCypress (defaultOptions = {}) {
       config: { video: false },
       onBeforeRun () {},
       visitUrl: 'http://localhost:3500/fixtures/dom.html',
+      visitSuccess: true,
     })
 
     return cy.visit('/fixtures/isolated-runner.html#/tests/cypress/fixtures/empty_spec.js')
@@ -254,7 +255,7 @@ function createCypress (defaultOptions = {}) {
 
         .withArgs('backend:request', 'resolve:url')
         .yieldsAsync({ response: {
-          isOkStatusCode: true,
+          isOkStatusCode: opts.visitSuccess,
           isHtml: true,
           url: opts.visitUrl,
         } })

packages/runner/src/lib/event-manager.js

@@ -388,19 +388,15 @@ const eventManager = {
     Cypress.on('log:added', (log) => {
       const displayProps = Cypress.runner.getDisplayPropsForLog(log)
 
-      if (studioRecorder.isActive) {
-        displayProps.hookId = studioRecorder.hookId
-      }
+      this._interceptStudio(displayProps)
 
       reporterBus.emit('reporter:log:add', displayProps)
     })
 
     Cypress.on('log:changed', (log) => {
       const displayProps = Cypress.runner.getDisplayPropsForLog(log)
 
-      if (studioRecorder.isActive) {
-        displayProps.hookId = studioRecorder.hookId
-      }
+      this._interceptStudio(displayProps)
 
       reporterBus.emit('reporter:log:state:changed', displayProps)
     })
@@ -556,6 +552,19 @@ const eventManager = {
     }
   },
 
+  _interceptStudio (displayProps) {
+    if (studioRecorder.isActive) {
+      displayProps.hookId = studioRecorder.hookId
+
+      if (displayProps.name === 'visit' && displayProps.state === 'failed') {
+        studioRecorder.testFailed()
+        reporterBus.emit('test:set:state', studioRecorder.testError, _.noop)
+      }
+    }
+
+    return displayProps
+  },
+
   emit (event, ...args) {
     localBus.emit(event, ...args)
   },

packages/runner/src/studio/studio-recorder.js

@@ -68,6 +68,13 @@ export class StudioRecorder {
     return this.isActive && !this.url && !this.isFailed
   }
 
+  @computed get testError () {
+    return {
+      id: this.testId,
+      state: 'failed',
+    }
+  }
+
   @computed get saveError () {
     return {
       id: this.testId,