feat: test config overrides

cli/types/index.d.ts

@@ -127,6 +127,8 @@ declare namespace Cypress {
     clear: (keys?: string[]) => void
   }
 
+  type IsBrowserMatcher = BrowserName | Partial<Browser> | Array<BrowserName | Partial<Browser>>
+
   interface ViewportPosition extends WindowPosition {
     right: number
     bottom: number
@@ -379,12 +381,15 @@ declare namespace Cypress {
     isCy(obj: any): obj is Chainable
 
     /**
-     * Returns true if currently running the supplied browser name or matcher object.
+     * Returns true if currently running the supplied browser name or matcher object. Also accepts an array of matchers.
      * @example isBrowser('chrome') will be true for the browser 'chrome:canary' and 'chrome:stable'
      * @example isBrowser({ name: 'firefox', channel: 'dev' }) will be true only for the browser 'firefox:dev' (Firefox Developer Edition)
+     * @example isBrowser(['firefox', 'edge']) will be true only for the browsers 'firefox' and 'edge'
+     * @example isBrowser('!firefox') will be true for every browser other than 'firefox'
+     * @example isBrowser({ family: '!chromium'}) will be true for every browser not matching { family: 'chromium' }
      * @param matcher browser name or matcher object to check.
      */
-    isBrowser(name: BrowserName | Partial<Browser>): boolean
+    isBrowser(name: IsBrowserMatcher): boolean
 
     /**
      * Internal options for "cy.log" used in custom commands.
@@ -2458,6 +2463,11 @@ declare namespace Cypress {
     experimentalGetCookiesSameSite: boolean
   }
 
+  interface TestConfigOptions extends Partial<Pick<ConfigOptions, 'baseUrl' | 'defaultCommandTimeout' | 'animationDistanceThreshold' | 'waitForAnimations' | 'viewportHeight' | 'viewportWidth' | 'requestTimeout' | 'execTimeout' | 'env' | 'responseTimeout'>> {
+    // retries?: number
+    browser?: IsBrowserMatcher | IsBrowserMatcher[]
+  }
+
   interface PluginConfigOptions extends ConfigOptions {
     /**
     * Absolute path to the config file (default: <projectRoot>/cypress.json) or false
@@ -4854,3 +4864,67 @@ Cypress._ // => Lodash _
 ```
  */
 declare const Cypress: Cypress.Cypress & EventEmitter
+
+declare namespace Mocha {
+  interface TestFunction {
+        /**
+         * Describe a specification or test-case with the given `title` and callback `fn` acting
+         * as a thunk.
+         *
+         * - _Only available when invoked via the mocha CLI._
+         */
+        (title: string, config: Cypress.TestConfigOptions, fn?: Func): Test
+
+        /**
+         * Describe a specification or test-case with the given `title` and callback `fn` acting
+         * as a thunk.
+         *
+         * - _Only available when invoked via the mocha CLI._
+         */
+        (title: string, config: Cypress.TestConfigOptions, fn?: AsyncFunc): Test
+  }
+  interface ExclusiveTestFunction {
+        /**
+         * Describe a specification or test-case with the given `title` and callback `fn` acting
+         * as a thunk.
+         *
+         * - _Only available when invoked via the mocha CLI._
+         */
+        (title: string, config: Cypress.TestConfigOptions, fn?: Func): Test
+
+        /**
+         * Describe a specification or test-case with the given `title` and callback `fn` acting
+         * as a thunk.
+         *
+         * - _Only available when invoked via the mocha CLI._
+         */
+        (title: string, config: Cypress.TestConfigOptions, fn?: AsyncFunc): Test
+  }
+  interface PendingTestFunction {
+        /**
+         * Describe a specification or test-case with the given `title` and callback `fn` acting
+         * as a thunk.
+         *
+         * - _Only available when invoked via the mocha CLI._
+         */
+        (title: string, config: Cypress.TestConfigOptions, fn?: Func): Test
+
+        /**
+         * Describe a specification or test-case with the given `title` and callback `fn` acting
+         * as a thunk.
+         *
+         * - _Only available when invoked via the mocha CLI._
+         */
+        (title: string, config: Cypress.TestConfigOptions, fn?: AsyncFunc): Test
+  }
+
+  interface SuiteFunction {
+    /**
+     * [bdd, tdd] Describe a "suite" with the given `title` and callback `fn` containing
+     * nested suites.
+     *
+     * - _Only available when invoked via the mocha CLI._
+     */
+    (title: string, config: Cypress.TestConfigOptions, fn: (this: Suite) => void): Suite
+  }
+}

cli/types/tests/cypress-tests.ts

@@ -455,3 +455,32 @@ namespace CypressDomTests {
   Cypress.dom.getElementCoordinatesByPosition(doc, 'top') // $ExpectError
   Cypress.dom.getElementCoordinatesByPositionRelativeToXY(doc, 1, 2) // $ExpectError
 }
+
+namespace CypressTestConfigTests {
+  // set config on a per-test basis
+  it('test', {
+    browser: {name: 'firefox'}
+  }, () => {})
+  it('test', {
+    browser: [{name: 'firefox'}, {name: 'chrome'}]
+  }, () => {})
+  it('test', {
+    baseUrl: 'www.foobar.com',
+    browser: 'firefox'
+  }, () => {})
+  it('test', {
+    browser: {foo: 'bar'} // $ExpectError
+  }, () => {})
+
+  // set config on a per-suite basis
+  describe('suite', {
+    browser: {family: 'firefox'},
+    baseUrl: 'www.example.com'
+  }, () => {})
+
+  describe('suite', {
+    browser: {family: 'firefox'},
+    baseUrl: 'www.example.com'
+    foo: 'foo' // $ExpectError
+  }, () => {})
+}

packages/driver/src/cypress.js

@@ -264,17 +264,21 @@ class $Cypress {
       case 'runner:suite:start':
         // mocha runner started processing a suite
         if (this.config('isTextTerminal')) {
-          return this.emit('mocha', 'suite', ...args)
+          this.emit('mocha', 'suite', ...args)
         }
 
+        this.emit('suite:start', ...args)
+
         break
 
       case 'runner:suite:end':
         // mocha runner finished processing a suite
         if (this.config('isTextTerminal')) {
-          return this.emit('mocha', 'suite end', ...args)
+          this.emit('mocha', 'suite end', ...args)
         }
 
+        this.emit('suite:end', ...args)
+
         break
 
       case 'runner:hook:start':

packages/driver/src/cypress/browser.js

@@ -2,21 +2,60 @@ const _ = require('lodash')
 const $utils = require('./utils')
 const $errUtils = require('./error_utils')
 
-const isBrowser = function (config, obj = '') {
-  if (_.isString(obj)) {
-    const name = obj.toLowerCase()
-    const currentName = config.browser.name.toLowerCase()
+const _isBrowser = function (browser, matcher, errPrefix) {
+  let isMatch
+  let exclusive = false
 
-    return name === currentName
+  const matchWithExclusion = (objValue, srcValue) => {
+    if (srcValue.startsWith('!')) {
+      exclusive = true
+
+      return objValue !== srcValue.slice(1)
+    }
+
+    return objValue === srcValue
+  }
+
+  if (_.isString(matcher)) {
+    const name = matcher.toLowerCase()
+    const currentName = browser.name.toLowerCase()
+
+    isMatch = matchWithExclusion(currentName, name)
+  } else if (_.isObject(matcher)) {
+    isMatch = _.isMatchWith(browser, matcher, matchWithExclusion)
+  } else {
+    $errUtils.throwErrByPath('browser.invalid_arg', {
+      args: { prefix: errPrefix, obj: $utils.stringify(matcher) },
+    })
   }
 
-  if (_.isObject(obj)) {
-    return _.isMatch(config.browser, obj)
+  return {
+    isMatch,
+    exclusive,
   }
+}
+
+const isBrowser = function (config, obj = '', errPrefix = '`Cypress.isBrowser()`') {
+  if (!_.isArray(obj)) obj = [obj]
+
+  const result = obj.map((matcher) => _isBrowser(config.browser, matcher, errPrefix))
+  .reduce((a, b) => {
+    if (a === null) return b
+
+    if (a.exclusive && b.exclusive) {
+      return {
+        isMatch: a.isMatch && b.isMatch,
+        exclusive: true,
+      }
+    }
+
+    return {
+      isMatch: a.isMatch || b.isMatch,
+      exclusive: b.exclusive,
+    }
+  }, null)
 
-  $errUtils.throwErrByPath('browser.invalid_arg', {
-    args: { method: 'isBrowser', obj: $utils.stringify(obj) },
-  })
+  return !!result && result.isMatch
 }
 
 module.exports = (config) => {

packages/driver/src/cypress/error_messages.coffee

@@ -109,7 +109,7 @@ module.exports = {
     }
 
   browser:
-    invalid_arg: "`Cypress.{{method}}()` must be passed the name of a browser or an object to filter with. You passed: `{{obj}}`"
+    invalid_arg: "{{prefix}} must be passed the name of a browser or an object to filter with. You passed: `{{obj}}`"
 
   chai:
     length_invalid_argument: "You must provide a valid number to a `length` assertion. You passed: `{{length}}`"

packages/driver/src/cypress/mocha.js

@@ -18,6 +18,61 @@ const runnableResetTimeout = Runnable.prototype.resetTimeout
 delete window.mocha
 delete window.Mocha
 
+function overloadMochaFnForConfig (fnName, specWindow) {
+  const _fn = specWindow[fnName]
+
+  const fnType = fnName === 'it' ? 'Test' : 'Suite'
+
+  function overrideFn (fn) {
+    specWindow[fnName] = fn()
+    specWindow[fnName]['only'] = fn('only')
+    specWindow[fnName]['skip'] = fn('skip')
+  }
+
+  overrideFn(function (subFn) {
+    return function (...args) {
+      /**
+       * @type {Cypress.Cypress}
+       */
+      const Cypress = specWindow.Cypress
+
+      const origFn = subFn ? _fn[subFn] : _fn
+
+      if (args.length > 2 && _.isObject(args[1])) {
+        const opts = _.defaults({}, args[1], {
+          browser: null,
+        })
+
+        const mochaArgs = [args[0], args[2]]
+
+        const configMatchesBrowser = opts.browser == null || Cypress.isBrowser(opts.browser, `${fnType} config value \`{ browser }\``)
+
+        if (!configMatchesBrowser) {
+          mochaArgs[0] = `[browser skip (${opts.browser})]${mochaArgs[0]}`
+
+          if (subFn === 'only') {
+            mochaArgs[1] = function () {
+              this.skip()
+            }
+
+            return origFn.apply(this, mochaArgs)
+          }
+
+          return _fn['skip'].apply(this, mochaArgs)
+        }
+
+        const ret = origFn.apply(this, mochaArgs)
+
+        ret.cfg = opts
+
+        return ret
+      }
+
+      return origFn.apply(this, args)
+    }
+  })
+}
+
 const ui = (specWindow, _mocha) => {
   // Override mocha.ui so that the pre-require event is emitted
   // with the iframe's `window` reference, rather than the parent's.
@@ -34,13 +89,19 @@ const ui = (specWindow, _mocha) => {
     // such as describe, it, before, beforeEach, etc
     this.suite.emit('pre-require', specWindow, null, this)
 
+    // allow per-test-config/per-suite-config
+    // by accepting 3 arguments to it/describe/context
+    overloadMochaFnForConfig('it', specWindow)
+    overloadMochaFnForConfig('describe', specWindow)
+    overloadMochaFnForConfig('context', specWindow)
+
     return this
   }
 
   return _mocha.ui('bdd')
 }
 
-const set = (specWindow, _mocha) => {
+const setMochaProps = (specWindow, _mocha) => {
   // Mocha is usually defined in the spec when used normally
   // in the browser or node, so we add it as a global
   // for our users too
@@ -67,7 +128,7 @@ const globals = (specWindow, reporter) => {
   })
 
   // set mocha props on the specWindow
-  set(specWindow, _mocha)
+  setMochaProps(specWindow, _mocha)
 
   // return the newly created mocha instance
   return _mocha

packages/driver/src/cypress/runner.js

@@ -17,7 +17,7 @@ const TEST_BEFORE_RUN_EVENT = 'runner:test:before:run'
 const TEST_AFTER_RUN_EVENT = 'runner:test:after:run'
 
 const RUNNABLE_LOGS = 'routes agents commands'.split(' ')
-const RUNNABLE_PROPS = 'id order title root hookName hookId err state failedFromHookId body speed type duration wallClockStartedAt wallClockDuration timings file'.split(' ')
+const RUNNABLE_PROPS = 'id order title root hookName hookId err state failedFromHookId body speed type duration wallClockStartedAt wallClockDuration timings file cfg'.split(' ')
 
 const debug = require('debug')('cypress:driver:runner')
 // ## initial payload

packages/driver/test/cypress/integration/commands/actions/check_spec.coffee

@@ -2,7 +2,10 @@ $ = Cypress.$.bind(Cypress)
 _ = Cypress._
 Promise = Cypress.Promise
 
-describe "src/cy/commands/actions/check", ->
+describe "src/cy/commands/actions/check", {
+  isInteractive: true
+  numTestsKeptInMemory: 1
+}, ->
   before ->
     cy
       .visit("/fixtures/dom.html")
@@ -207,10 +210,10 @@ describe "src/cy/commands/actions/check", ->
         .check("blue")
         .then -> expect(focus).to.eq true
 
-    describe "errors", ->
+    describe "errors", {
+      defaultCommandTimeout: 100
+    }, ->
       beforeEach ->
-        Cypress.config("defaultCommandTimeout", 100)
-
         @logs = []
 
         cy.on "log:added", (attrs, log) =>
@@ -638,10 +641,10 @@ describe "src/cy/commands/actions/check", ->
         $("[name=colors][value=blue]").prop("checked", true).change -> done()
         cy.get("[name=colors]").uncheck("blue")
 
-    describe "errors", ->
+    describe "errors", {
+      defaultCommandTimeout: 100
+    }, ->
       beforeEach ->
-        Cypress.config("defaultCommandTimeout", 100)
-
         @logs = []
 
         cy.on "log:added", (attrs, log) =>