feat: test config overrides (#5346)

* decaffeinate: Rename browser.coffee from .coffee to .js

* decaffeinate: Convert browser.coffee to JS

* decaffeinate: Run post-processing cleanups on browser.coffee

* temp 02/14/20 [skip ci]

* add beforeEachRestoreRunner to select specs, remove support file hook

* temp 03/03/20 [skip ci]

* fix more specs due to support file change

* fix errored spec

* add isInteractive config to many specs

* update yarn.lock

* use yarn.lock from develop

* fix nested suite configurations

* fix nested suite configurations 2

* force isInteractive in driver/support file

* update more specs to use test config format

* update more specs to use test config format 2

* update more specs to use test config format 3

* update more specs to use test config format 4

* update more specs to use test config format 5

* fix cli/types tests, unit tests

* cleanup

* allow .isBrowser to support array, use .isBrowser for per-test-config

* allow null browser test/suite config value

* temp 04/10/20 [skip ci]

* restore test originalTitle when skip due to browser

* add tests for per-test-config baseUrl

* update .isBrowser error message

* fix rerunning hooks on top navigation

* rename duplicate issue number

* up timeout for server/integration test

* change test to be more specific

* rename TestOptions, add baseUrl test, override xit xdescribe methods

* add tests for xit/xdescribe

* disable video for flaky e2e test

* fix lint-types, e2e snapshot

* try 2: fix rerun before/after hooks

* temp 04/29/20 [skip ci]

* change logic to rerun before hooks after top navigation

* fix windowSize for browser e2e test

* fix windowSize for xvfb chrome in e2e test

* ok fine, just disable screenshots

* perf: faster lookup for hooks without runnables

* fix afterAll hook switch logic

* backport to before/after fix

* backport to before/after fix 2

* fix noExit passed to e2e test inline options

* remove extra root afterhook check

* add issue link..twice

* cleanup function to arrows

* remove Cypress object proxying related code for certain utils

* use getTest() as we did previously

* remove Cypress object proxying related code for certain utils

* fix Cypress._RESUMED_AT_TEST access

* fix Cypress._RESUMED_AT_TEST access 2

* fix runner.getResumedAtTestIndex, state accesses

* fix firefoxgcinterval access

* fix arrow function

* fix firefoxgcinterval access

* try a simpler way to fix afterAll hook issue

* fix decaf after merge

* cleanup internal-types.d.ts

* fix internal-types

* fix comment, getTestFromRunnable signature

* remove unneeded lastTestInSuiteLogic

* fix after merge: many decaffed specs, typedefs

* minor cleanup

* fix typedefs: add taskTimeout

* minor typedef fix, fix more specs modifying config()

* fix e2e snapshot

* fix flake: waiting_spec cancel outstanding XHR between tests

* fix flake

* change config mutation logic, add tests, update typedefs

* add env support to testConfigOverride

* fix specs: remove isInteractive override, add to beforeEach

* move testConfigOverride to file

* finish moving local config logic to cy.js

* fix typedefs cypress.d.ts

* fix minor minor dtslint

* minor spec cleanup

* chunk: typescript spec fixes, bind Cypress.$

* apply chunk: typescript spec fixes, bind Cypress.$

* fix stop-only

* fix stop-only 2

* extend the jqueryProxyFn in the constructor

* remove experimental code

* rename spec files for overrides, cleanup

* rename snapshots + tests to match updated test config overrides

* fix firefox flake: navigation_spec

Co-authored-by: Brian Mann <brian.mann86@gmail.com>

cli/types/cypress.d.ts

@@ -75,6 +75,8 @@ declare namespace Cypress {
     clear: (keys?: string[]) => void
   }
 
+  type IsBrowserMatcher = BrowserName | Partial<Browser> | Array<BrowserName | Partial<Browser>>
+
   interface ViewportPosition extends WindowPosition {
     right: number
     bottom: number
@@ -332,12 +334,15 @@ declare namespace Cypress {
     isCy(obj: any): obj is Chainable
 
     /**
-     * Returns true if currently running the supplied browser name or matcher object.
+     * Returns true if currently running the supplied browser name or matcher object. Also accepts an array of matchers.
      * @example isBrowser('chrome') will be true for the browser 'chrome:canary' and 'chrome:stable'
      * @example isBrowser({ name: 'firefox', channel: 'dev' }) will be true only for the browser 'firefox:dev' (Firefox Developer Edition)
+     * @example isBrowser(['firefox', 'edge']) will be true only for the browsers 'firefox' and 'edge'
+     * @example isBrowser('!firefox') will be true for every browser other than 'firefox'
+     * @example isBrowser({ family: '!chromium'}) will be true for every browser not matching { family: 'chromium' }
      * @param matcher browser name or matcher object to check.
      */
-    isBrowser(name: BrowserName | Partial<Browser>): boolean
+    isBrowser(name: IsBrowserMatcher): boolean
 
     /**
      * Internal options for "cy.log" used in custom commands.
@@ -2422,6 +2427,11 @@ declare namespace Cypress {
     experimentalSourceRewriting: boolean
   }
 
+  interface TestConfigOverrides extends Partial<Pick<ConfigOptions, 'baseUrl' | 'defaultCommandTimeout' | 'taskTimeout' | 'animationDistanceThreshold' | 'waitForAnimations' | 'viewportHeight' | 'viewportWidth' | 'requestTimeout' | 'execTimeout' | 'env' | 'responseTimeout'>> {
+    // retries?: number
+    browser?: IsBrowserMatcher | IsBrowserMatcher[]
+  }
+
   /**
    * All configuration items are optional.
    */
@@ -4815,3 +4825,65 @@ declare namespace Cypress {
    */
   interface cy extends Chainable<undefined> {}
 }
+
+declare namespace Mocha {
+  interface TestFunction {
+        /**
+         * Describe a specification or test-case with the given `title`, TestCptions, and callback `fn` acting
+         * as a thunk.
+         */
+        (title: string, config: Cypress.TestConfigOverrides, fn?: Func): Test
+
+        /**
+         * Describe a specification or test-case with the given `title`, TestCptions, and callback `fn` acting
+         * as a thunk.
+         */
+        (title: string, config: Cypress.TestConfigOverrides, fn?: AsyncFunc): Test
+  }
+  interface ExclusiveTestFunction {
+        /**
+         * Describe a specification or test-case with the given `title`, TestCptions, and callback `fn` acting
+         * as a thunk.
+         */
+        (title: string, config: Cypress.TestConfigOverrides, fn?: Func): Test
+
+        /**
+         * Describe a specification or test-case with the given `title`, TestCptions, and callback `fn` acting
+         * as a thunk.
+         */
+        (title: string, config: Cypress.TestConfigOverrides, fn?: AsyncFunc): Test
+  }
+  interface PendingTestFunction {
+        /**
+         * Describe a specification or test-case with the given `title`, TestCptions, and callback `fn` acting
+         * as a thunk.
+         */
+        (title: string, config: Cypress.TestConfigOverrides, fn?: Func): Test
+
+        /**
+         * Describe a specification or test-case with the given `title`, TestCptions, and callback `fn` acting
+         * as a thunk.
+         */
+        (title: string, config: Cypress.TestConfigOverrides, fn?: AsyncFunc): Test
+  }
+
+  interface SuiteFunction {
+    /**
+     * Describe a "suite" with the given `title`, TestCptions, and callback `fn` containing
+     * nested suites.
+     */
+    (title: string, config: Cypress.TestConfigOverrides, fn: (this: Suite) => void): Suite
+  }
+
+  interface ExclusiveSuiteFunction {
+    /**
+     * Describe a "suite" with the given `title`, TestCptions, and callback `fn` containing
+     * nested suites. Indicates this suite should be executed exclusively.
+     */
+    (title: string, config: Cypress.TestConfigOverrides, fn: (this: Suite) => void): Suite
+  }
+
+  interface PendingSuiteFunction {
+    (title: string,  config: Cypress.TestConfigOverrides, fn: (this: Suite) => void): Suite | void
+  }
+}

cli/types/tests/cypress-tests.ts

@@ -466,3 +466,47 @@ namespace CypressDomTests {
   Cypress.dom.getElementCoordinatesByPosition(doc, 'top') // $ExpectError
   Cypress.dom.getElementCoordinatesByPositionRelativeToXY(doc, 1, 2) // $ExpectError
 }
+
+namespace CypressTestConfigOverridesTests {
+  // set config on a per-test basis
+  it('test', {
+    browser: {name: 'firefox'}
+  }, () => {})
+  it('test', {
+    browser: [{name: 'firefox'}, {name: 'chrome'}]
+  }, () => {})
+  it('test', {
+    baseUrl: 'www.foobar.com',
+    browser: 'firefox'
+  }, () => {})
+  it('test', {
+    browser: {foo: 'bar'} // $ExpectError
+  }, () => {})
+
+  it.skip('test', {}, () => {})
+  it.only('test', {}, () => {})
+  xit('test', {}, () => {})
+
+  specify('test', {}, () => {})
+  specify.only('test', {}, () => {})
+  specify.skip('test', {}, () => {})
+  xspecify('test', {}, () => {})
+
+  // set config on a per-suite basis
+  describe('suite', {
+    browser: {family: 'firefox'},
+    baseUrl: 'www.example.com'
+  }, () => {})
+
+  context('suite', {}, () => {})
+
+  describe('suite', {
+    browser: {family: 'firefox'},
+    baseUrl: 'www.example.com'
+    foo: 'foo' // $ExpectError
+  }, () => {})
+
+  describe.only('suite', {}, () => {})
+  describe.skip('suite', {}, () => {})
+  xdescribe('suite', {}, () => {})
+}

package.json

@@ -48,7 +48,7 @@
     "set-next-ci-version": "node ./scripts/binary.js setNextVersion",
     "prestart": "yarn ensure-deps",
     "start": "node $(yarn bin cypress) open --dev --global",
-    "stop-only": "npx stop-only --skip .cy,.publish,.projects,node_modules,dist,dist-test,fixtures,lib,bower_components,src --exclude e2e.ts",
+    "stop-only": "npx stop-only --skip .cy,.publish,.projects,node_modules,dist,dist-test,fixtures,lib,bower_components,src --exclude e2e.ts,cypress-tests.ts",
     "stop-only-all": "yarn stop-only --folder packages",
     "pretest": "yarn ensure-deps",
     "test": "yarn lerna exec yarn test --scope cypress --scope \"'@packages/{electron,extension,https-proxy,launcher,network,proxy,rewriter,reporter,runner,socket}'\"",

packages/driver/src/cy/testConfigOverrides.ts

@@ -0,0 +1,76 @@
+const _ = require('lodash')
+
+function mutateConfiguration (testConfigOverride, config, env) {
+  const globalConfig = _.clone(config())
+  const globalEnv = _.clone(env())
+
+  delete testConfigOverride.browser
+  config(testConfigOverride)
+
+  const localTestConfig = config()
+  const localTestConfigBackup = _.clone(localTestConfig)
+
+  if (testConfigOverride.env) {
+    env(testConfigOverride.env)
+  }
+
+  const localTestEnv = env()
+  const localTestEnvBackup = _.clone(localTestEnv)
+
+  // we restore config back to what it was before the test ran
+  // UNLESS the user mutated config with Cypress.config, in which case
+  // we apply those changes to the global config
+  // TODO: (NEXT_BREAKING) always restore configuration
+  //   do not allow global mutations inside test
+  const restoreConfigFn = function () {
+    _.each(localTestConfig, (val, key) => {
+      if (localTestConfigBackup[key] !== val) {
+        globalConfig[key] = val
+      }
+    })
+
+    _.each(localTestEnv, (val, key) => {
+      if (localTestEnvBackup[key] !== val) {
+        globalEnv[key] = val
+      }
+    })
+
+    config.reset()
+    config(globalConfig)
+    env.reset()
+    env(globalEnv)
+  }
+
+  return restoreConfigFn
+}
+
+function getResolvedTestConfigOverride (test) {
+  let curParent = test.parent
+
+  const cfgs = [test.cfg]
+
+  while (curParent) {
+    if (curParent.cfg) {
+      cfgs.push(curParent.cfg)
+    }
+
+    curParent = curParent.parent
+  }
+
+  return _.reduceRight(cfgs, (acc, cfg) => _.extend(acc, cfg), {})
+}
+
+class TestConfigOverride {
+  private restoreTestConfigFn: Nullable<() => void> = null
+  restoreAndSetTestConfigOverrides (test, config, env) {
+    if (this.restoreTestConfigFn) this.restoreTestConfigFn()
+
+    const resolvedTestConfig = getResolvedTestConfigOverride(test)
+
+    this.restoreTestConfigFn = mutateConfiguration(resolvedTestConfig, config, env)
+  }
+}
+
+export function create () {
+  return new TestConfigOverride()
+}

packages/driver/src/cypress.js

@@ -152,6 +152,8 @@ class $Cypress {
       longStackTraces: config.isInteractive,
     })
 
+    // TODO: env is unintentionally preserved between soft reruns unlike config.
+    // change this in the NEXT_BREAKING
     const { env } = config
 
     config = _.omit(config, 'env', 'remote', 'resolved', 'scaffoldedFiles', 'javascripts', 'state')
@@ -362,7 +364,7 @@ class $Cypress {
 
       case 'runner:test:before:run':
         // get back to a clean slate
-        this.cy.reset()
+        this.cy.reset(...args)
 
         return this.emit('test:before:run', ...args)
 

packages/driver/src/cypress/browser.js

@@ -2,21 +2,63 @@ const _ = require('lodash')
 const $utils = require('./utils')
 const $errUtils = require('./error_utils')
 
-const isBrowser = function (config, obj = '') {
-  if (_.isString(obj)) {
-    const name = obj.toLowerCase()
-    const currentName = config.browser.name.toLowerCase()
+const _isBrowser = (browser, matcher, errPrefix) => {
+  let isMatch
+  let exclusive = false
 
-    return name === currentName
+  const matchWithExclusion = (objValue, srcValue) => {
+    if (srcValue.startsWith('!')) {
+      exclusive = true
+
+      return objValue !== srcValue.slice(1)
+    }
+
+    return objValue === srcValue
   }
 
-  if (_.isObject(obj)) {
-    return _.isMatch(config.browser, obj)
+  if (_.isString(matcher)) {
+    const name = matcher.toLowerCase()
+    const currentName = browser.name.toLowerCase()
+
+    isMatch = matchWithExclusion(currentName, name)
+  } else if (_.isObject(matcher)) {
+    isMatch = _.isMatchWith(browser, matcher, matchWithExclusion)
+  } else {
+    $errUtils.throwErrByPath('browser.invalid_arg', {
+      args: { prefix: errPrefix, obj: $utils.stringify(matcher) },
+    })
   }
 
-  $errUtils.throwErrByPath('browser.invalid_arg', {
-    args: { method: 'isBrowser', obj: $utils.stringify(obj) },
+  return {
+    isMatch,
+    exclusive,
+  }
+}
+
+const isBrowser = (config, obj = '', errPrefix = '`Cypress.isBrowser()`') => {
+  return _
+  .chain(obj)
+  .concat([])
+  .map((matcher) => _isBrowser(config.browser, matcher, errPrefix))
+  .reduce((a, b) => {
+    if (!a) return b
+
+    if (a.exclusive && b.exclusive) {
+      return {
+        isMatch: a.isMatch && b.isMatch,
+        exclusive: true,
+      }
+    }
+
+    return {
+      isMatch: a.isMatch || b.isMatch,
+      exclusive: b.exclusive,
+    }
+  }, null)
+  .thru((result) => {
+    return Boolean(result) && result.isMatch
   })
+  .value()
 }
 
 module.exports = (config) => {

packages/driver/src/cypress/cy.js

@@ -29,6 +29,7 @@ const $selection = require('../dom/selection')
 const $Snapshots = require('../cy/snapshots')
 const $CommandQueue = require('./command_queue')
 const $VideoRecorder = require('../cy/video-recorder')
+const $TestConfigOverrides = require('../cy/testConfigOverrides')
 
 const privateProps = {
   props: { name: 'state', url: true },
@@ -147,6 +148,7 @@ const create = function (specWindow, Cypress, Cookies, state, config, log) {
   const ensures = $Ensures.create(state, expect)
 
   const snapshots = $Snapshots.create($$, state)
+  const testConfigOverrides = $TestConfigOverrides.create()
 
   const isCy = (val) => {
     return (val === cy) || $utils.isInstanceOf(val, $Chainer)
@@ -950,7 +952,7 @@ const create = function (specWindow, Cypress, Cookies, state, config, log) {
       return doneEarly()
     },
 
-    reset () {
+    reset (attrs, test) {
       stopped = false
 
       const s = state()
@@ -969,6 +971,7 @@ const create = function (specWindow, Cypress, Cookies, state, config, log) {
 
       queue.reset()
       timers.reset()
+      testConfigOverrides.restoreAndSetTestConfigOverrides(test, Cypress.config, Cypress.env)
 
       return cy.removeAllListeners()
     },

packages/driver/src/cypress/error_messages.js

@@ -154,7 +154,7 @@ module.exports = {
   },
 
   browser: {
-    invalid_arg: '`Cypress.{{method}}()` must be passed the name of a browser or an object to filter with. You passed: `{{obj}}`',
+    invalid_arg: '{{prefix}} must be passed a string, object, or an array. You passed: `{{obj}}`',
   },
 
   chai: {