fix: test config overrides leak for .only execution

package.json

@@ -43,7 +43,7 @@
     "npm-release": "node scripts/npm-release.js",
     "prestart": "yarn ensure-deps",
     "start": "cypress open --dev --global",
-    "stop-only": "npx stop-only --skip .cy,.publish,.projects,node_modules,dist,dist-test,fixtures,lib,bower_components,src,__snapshots__ --exclude e2e.ts,cypress-tests.ts",
+    "stop-only": "npx stop-only --skip .cy,.publish,.projects,node_modules,dist,dist-test,fixtures,lib,bower_components,src,__snapshots__ --exclude e2e.ts,cypress-tests.ts,*only_spec.js",
     "stop-only-all": "yarn stop-only --folder packages",
     "pretest": "yarn ensure-deps",
     "test": "yarn lerna exec yarn test --scope cypress --scope \"'@packages/{config,electron,extension,https-proxy,launcher,net-stubbing,network,proxy,rewriter,runner,runner-shared,socket}'\"",

packages/config/lib/index.js

@@ -59,7 +59,7 @@ module.exports = {
   options,
 
   validate: (cfg, onErr) => {
-    debug('validating configuration')
+    debug('validating configuration', cfg)
 
     return _.each(cfg, (value, key) => {
       const validationFn = validationRules[key]

packages/driver/cypress/integration/e2e/testConfigOverrides-describe-only_spec.js

@@ -0,0 +1,27 @@
+const shouldNotExecute = () => {
+  throw new Error('Test Override validation should have failed & it block should not have executed.')
+}
+
+it('first should not run', () => {
+  shouldNotExecute()
+})
+
+describe('second should not run', () => {
+  it('test', () => {
+    shouldNotExecute()
+  })
+})
+
+describe('correctly applies overrides ', { retries: 1 }, () => {
+  // eslint-disable-next-line mocha/no-exclusive-tests
+  describe.only('when valid configuration', () => {
+    it('for describe.only', { baseUrl: null }, () => {
+      const config = Cypress.config()
+
+      expect(config.testConfigList).to.be.undefined
+      expect(config.unverifiedTestConfig).to.be.undefined
+      expect(config.baseUrl).to.be.null
+      expect(config.retries).to.eq(1)
+    })
+  })
+})

packages/driver/cypress/integration/e2e/testConfigOverrides-it-only_spec.js

@@ -0,0 +1,25 @@
+const shouldNotExecute = () => {
+  throw new Error('Test Override validation should have failed & it block should not have executed.')
+}
+
+it('first should not run', () => {
+  shouldNotExecute()
+})
+
+describe('second should not run', () => {
+  it('test', () => {
+    shouldNotExecute()
+  })
+})
+
+describe('correctly applies overrides when valid configuration', { retries: 1 }, () => {
+  // eslint-disable-next-line mocha/no-exclusive-tests
+  it.only('for it.only', { baseUrl: null }, () => {
+    const config = Cypress.config()
+
+    expect(config.testConfigList).to.be.undefined
+    expect(config.unverifiedTestConfig).to.be.undefined
+    expect(config.baseUrl).to.be.null
+    expect(config.retries).to.eq(1)
+  })
+})

packages/driver/src/cy/testConfigOverrides.ts

@@ -109,27 +109,29 @@ function mutateConfiguration (testConfig: ResolvedTestConfigOverride, config, en
 // in order to resolve the test config upfront before test runs
 // note: must return as an object to meet the dashboard recording API
 export function getResolvedTestConfigOverride (test): ResolvedTestConfigOverride {
-  let curParent = test.parent
-  const testConfigList = [{
-    overrides: test._testConfig,
-    invocationDetails: test.invocationDetails,
-  }]
-
-  while (curParent) {
-    if (curParent._testConfig) {
-      testConfigList.unshift({
-        overrides: curParent._testConfig,
-        invocationDetails: curParent.invocationDetails,
-      })
+  let curr = test
+  let testConfigList: TestConfig[] = []
+
+  while (curr) {
+    if (curr._testConfig) {
+      if (curr._testConfig.testConfigList) {
+        // configuration for mocha function has already been processed
+        testConfigList = testConfigList.concat(curr._testConfig.testConfigList)
+      } else {
+        testConfigList.unshift({
+          overrides: curr._testConfig,
+          invocationDetails: curr.invocationDetails,
+        })
+      }
     }
 
-    curParent = curParent.parent
+    curr = curr.parent
   }
 
   const testConfig = {
-    testConfigList: testConfigList.filter((opt) => opt.overrides !== undefined),
+    testConfigList: testConfigList.filter(({ overrides }) => overrides !== undefined),
     // collect test overrides to send to the dashboard api when @packages/server is ran in record mode
-    unverifiedTestConfig: _.reduce(testConfigList, (acc, opts) => _.extend(acc, opts.overrides), {}),
+    unverifiedTestConfig: _.reduce(testConfigList, (acc, { overrides }) => _.extend(acc, overrides), {}),
   }
 
   return testConfig

packages/driver/src/cypress/mocha.ts

@@ -107,7 +107,7 @@ function overloadMochaFnForConfig (fnName, specWindow) {
   overrideMochaFn(replacementFn)
 }
 
-const ui = (specWindow, _mocha, config) => {
+const ui = (specWindow, _mocha) => {
   // Override mocha.ui so that the pre-require event is emitted
   // with the iframe's `window` reference, rather than the parent's.
   _mocha.ui = function (name) {

packages/runner/__snapshots__/retries.mochaEvents.spec.js

@@ -1007,19 +1007,8 @@ exports['src/cypress/runner retries mochaEvents test retry with [only] #1'] = [
     "test:before:run",
     {
       "_testConfig": {
-        "testConfigList": [
-          {
-            "overrides": {
-              "testConfigList": [],
-              "unverifiedTestConfig": {}
-            },
-            "invocationDetails": "{Object 9}"
-          }
-        ],
-        "unverifiedTestConfig": {
-          "testConfigList": [],
-          "unverifiedTestConfig": {}
-        }
+        "testConfigList": [],
+        "unverifiedTestConfig": {}
       },
       "id": "r3",
       "order": 1,
@@ -1057,19 +1046,8 @@ exports['src/cypress/runner retries mochaEvents test retry with [only] #1'] = [
     "test",
     {
       "_testConfig": {
-        "testConfigList": [
-          {
-            "overrides": {
-              "testConfigList": [],
-              "unverifiedTestConfig": {}
-            },
-            "invocationDetails": "{Object 9}"
-          }
-        ],
-        "unverifiedTestConfig": {
-          "testConfigList": [],
-          "unverifiedTestConfig": {}
-        }
+        "testConfigList": [],
+        "unverifiedTestConfig": {}
       },
       "id": "r3",
       "order": 1,
@@ -1152,19 +1130,8 @@ exports['src/cypress/runner retries mochaEvents test retry with [only] #1'] = [
     "retry",
     {
       "_testConfig": {
-        "testConfigList": [
-          {
-            "overrides": {
-              "testConfigList": [],
-              "unverifiedTestConfig": {}
-            },
-            "invocationDetails": "{Object 9}"
-          }
-        ],
-        "unverifiedTestConfig": {
-          "testConfigList": [],
-          "unverifiedTestConfig": {}
-        }
+        "testConfigList": [],
+        "unverifiedTestConfig": {}
       },
       "id": "r3",
       "order": 1,
@@ -1264,19 +1231,8 @@ exports['src/cypress/runner retries mochaEvents test retry with [only] #1'] = [
     "test:after:run",
     {
       "_testConfig": {
-        "testConfigList": [
-          {
-            "overrides": {
-              "testConfigList": [],
-              "unverifiedTestConfig": {}
-            },
-            "invocationDetails": "{Object 9}"
-          }
-        ],
-        "unverifiedTestConfig": {
-          "testConfigList": [],
-          "unverifiedTestConfig": {}
-        }
+        "testConfigList": [],
+        "unverifiedTestConfig": {}
       },
       "id": "r3",
       "order": 1,
@@ -1353,19 +1309,8 @@ exports['src/cypress/runner retries mochaEvents test retry with [only] #1'] = [
     "test:before:run",
     {
       "_testConfig": {
-        "testConfigList": [
-          {
-            "overrides": {
-              "testConfigList": [],
-              "unverifiedTestConfig": {}
-            },
-            "invocationDetails": "{Object 9}"
-          }
-        ],
-        "unverifiedTestConfig": {
-          "testConfigList": [],
-          "unverifiedTestConfig": {}
-        }
+        "testConfigList": [],
+        "unverifiedTestConfig": {}
       },
       "id": "r3",
       "title": "test 2",
@@ -1472,19 +1417,8 @@ exports['src/cypress/runner retries mochaEvents test retry with [only] #1'] = [
     "pass",
     {
       "_testConfig": {
-        "testConfigList": [
-          {
-            "overrides": {
-              "testConfigList": [],
-              "unverifiedTestConfig": {}
-            },
-            "invocationDetails": "{Object 9}"
-          }
-        ],
-        "unverifiedTestConfig": {
-          "testConfigList": [],
-          "unverifiedTestConfig": {}
-        }
+        "testConfigList": [],
+        "unverifiedTestConfig": {}
       },
       "id": "r3",
       "title": "test 2",
@@ -1533,19 +1467,8 @@ exports['src/cypress/runner retries mochaEvents test retry with [only] #1'] = [
     "test end",
     {
       "_testConfig": {
-        "testConfigList": [
-          {
-            "overrides": {
-              "testConfigList": [],
-              "unverifiedTestConfig": {}
-            },
-            "invocationDetails": "{Object 9}"
-          }
-        ],
-        "unverifiedTestConfig": {
-          "testConfigList": [],
-          "unverifiedTestConfig": {}
-        }
+        "testConfigList": [],
+        "unverifiedTestConfig": {}
       },
       "id": "r3",
       "title": "test 2",
@@ -1608,19 +1531,8 @@ exports['src/cypress/runner retries mochaEvents test retry with [only] #1'] = [
     "test:after:run",
     {
       "_testConfig": {
-        "testConfigList": [
-          {
-            "overrides": {
-              "testConfigList": [],
-              "unverifiedTestConfig": {}
-            },
-            "invocationDetails": "{Object 9}"
-          }
-        ],
-        "unverifiedTestConfig": {
-          "testConfigList": [],
-          "unverifiedTestConfig": {}
-        }
+        "testConfigList": [],
+        "unverifiedTestConfig": {}
       },
       "id": "r3",
       "title": "test 2",