chore: move server e2e tests to system-tests (#16354)

Co-authored-by: Brian Mann <brian.mann86@gmail.com>
Co-authored-by: Zach Bloomquist <git@chary.us>
Co-authored-by: Zach Bloomquist <github@chary.us>

.eslintignore

@@ -14,6 +14,21 @@
 **/support/fixtures/projects/**/static/*
 **/support/fixtures/projects/**/*.jsx
 **/support/fixtures/projects/**/fail.js
+
+system-tests/fixtures/*
+!system-tests/projects
+system-tests/projects/**/_fixtures/*
+system-tests/projects/**/static/*
+system-tests/projects/**/*.jsx
+system-tests/projects/**/fail.js
+system-tests/lib/scaffold/plugins/index.js
+system-tests/lib/scaffold/support/index.js
+system-tests/lib/scaffold/support/commands.js
+system-tests/test/support/projects/e2e/cypress/
+system-tests/projects/e2e/cypress/integration/stdout_exit_early_failing_spec.js
+system-tests/projects/e2e/cypress/integration/typescript_syntax_error_spec.ts
+
+
 **/test/fixtures
 **/vendor
 
@@ -23,11 +38,7 @@ cli/types
 packages/example
 
 packages/extension/test/helpers/background.js
-packages/server/lib/scaffold/plugins/index.js
-packages/server/lib/scaffold/support/index.js
-packages/server/lib/scaffold/support/commands.js
-packages/server/test/support/fixtures/projects/e2e/cypress/integration/stdout_exit_early_failing_spec.js
-packages/server/test/support/fixtures/projects/e2e/cypress/integration/typescript_syntax_error_spec.ts
+integration/stdout_exit_early_failing_spec.js
 
 npm/webpack-preprocessor/cypress/tests/e2e/compile-error.js
 npm/webpack-preprocessor/examples/use-babelrc/cypress/integration/spec.js

.gitignore

@@ -40,6 +40,10 @@ packages/server/support
 packages/server/test/support/fixtures/server/imgs
 packages/server/test/support/fixtures/server/libs
 
+# from system-tests
+system-tests/.projects
+system-tests/fixtures/large-img
+
 # from npm/react
 /npm/react/bin/*
 /npm/react/cypress/videos

.vscode/terminals.json

@@ -24,12 +24,20 @@
       "command": "yarn cypress:run --project ../project"
     },
     {
-      "name": "packages/server test-e2e",
+      "name": "cypress open (CT)",
+      "focus": true,
+      "onlySingle": true,
+      "execute": true,
+      "cwd": "[cwd]/packages/server-ct",
+      "command": "yarn cypress:open"
+    },
+    {
+      "name": "system-tests test",
       "focus": true,
       "onlySingle": true,
       "execute": false,
-      "cwd": "[cwd]/packages/server",
-      "command": "yarn test-e2e [fileBasename]"
+      "cwd": "[cwd]/system-tests",
+      "command": "yarn test [fileBasename]"
     },
     {
       "name": "packages/server test-watch",

CONTRIBUTING.md

@@ -396,6 +396,7 @@ By default, top level tasks will execute for all packages. However, most scripts
 | `test-unit`        | Run unit tests                                                   |
 | `test-integration` | Run integration tests                                            |
 | `test-e2e`         | Run end-to-end tests                                             |
+| `test-system`      | Run system tests                                                 |
 | `test-watch`       | Run unit tests and rebuild/rerun on file changes                 |
 
 > Most of the time you will only want to run a task within a specific package; this can be done by providing the package name as a scope to the top level task.
@@ -428,7 +429,6 @@ Each package is responsible for building itself and testing itself and can do so
 | `test`             | Runs all tests once (this usually means running unit tests; via `yarn test-unit`)                                                                        |
 | `test-unit`        | Run all unit tests within the package; `exit 0` if N/A                                                                                                   |
 | `test-integration` | Run all integration tests within the package; `exit 0` if N/A                                                                                            |
-| `test-e2e`         | Run all e2e tests within the package; `exit 0` if N/A                                                                                                    |
 | `test-watch`       | Run all unit tests in the package in watch mode                                                                                                          |
 
 #### Debugging
@@ -486,11 +486,11 @@ This is to ensure that links do not go dead in older versions of Cypress when th
 
 For most packages there are typically unit and integration tests.
 
-Our true e2e tests are in [`packages/server`](packages/server), which test the full stack all together.
+Please refer to each packages' `README.md` which documents how to run tests. It is not feasible to try to run all of the tests together. We run our entire test fleet across over a dozen containers in CI.
 
-Additionally, we test the code by running it against various other example projects in CI. See CI badges and links at the top of this document.
+There are also a set of system tests in [`system-tests`](system-tests) which attempt to test the entire Cypress App as close to real world as possible. See  the [`README`](system-tests/README.md) for more information.
 
-Please refer to each packages' `README.md` which documents how to run tests. It is not feasible to try to run all of the tests together. We run our entire test fleet across over a dozen containers in CI.
+Additionally, we test the code by running it against various other example projects in CI. See CI badges and links at the top of this document.
 
 If you're curious how we manage all of these tests in CI check out our [`circle.yml`](circle.yml) file found in the root `cypress` directory.
 

circle.yml

@@ -112,6 +112,7 @@ commands:
                   mkdir -p /tmp/node_modules_cache
                   mv ~/cypress/node_modules /tmp/node_modules_cache/root_node_modules
                   mv ~/cypress/cli/node_modules /tmp/node_modules_cache/cli_node_modules
+                  mv ~/cypress/system-tests/node_modules /tmp/node_modules_cache/system-tests_node_modules
                   mv ~/cypress/globbed_node_modules /tmp/node_modules_cache/globbed_node_modules
 
   build-and-persist:
@@ -142,6 +143,7 @@ commands:
             if [[ -d "/tmp/node_modules_cache" ]]; then
               mv /tmp/node_modules_cache/root_node_modules ~/cypress/node_modules
               mv /tmp/node_modules_cache/cli_node_modules ~/cypress/cli/node_modules
+              mv /tmp/node_modules_cache/system-tests_node_modules ~/cypress/system-tests/node_modules
               mv /tmp/node_modules_cache/globbed_node_modules ~/cypress/globbed_node_modules
               rm -rf /tmp/node_modules_cache
             fi
@@ -194,6 +196,7 @@ commands:
                 paths:
                   - node_modules
                   - cli/node_modules
+                  - system-tests/node_modules
                   - globbed_node_modules
       - unless:
           condition: <<parameters.is-mac>>
@@ -417,16 +420,17 @@ commands:
           path: ./packages/runner-ct/cypress/videos
       - store-npm-logs
 
-  run-e2e-tests:
+  run-system-tests:
     parameters:
       browser:
         description: browser shortname to target
         type: string
     steps:
       - restore_cached_workspace
       - run:
+          name: Run system tests
           command: |
-            ALL_SPECS=`circleci tests glob "/root/cypress/packages/server/test/e2e/*spec*"`
+            ALL_SPECS=`circleci tests glob "/root/cypress/system-tests/test/*spec*"`
             SPECS=
             for file in $ALL_SPECS; do
               # filter out non_root tests, they have their own stage
@@ -438,7 +442,7 @@ commands:
             done
             SPECS=`echo $SPECS | xargs -n 1 | circleci tests split --split-by=timings`
             echo SPECS=$SPECS
-            yarn workspace @packages/server test $SPECS --browser <<parameters.browser>>
+            yarn workspace @tooling/system-tests test:ci $SPECS --browser <<parameters.browser>>
       - verify-mocha-results
       - store_test_results:
           path: /tmp/cypress
@@ -1080,37 +1084,37 @@ jobs:
           path: /tmp/artifacts
       - store-npm-logs
 
-  server-e2e-tests-chrome:
+  system-tests-chrome:
     <<: *defaults
     resource_class: medium
     parallelism: 8
     steps:
-      - run-e2e-tests:
+      - run-system-tests:
           browser: chrome
 
-  server-e2e-tests-electron:
+  system-tests-electron:
     <<: *defaults
     resource_class: medium
     parallelism: 8
     steps:
-      - run-e2e-tests:
+      - run-system-tests:
           browser: electron
 
-  server-e2e-tests-firefox:
+  system-tests-firefox:
     <<: *defaults
     resource_class: medium
     parallelism: 8
     steps:
-      - run-e2e-tests:
+      - run-system-tests:
           browser: firefox
 
-  server-e2e-tests-non-root:
+  system-tests-non-root:
     <<: *defaults
     resource_class: medium
     steps:
       - restore_cached_workspace
       - run:
-          command: yarn workspace @packages/server test ./test/e2e/non_root*spec* --browser electron
+          command: yarn workspace @tooling/system-tests test:ci "test/non_root*spec*" --browser electron
       - verify-mocha-results
       - store_test_results:
           path: /tmp/cypress
@@ -2007,16 +2011,16 @@ linux-workflow: &linux-workflow
     - server-performance-tests:
         requires:
           - build
-    - server-e2e-tests-chrome:
+    - system-tests-chrome:
         requires:
           - build
-    - server-e2e-tests-electron:
+    - system-tests-electron:
         requires:
           - build
-    - server-e2e-tests-firefox:
+    - system-tests-firefox:
         requires:
           - build
-    - server-e2e-tests-non-root:
+    - system-tests-non-root:
         executor: non-root-docker-user
         requires:
           - build
@@ -2123,10 +2127,10 @@ linux-workflow: &linux-workflow
           - driver-integration-tests-firefox
           - driver-integration-tests-chrome
           - driver-integration-tests-electron
-          - server-e2e-tests-non-root
-          - server-e2e-tests-firefox
-          - server-e2e-tests-electron
-          - server-e2e-tests-chrome
+          - system-tests-non-root
+          - system-tests-firefox
+          - system-tests-electron
+          - system-tests-chrome
           - server-performance-tests
           - server-integration-tests
           - server-unit-tests

lerna.json

@@ -3,7 +3,8 @@
   "packages": [
     "cli",
     "packages/*",
-    "npm/*"
+    "npm/*",
+    "system-tests"
   ],
   "useWorkspaces": true,
   "version": "0.0.0"

npm/webpack-preprocessor/index.ts

@@ -268,6 +268,11 @@ const preprocessor: WebpackPreprocessor = (options: PreprocessorOptions = {}): F
 
       const jsonStats = stats.toJson()
 
+      // these stats are really only useful for debugging
+      if (jsonStats.warnings.length > 0) {
+        debug(`warnings for ${outputPath} %o`, jsonStats.warnings)
+      }
+
       if (stats.hasErrors()) {
         err = new Error('Webpack Compilation Error')
 
@@ -279,17 +284,11 @@ const preprocessor: WebpackPreprocessor = (options: PreprocessorOptions = {}): F
 
         err.message += `\n${errorsToAppend}`
 
-        debug('stats had error(s)')
+        debug('stats had error(s) %o', jsonStats.errors)
 
         return rejectWithErr(err)
       }
 
-      // these stats are really only useful for debugging
-      if (jsonStats.warnings.length > 0) {
-        debug(`warnings for ${outputPath}`)
-        debug(jsonStats.warnings)
-      }
-
       debug('finished bundling', outputPath)
       if (debugStats.enabled) {
         /* eslint-disable-next-line no-console */

npm/webpack-preprocessor/test/e2e/helpers.js

@@ -83,7 +83,7 @@ exports.runTest = async (options = {}) => {
     VIDEO_COMPRESSION_THROTTLE: 120000,
 
     // don't fail our own tests running from forked PR's
-    CYPRESS_INTERNAL_E2E_TESTS: '1',
+    CYPRESS_INTERNAL_SYSTEM_TESTS: '1',
     CYPRESS_ENV: 'test',
   })
 

package.json

@@ -49,12 +49,12 @@
     "test": "yarn lerna exec yarn test --scope cypress --scope \"'@packages/{electron,extension,https-proxy,launcher,net-stubbing,network,proxy,rewriter,runner,runner-shared,socket}'\"",
     "test-debug": "lerna exec yarn test-debug --ignore \"'@packages/{desktop-gui,driver,root,static,web-config}'\"",
     "pretest-e2e": "yarn ensure-deps",
-    "test-e2e": "lerna exec yarn test-e2e --ignore \"'@packages/{desktop-gui,driver,root,static,web-config}'\"",
     "test-integration": "lerna exec yarn test-integration --ignore \"'@packages/{desktop-gui,driver,root,static,web-config}'\"",
     "test-mocha": "mocha --reporter spec scripts/spec.js",
     "test-mocha-snapshot": "mocha scripts/mocha-snapshot-spec.js",
     "test-npm-package-release-script": "npx lerna exec --scope \"@cypress/*\" -- npx --no-install semantic-release --dry-run",
     "test-s3-api": "node -r ./packages/ts/register scripts/binary/s3-api-demo.ts",
+    "test-system": "yarn workspace @tooling/system-tests test",
     "test-scripts": "mocha -r packages/ts/register --reporter spec 'scripts/unit/**/*spec.js'",
     "test-scripts-watch": "yarn test-scripts --watch --watch-extensions 'ts,js'",
     "pretest-unit": "yarn ensure-deps",
@@ -224,7 +224,8 @@
     "packages": [
       "cli",
       "packages/*",
-      "npm/*"
+      "npm/*",
+      "system-tests"
     ],
     "nohoist": [
       "**/@ffmpeg-installer",

packages/runner/cypress/fixtures/errors/exceptions_spec.js

@@ -1,6 +1,6 @@
 import './setup'
 
-const outsideError = require('../../../../server/test/support/fixtures/projects/todos/throws-error')
+const outsideError = require('@tooling/system-tests/projects/todos/throws-error')
 
 describe('exception failures', () => {
   it('in spec file', () => {

packages/server/README.md

@@ -39,16 +39,13 @@ yarn workspace @packages/server build-prod
 * `yarn test-unit` executes unit tests in [`test/unit`](./test/unit)
 * `yarn test-integration` executes integration tests in [`test/integration`](./test/integration)
 * `yarn test-performance` executes performance tests in [`test/performance`](./test/performance)
-* `yarn test-e2e` executes the large (slow) end to end tests in [`test/e2e`](./test/e2e)
 
 You can also use the `test-watch` command to rerun a test file whenever there is a change:
 
 ```bash
 yarn test-watch /test/path/to/spec.js
 ```
 
-When running e2e tests, some test projects output verbose logs. To see them run the test with `DEBUG=cypress:e2e` environment variable.
-
 ### Running individual unit tests
 
 ```bashtest-kitchensink
@@ -67,19 +64,9 @@ yarn test test/integration/cli_spec.js
 yarn test-integration cli_spec ## shorthand, uses globbing to find spec
 ```
 
-### Running individual e2e tests
+### Running e2e/system tests
 
-```bash
-yarn test <path/to/test>
-yarn test test/e2e/1_async_timeouts_spec.js
-## or
-yarn test-e2e async_timeouts ## shorthand, uses globbing to find spec
-```
-
-To keep the browser open after a spec run (for easier debugging and iterating on specs), you can pass the `--no-exit` flag to the e2e test command. Live reloading due to spec changes should also work:
-```sh
-yarn test test/e2e/go_spec.js --browser chrome --no-exit
-```
+> With the addition of Component Testing, `e2e` tests have been renamed to `system-tests` and moved to the [`system-tests`](../../system-tests) directory.
 
 ### Updating snaphots
 
@@ -88,5 +75,4 @@ Prepend `SNAPSHOT_UPDATE=1` to any test command. See [`snap-shot-it` instruction
 ```bash
 SNAPSHOT_UPDATE=1 yarn test test/unit/api_spec.js
 SNAPSHOT_UPDATE=1 yarn test test/integration/cli_spec.js
-SNAPSHOT_UPDATE=1 yarn test-e2e async_timeout
 ```

packages/server/lib/modes/record.js

@@ -32,7 +32,7 @@ const logException = (err) => {
 // dont yell about any errors either
 
 const runningInternalTests = () => {
-  return env.get('CYPRESS_INTERNAL_E2E_TESTS') === '1'
+  return env.get('CYPRESS_INTERNAL_SYSTEM_TESTS') === '1'
 }
 
 const haveProjectIdAndKeyButNoRecordOption = (projectId, options) => {