Merge pull request #20691 from emberjs/smoke-test-matrix

Expand smoke test coverage

.github/workflows/ci.yml

@@ -24,11 +24,18 @@ jobs:
   lint:
     name: Linting
     runs-on: ubuntu-latest
+    outputs:
+       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/setup
       - name: linting
         run: pnpm lint
+      - id: set-matrix
+        working-directory: smoke-tests/scenarios
+        run: |
+          matrix_json=$(pnpm scenario-tester list --require @swc-node/register --files "*-test.ts" --matrix "pnpm run test --filter %s"   )
+          echo "matrix=$matrix_json" >> $GITHUB_OUTPUT        
 
   types:
     name: Type Checking (current version)
@@ -149,18 +156,18 @@ jobs:
     name: Smoke tests (Full Ember Apps)
     runs-on: ubuntu-latest
     needs: [basic-test, lint, types]
+    strategy:
+       fail-fast: false
+       matrix: ${{fromJson(needs.lint.outputs.matrix)}}
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/setup
       - name: build
         run: pnpm ember build -prod
       - name: test
-        # todo: potentially more apps? use matrix?
+        working-directory: smoke-tests/scenarios
         run: |
-          cd smoke-tests/ember-test-app/
-          pnpm install
-          pnpm link ../..
-          pnpm test
+          ${{ matrix.command }}
 
   node-test:
     name: Node.js Tests

CONTRIBUTING.md

@@ -226,7 +226,7 @@ Within the [CI workflow](https://github.com/emberjs/ember.js/blob/main/.github/w
 * `Variant Tests` run the test suite under different combinations of settings. See documentation in run-tests.js for the meaning of each setting.
 * `BrowserStack` and `Browser Tests` test suites run tests against various supported browsers.
 * `Blueprint Tests` runs tests for the Ember CLI blueprints provided by Ember in this package.
-* `Smoke Test` builds and runs a simple app.
+* `Smoke Tests` builds and runs entire end-to-end apps.
 * `Node.js Tests` runs tests for the node-side code in this package.
 
 Each commit to canary and beta publishes to the relevant channel on S3. These 
@@ -235,6 +235,38 @@ for addons to test against beta and canary of ember-source.
 
 The CI workflow is also run nightly for beta and canary.
 
+## Working with the Smoke Test apps
+
+The tests under `/smoke-tests` build complete apps by starting from an app-template and modifying it for various scenarios.
+
+You can **list all scenarios**:
+
+```sh
+cd smoke-tests/scenarios
+pnpm test:list
+```
+
+You can run one of them non-interactively:
+
+```sh
+cd smoke-tests/scenarios
+pnpm run test --filter $SCENARIO_NAME
+```
+
+Or you can write the whole application out as a normal on-disk app in order to run and debug it interactively:
+
+```sh
+cd smoke-tests/scenarios
+pnpm test:output --scenario $SCENARIO_NAME --outdir /tmp/my-scenario
+cd /tmp/my-scenario
+pnpm start
+```
+
+
+
+
+
+
 ## Common CI Build Issues
 
 ### Production Build Failures