Skip to content

Commit

Permalink
Merge pull request #20687 from emberjs/ci-cleanup-2
Browse files Browse the repository at this point in the history 
rationalize test matrix
  • Loading branch information
@ef4
ef4 committed Apr 24, 2024
2 parents 3210449 + 3c992a1 commit a6e93bc
Show file tree
Hide file tree
Showing 4 changed files with 100 additions and 205 deletions.
108 changes: 39 additions & 69 deletions .github/workflows/ci.yml
View file
Expand Up @@ -59,7 +59,7 @@ jobs:
run: pnpm type-check

basic-test:
name: Debug and Prebuilt (All Tests by Package + Canary Features)
name: Basic Test
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
Expand All @@ -72,48 +72,51 @@ jobs:
name: dist
path: dist
- name: test
env:
TEST_SUITE: each-package
run: pnpm test

deprecations-enabled-test:
name: Debug and Prebuilt (All Tests by Package + Canary Features) with all Deprecations enabled
variant-tests:
name: ${{ matrix.name }}
runs-on: ubuntu-latest
needs: [basic-test, lint, types]
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/setup
- name: build
run: pnpm ember build
- name: Upload build
uses: actions/upload-artifact@v3
with:
name: dist
path: dist
- name: test
env:
TEST_SUITE: each-package
ALL_DEPRECATIONS_ENABLED: true
run: pnpm test
strategy:
matrix:
include:
- name: "All deprecations enabled"
ALL_DEPRECATIONS_ENABLED: "true"
- name: "All deprecations enabled, with optional features"
ALL_DEPRECATIONS_ENABLED: "true"
ENABLE_OPTIONAL_FEATURES: "true"
- name: "Deprecations as errors"
OVERRIDE_DEPRECATION_VERSION: "15.0.0"
- name: "Deprecations as errors, with optional features"
OVERRIDE_DEPRECATION_VERSION: "15.0.0"
ENABLE_OPTIONAL_FEATURES: "true"
- name: "Production build"
BUILD: "-prod"
- name: "Production build, with optional features"
BUILD: "-prod"
ENABLE_OPTIONAL_FEATURES: "true"
- name: "Extend prototypes"
EXTEND_PROTOTYPES: "true"
- name: "Extend prototypes, with optional features"
EXTEND_PROTOTYPES: "true"
ENABLE_OPTIONAL_FEATURES: "true"
- name: "Prebuilt"
PREBUILT: "true"

deprecations-broken-test:
name: Debug and Prebuilt (All Tests by Package + Canary Features) with Deprecations as Errors
runs-on: ubuntu-latest
needs: [ basic-test, lint, types ]
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/setup
- name: build
run: pnpm ember build
- name: Upload build
uses: actions/upload-artifact@v3
with:
name: dist
path: dist
run: pnpm ember build ${{ matrix.BUILD }}
- name: test
env:
TEST_SUITE: each-package
OVERRIDE_DEPRECATION_VERSION: '15.0.0' # Throws on all deprecations with an until before or equal to this version
ALL_DEPRECATIONS_ENABLED: ${{ matrix.ALL_DEPRECATIONS_ENABLED }}
OVERRIDE_DEPRECATION_VERSION: ${{ matrix.OVERRIDE_DEPRECATION_VERSION }}
EXTEND_PROTOTYPES: ${{ matrix.EXTEND_PROTOTYPES }}
ENABLE_OPTIONAL_FEATURES: ${{ matrix.ENABLE_OPTIONAL_FEATURES }}
PREBUIlT: ${{ matrix.PREBUILT }}

run: pnpm test

browserstack-test:
Expand Down Expand Up @@ -159,35 +162,6 @@ jobs:
pnpm link ../..
pnpm test
production-test:
name: Production (All Tests + Canary Features)
runs-on: ubuntu-latest
needs: [basic-test, lint, types]
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/setup
- name: build
run: pnpm ember build -prod
- name: test
run: pnpm test


extend-prototypes-test:
name: Extend Prototypes
runs-on: ubuntu-latest
needs: [basic-test, lint, types]
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/setup
- uses: actions/download-artifact@v3
with:
name: dist
path: dist
- name: test
env:
TEST_SUITE: extend-prototypes
run: pnpm test

node-test:
name: Node.js Tests
runs-on: ubuntu-latest
Expand Down Expand Up @@ -238,9 +212,8 @@ jobs:
[
basic-test,
lint,
variant-tests,
browserstack-test,
production-test,
extend-prototypes-test,
node-test,
blueprint-test,
browser-test,
Expand All @@ -263,9 +236,8 @@ jobs:
[
basic-test,
lint,
variant-tests,
browserstack-test,
production-test,
extend-prototypes-test,
node-test,
blueprint-test,
browser-test,
Expand All @@ -291,9 +263,8 @@ jobs:
[
basic-test,
lint,
variant-tests,
browserstack-test,
production-test,
extend-prototypes-test,
node-test,
blueprint-test,
browser-test,
Expand Down Expand Up @@ -321,9 +292,8 @@ jobs:
[
basic-test,
lint,
variant-tests,
browserstack-test,
production-test,
extend-prototypes-test,
node-test,
blueprint-test,
browser-test,
Expand Down
30 changes: 3 additions & 27 deletions CONTRIBUTING.md
View file
Expand Up @@ -124,33 +124,10 @@ Pull requests should pass the Ember.js unit tests. Do the following to run these

3. To run all tests, visit <http://localhost:4200/>.

4. To test a specific package, visit `http://localhost:4200/tests/index.html?package=PACKAGE_NAME`. Replace
`PACKAGE_NAME` with the name of the package you want to test. For
example:

* [Ember.js Internals](http://localhost:4200/tests/index.html?package=@ember/-internals)
4. To test a specific package, use the interactive QUnit filters on the page.

To test multiple packages, you can separate them with commas.

## From the CLI

Run `pnpm test` to run a basic test suite or run `TEST_SUITE=all pnpm test` to
run a more comprehensive suite.

## From ember-cli

1. `ember test --server`

2. Connect the browsers you want.

To run a specific browser, you can use the `--launch` flag

* `ember test --server --launch SL_Firefox_Current`
* `ember test --launch SL_Firefox_Current`
* `ember test --launch SL_Firefox_Current,Chrome`

To test multiple launchers, you can separate them with commas.

# Pull Requests

We love pull requests. Here's a quick guide:
Expand Down Expand Up @@ -245,12 +222,11 @@ Within the [CI workflow](https://github.com/emberjs/ember.js/blob/main/.github/w

* `Linting` runs `pnpm lint` to check for code style issues.
* `Type Checking` runs `pnpm type-check` to check for TypeScript type errors. This also runs against several version of TypeScript that are supported.
* `Basic Test` / `each-package` test suite is closest to what you normally run locally on your machine. This is also run with all deprecations enabled -- that is, a deprecation that has been added but is not yet up to it's enabled version will be forced "on" and the test suite is required to pass both ways.
* `Production` test suite runs tests against a production build. This also runs with the "Debug Render Tree" feature enabled.
* `Basic Test` test suite is closest to what you normally run locally on your machine.
* `Variant Tests` run the test suite under different combinations of settings. See documentation in run-tests.js for the meaning of each setting.
* `BrowserStack` and `Browser Tests` test suites run tests against various supported browsers.
* `Blueprint Tests` runs tests for the Ember CLI blueprints provided by Ember in this package.
* `Smoke Test` builds and runs a simple app.
* `Extend Prototypes` runs the `extend-prototypes` test suite.
* `Node.js Tests` runs tests for the node-side code in this package.

Each commit to canary and beta publishes to the relevant channel on S3. These
Expand Down
149 changes: 49 additions & 100 deletions bin/run-tests.js
View file
@@ -1,13 +1,43 @@
/* eslint-disable no-console */
'use strict';

/*
Test Variants
These are all accepted as environment variables when running `ember test` or
as query params when directly invoking the test suite in the browser.
*/
const variants = [
// When true, even deprecations that are not yet at the "enabled" version will
// be enabled, so we can ensure that they and their tests will continue to
// function correctly when we hit the enabled version.
'ALL_DEPRECATIONS_ENABLED',

// This overrides the current version of ember for purposes of seeing how
// deprecations behave. We use it in CI to prove that after a deprecation has
// hit its "until" version, the tests for it will behave correctly.
'OVERRIDE_DEPRECATION_VERSION',

// This enables the legacy Ember feature that causes Ember to extend built-in
// platform features like Array.
'EXTEND_PROTOTYPES',

// This enables all canary feature flags for unreleased feature within Ember
// itself.
'ENABLE_OPTIONAL_FEATURES',

// This forces the test suite to run against the prepackaged copy of
// ember.debug.js that publishes in the ember-source NPM package. That copy is
// essentially an optimization if you happen to be doing development under the
// default babel targets.
'PREBUILT',
];

const chalk = require('chalk');
const path = require('path');

const finalhandler = require('finalhandler');
const http = require('http');
const serveStatic = require('serve-static');
const fs = require('fs');

// Serve up public/ftp folder.
const serve = serveStatic('./dist/', { index: ['index.html', 'index.htm'] });
Expand All @@ -34,13 +64,13 @@ function getBrowserRunner() {
return browserRunner;
}

function run(queryString) {
if (process.env.ALL_DEPRECATIONS_ENABLED) {
queryString = `${queryString}&alldeprecationsenabled=${process.env.ALL_DEPRECATIONS_ENABLED}`;
}

if (process.env.OVERRIDE_DEPRECATION_VERSION) {
queryString = `${queryString}&overridedeprecationversion=${process.env.OVERRIDE_DEPRECATION_VERSION}`;
function run() {
let queryString = '';
for (let variant of variants) {
if (process.env[variant]) {
console.log(`Applying variant ${variant}=${process.env[variant]}`);
queryString = `${queryString}&${variant}=${process.env[variant]}`;
}
}

let url = 'http://localhost:' + PORT + '/tests/?' + queryString;
Expand All @@ -52,94 +82,13 @@ function runInBrowser(url, attempts) {
return getBrowserRunner().run(url, attempts);
}

let testFunctions = [];

function generateTestsFor(packageName) {
let relativePath = path.join('packages', packageName);

if (!fs.existsSync(path.join(relativePath, 'tests'))) {
return;
}

testFunctions.push(() => run('package=' + packageName));
testFunctions.push(() => run('package=' + packageName + '&prebuilt=true'));
testFunctions.push(() => run('package=' + packageName + '&enableoptionalfeatures=true'));
}

function generateEachPackageTests() {
fs.readdirSync('packages/@ember')
.filter((e) => e !== '-internals')
.forEach((e) => generateTestsFor(`@ember/${e}`));

fs.readdirSync('packages/@ember/-internals').forEach((e) =>
generateTestsFor(`@ember/-internals/${e}`)
);

fs.readdirSync('packages')
.filter((e) => e !== '@ember')
.forEach(generateTestsFor);
}

function generateStandardTests() {
testFunctions.push(() => run(''));
testFunctions.push(() => run('enableoptionalfeatures=true'));
}

function generateExtendPrototypeTests() {
testFunctions.push(() => run('extendprototypes=true'));
testFunctions.push(() => run('extendprototypes=true&enableoptionalfeatures=true'));
}

function runInSequence(tasks) {
let length = tasks.length;
let current = Promise.resolve();
let results = new Array(length);

for (let i = 0; i < length; ++i) {
current = results[i] = current.then(tasks[i]);
}

return Promise.all(results);
}

function runAndExit() {
runInSequence(testFunctions)
.then(function () {
console.log(chalk.green('Passed!'));
process.exit(0); // eslint-disable-line n/no-process-exit
})
.catch(function (err) {
console.error(chalk.red(err.toString()));
console.error(chalk.red('Failed!'));
process.exit(1); // eslint-disable-line n/no-process-exit
});
}

let p = process.env.PACKAGE || 'ember';
switch (process.env.TEST_SUITE) {
case 'package':
console.log(`suite: package ${p}`);
generateTestsFor(p);
runAndExit();
break;
case 'each-package':
console.log('suite: each-package');
generateEachPackageTests();
runAndExit();
break;
case 'extend-prototypes':
console.log('suite: extend-prototypes');
generateExtendPrototypeTests();
runAndExit();
break;
case 'all':
console.log('suite: all');
generateExtendPrototypeTests();
generateEachPackageTests();
runAndExit();
break;
default:
console.log('suite: default (generate each package)');
generateStandardTests();
runAndExit();
}
run()
.then(function () {
console.log(chalk.green('Passed!'));
process.exit(0); // eslint-disable-line n/no-process-exit
})
.catch(function (err) {
console.error(chalk.red(err.toString()));
console.error(chalk.red('Failed!'));
process.exit(1); // eslint-disable-line n/no-process-exit
});

0 comments on commit a6e93bc

Please sign in to comment.