diff --git a/docs/06-configuration.md b/docs/06-configuration.md index bd6545aff..ccf081d50 100644 --- a/docs/06-configuration.md +++ b/docs/06-configuration.md @@ -58,6 +58,7 @@ Arguments passed to the CLI will always take precedence over the CLI options con - `require`: extra modules to require before tests are run. Modules are required in the [worker processes](./01-writing-tests.md#process-isolation) - `timeout`: Timeouts in AVA behave differently than in other test frameworks. AVA resets a timer after each test, forcing tests to quit if no new test results were received within the specified timeout. This can be used to handle stalled tests. See our [timeout documentation](./07-test-timeouts.md) for more options. - `nodeArguments`: Configure Node.js arguments used to launch worker processes. +- `sortTestFiles`: A comparator function to sort test files with. Available only when using a `ava.config.*` file. See an example use case [here](recipes/splitting-tests-ci.md). Note that providing files on the CLI overrides the `files` option. diff --git a/docs/recipes/splitting-tests-ci.md b/docs/recipes/splitting-tests-ci.md new file mode 100644 index 000000000..95849d3dd --- /dev/null +++ b/docs/recipes/splitting-tests-ci.md @@ -0,0 +1,22 @@ +# Splitting tests in CI + +AVA automatically detects whether your CI environment supports parallel builds using [ci-parallel-vars](https://www.npmjs.com/package/ci-parallel-vars). When parallel builds support is detected, AVA sorts the all detected test files by name, and splits them into chunks. Each CI machine is assigned a chunk (subset) of the tests, and then each chunk is run in parallel. + +To better distribute the tests across the machines, you can configure a custom comparator function: + +**`ava.config.js`:** + +```js +import fs from 'node:fs'; + +// Assuming 'test-data.json' structure is: +// { +// 'tests/test1.js': { order: 1 }, +// 'tests/test2.js': { order: 0 } +// } +const testData = JSON.parse(fs.readFileSync('test-data.json', 'utf8')); + +export default { + sortTestFiles: (file1, file2) => testData[file1].order - testData[file2].order, +}; +``` diff --git a/lib/api.js b/lib/api.js index bf0cb2498..1ebe27ce5 100644 --- a/lib/api.js +++ b/lib/api.js @@ -177,13 +177,19 @@ export default class Api extends Emittery { const fileCount = selectedFiles.length; // The files must be in the same order across all runs, so sort them. - selectedFiles = selectedFiles.sort((a, b) => a.localeCompare(b, [], {numeric: true})); + const defaultComparator = (a, b) => a.localeCompare(b, [], {numeric: true}); + selectedFiles = selectedFiles.sort(this.options.sortTestFiles || defaultComparator); selectedFiles = chunkd(selectedFiles, currentIndex, totalRuns); const currentFileCount = selectedFiles.length; runStatus = new RunStatus(fileCount, {currentFileCount, currentIndex, totalRuns}, selectionInsights); } else { + // If a custom sorter was configured, use it. + if (this.options.sortTestFiles) { + selectedFiles = selectedFiles.sort(this.options.sortTestFiles); + } + runStatus = new RunStatus(selectedFiles.length, null, selectionInsights); } @@ -261,8 +267,8 @@ export default class Api extends Emittery { } const lineNumbers = getApplicableLineNumbers(globs.normalizeFileForMatching(apiOptions.projectDir, file), filter); - // Removing `providers` field because they cannot be transfered to the worker threads. - const {providers, ...forkOptions} = apiOptions; + // Removing `providers` and `sortTestFiles` fields because they cannot be transferred to the worker threads. + const {providers, sortTestFiles, ...forkOptions} = apiOptions; const options = { ...forkOptions, providerStates, diff --git a/lib/cli.js b/lib/cli.js index 48c87caa5..80d49a69f 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -321,6 +321,10 @@ export default async function loadCli() { // eslint-disable-line complexity exit('’sources’ has been removed. Use ’ignoredByWatcher’ to provide glob patterns of files that the watcher should ignore.'); } + if (Reflect.has(conf, 'sortTestFiles') && typeof conf.sortTestFiles !== 'function') { + exit('’sortTestFiles’ must be a comparator function.'); + } + let projectPackageObject; try { projectPackageObject = JSON.parse(fs.readFileSync(path.resolve(projectDir, 'package.json'))); @@ -413,6 +417,7 @@ export default async function loadCli() { // eslint-disable-line complexity moduleTypes, nodeArguments, parallelRuns, + sortTestFiles: conf.sortTestFiles, projectDir, providers, ranFromCli: true, diff --git a/readme.md b/readme.md index ab865265c..78b4a9cd9 100644 --- a/readme.md +++ b/readme.md @@ -139,15 +139,16 @@ We have a growing list of [common pitfalls](docs/08-common-pitfalls.md) you may ### Recipes -- [Shared workers](docs/recipes/shared-workers.md) - [Test setup](docs/recipes/test-setup.md) -- [Code coverage](docs/recipes/code-coverage.md) +- [TypeScript](docs/recipes/typescript.md) +- [Shared workers](docs/recipes/shared-workers.md) - [Watch mode](docs/recipes/watch-mode.md) -- [Endpoint testing](docs/recipes/endpoint-testing.md) - [When to use `t.plan()`](docs/recipes/when-to-use-plan.md) -- [Browser testing](docs/recipes/browser-testing.md) -- [TypeScript](docs/recipes/typescript.md) - [Passing arguments to your test files](docs/recipes/passing-arguments-to-your-test-files.md) +- [Splitting tests in CI](docs/recipes/splitting-tests-ci.md) +- [Code coverage](docs/recipes/code-coverage.md) +- [Endpoint testing](docs/recipes/endpoint-testing.md) +- [Browser testing](docs/recipes/browser-testing.md) - [Testing Vue.js components](docs/recipes/vue.md) - [Debugging tests with Chrome DevTools](docs/recipes/debugging-with-chrome-devtools.md) - [Debugging tests with VSCode](docs/recipes/debugging-with-vscode.md) diff --git a/test-tap/fixture/parallel-runs/custom-comparator/0-1.cjs b/test-tap/fixture/parallel-runs/custom-comparator/0-1.cjs new file mode 100644 index 000000000..5b1c05662 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/0-1.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '2'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/0-2.cjs b/test-tap/fixture/parallel-runs/custom-comparator/0-2.cjs new file mode 100644 index 000000000..5b1c05662 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/0-2.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '2'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/0-3.cjs b/test-tap/fixture/parallel-runs/custom-comparator/0-3.cjs new file mode 100644 index 000000000..5b1c05662 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/0-3.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '2'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/1-1.cjs b/test-tap/fixture/parallel-runs/custom-comparator/1-1.cjs new file mode 100644 index 000000000..ccb999168 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/1-1.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '1'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/1-2.cjs b/test-tap/fixture/parallel-runs/custom-comparator/1-2.cjs new file mode 100644 index 000000000..ccb999168 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/1-2.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '1'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/1-3.cjs b/test-tap/fixture/parallel-runs/custom-comparator/1-3.cjs new file mode 100644 index 000000000..ccb999168 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/1-3.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '1'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/2-1.cjs b/test-tap/fixture/parallel-runs/custom-comparator/2-1.cjs new file mode 100644 index 000000000..a40e62fb8 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/2-1.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '0'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/2-2.cjs b/test-tap/fixture/parallel-runs/custom-comparator/2-2.cjs new file mode 100644 index 000000000..a40e62fb8 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/2-2.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '0'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/2-3.cjs b/test-tap/fixture/parallel-runs/custom-comparator/2-3.cjs new file mode 100644 index 000000000..a40e62fb8 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/2-3.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '0'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/ava.config.js b/test-tap/fixture/parallel-runs/custom-comparator/ava.config.js new file mode 100644 index 000000000..935e14cf2 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/ava.config.js @@ -0,0 +1,5 @@ +export default { + files: ['*.cjs'], + // Descending order + sortTestFiles: (a, b) => b.localeCompare(a, [], {numeric: true}), +}; diff --git a/test-tap/fixture/parallel-runs/custom-comparator/package.json b/test-tap/fixture/parallel-runs/custom-comparator/package.json new file mode 100644 index 000000000..bedb411a9 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/test-tap/fixture/sort-tests/0.cjs b/test-tap/fixture/sort-tests/0.cjs new file mode 100644 index 000000000..1af77b598 --- /dev/null +++ b/test-tap/fixture/sort-tests/0.cjs @@ -0,0 +1,5 @@ +const test = require('../../../entrypoints/main.cjs'); + +test('should run third', t => { + t.pass(); +}); diff --git a/test-tap/fixture/sort-tests/1.cjs b/test-tap/fixture/sort-tests/1.cjs new file mode 100644 index 000000000..e1e0ff47b --- /dev/null +++ b/test-tap/fixture/sort-tests/1.cjs @@ -0,0 +1,5 @@ +const test = require('../../../entrypoints/main.cjs'); + +test('should run second', t => { + t.pass(); +}); diff --git a/test-tap/fixture/sort-tests/2.cjs b/test-tap/fixture/sort-tests/2.cjs new file mode 100644 index 000000000..9ad7df09f --- /dev/null +++ b/test-tap/fixture/sort-tests/2.cjs @@ -0,0 +1,5 @@ +const test = require('../../../entrypoints/main.cjs'); + +test('should run first', t => { + t.pass(); +}); diff --git a/test-tap/fixture/sort-tests/ava.config.js b/test-tap/fixture/sort-tests/ava.config.js new file mode 100644 index 000000000..58d098288 --- /dev/null +++ b/test-tap/fixture/sort-tests/ava.config.js @@ -0,0 +1,7 @@ +export default { + files: ['*.cjs'], + // Descending order + sortTestFiles: (a, b) => b.localeCompare(a, [], {numeric: true}), + concurrency: 1, + verbose: true, +}; diff --git a/test-tap/fixture/sort-tests/package.json b/test-tap/fixture/sort-tests/package.json new file mode 100644 index 000000000..bedb411a9 --- /dev/null +++ b/test-tap/fixture/sort-tests/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/test-tap/integration/assorted.js b/test-tap/integration/assorted.js index e866824c6..db292e8ac 100644 --- a/test-tap/integration/assorted.js +++ b/test-tap/integration/assorted.js @@ -152,3 +152,11 @@ test('load .js test files as ESM modules', t => { t.end(); }); }); + +test('uses sortTestFiles to sort test files', t => { + execCli([], {dirname: 'fixture/sort-tests'}, (error, stdout) => { + t.error(error); + t.match(stdout, /should run first[\s\S]+?should run second[\s\S]+?should run third/); + t.end(); + }); +}); diff --git a/test-tap/integration/parallel-runs.js b/test-tap/integration/parallel-runs.js index fe30e38c6..47f4c0781 100644 --- a/test-tap/integration/parallel-runs.js +++ b/test-tap/integration/parallel-runs.js @@ -43,3 +43,17 @@ test('fail when there are no files', t => { }, error => t.ok(error)); } }); + +test('correctly applies custom comparator', t => { + t.plan(3); + for (let i = 0; i < 3; i++) { + execCli([], { + dirname: 'fixture/parallel-runs/custom-comparator', + env: { + AVA_FORCE_CI: 'ci', + CI_NODE_INDEX: String(i), + CI_NODE_TOTAL: '3', + }, + }, error => t.error(error)); + } +});