From ada1a4f8876ad7b2476440609691e301a02f3951 Mon Sep 17 00:00:00 2001 From: Erez Rokah Date: Tue, 1 Mar 2022 19:39:57 +0100 Subject: [PATCH] Allow customized sorting of test files prior to execution This allows you to better control the distribution of test files across parallel runs. You can also control execution order for regular runs, including on your local machine. Co-authored-by: Mark Wubben --- docs/06-configuration.md | 1 + docs/recipes/splitting-tests-ci.md | 22 +++++++++++++++++++ lib/api.js | 12 +++++++--- lib/cli.js | 5 +++++ readme.md | 11 +++++----- .../parallel-runs/custom-comparator/0-1.cjs | 5 +++++ .../parallel-runs/custom-comparator/0-2.cjs | 5 +++++ .../parallel-runs/custom-comparator/0-3.cjs | 5 +++++ .../parallel-runs/custom-comparator/1-1.cjs | 5 +++++ .../parallel-runs/custom-comparator/1-2.cjs | 5 +++++ .../parallel-runs/custom-comparator/1-3.cjs | 5 +++++ .../parallel-runs/custom-comparator/2-1.cjs | 5 +++++ .../parallel-runs/custom-comparator/2-2.cjs | 5 +++++ .../parallel-runs/custom-comparator/2-3.cjs | 5 +++++ .../custom-comparator/ava.config.js | 5 +++++ .../custom-comparator/package.json | 3 +++ test-tap/fixture/sort-tests/0.cjs | 5 +++++ test-tap/fixture/sort-tests/1.cjs | 5 +++++ test-tap/fixture/sort-tests/2.cjs | 5 +++++ test-tap/fixture/sort-tests/ava.config.js | 7 ++++++ test-tap/fixture/sort-tests/package.json | 3 +++ test-tap/integration/assorted.js | 8 +++++++ test-tap/integration/parallel-runs.js | 14 ++++++++++++ 23 files changed, 143 insertions(+), 8 deletions(-) create mode 100644 docs/recipes/splitting-tests-ci.md create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/0-1.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/0-2.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/0-3.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/1-1.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/1-2.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/1-3.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/2-1.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/2-2.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/2-3.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/ava.config.js create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/package.json create mode 100644 test-tap/fixture/sort-tests/0.cjs create mode 100644 test-tap/fixture/sort-tests/1.cjs create mode 100644 test-tap/fixture/sort-tests/2.cjs create mode 100644 test-tap/fixture/sort-tests/ava.config.js create mode 100644 test-tap/fixture/sort-tests/package.json diff --git a/docs/06-configuration.md b/docs/06-configuration.md index bd6545aff..ccf081d50 100644 --- a/docs/06-configuration.md +++ b/docs/06-configuration.md @@ -58,6 +58,7 @@ Arguments passed to the CLI will always take precedence over the CLI options con - `require`: extra modules to require before tests are run. Modules are required in the [worker processes](./01-writing-tests.md#process-isolation) - `timeout`: Timeouts in AVA behave differently than in other test frameworks. AVA resets a timer after each test, forcing tests to quit if no new test results were received within the specified timeout. This can be used to handle stalled tests. See our [timeout documentation](./07-test-timeouts.md) for more options. - `nodeArguments`: Configure Node.js arguments used to launch worker processes. +- `sortTestFiles`: A comparator function to sort test files with. Available only when using a `ava.config.*` file. See an example use case [here](recipes/splitting-tests-ci.md). Note that providing files on the CLI overrides the `files` option. diff --git a/docs/recipes/splitting-tests-ci.md b/docs/recipes/splitting-tests-ci.md new file mode 100644 index 000000000..95849d3dd --- /dev/null +++ b/docs/recipes/splitting-tests-ci.md @@ -0,0 +1,22 @@ +# Splitting tests in CI + +AVA automatically detects whether your CI environment supports parallel builds using [ci-parallel-vars](https://www.npmjs.com/package/ci-parallel-vars). When parallel builds support is detected, AVA sorts the all detected test files by name, and splits them into chunks. Each CI machine is assigned a chunk (subset) of the tests, and then each chunk is run in parallel. + +To better distribute the tests across the machines, you can configure a custom comparator function: + +**`ava.config.js`:** + +```js +import fs from 'node:fs'; + +// Assuming 'test-data.json' structure is: +// { +// 'tests/test1.js': { order: 1 }, +// 'tests/test2.js': { order: 0 } +// } +const testData = JSON.parse(fs.readFileSync('test-data.json', 'utf8')); + +export default { + sortTestFiles: (file1, file2) => testData[file1].order - testData[file2].order, +}; +``` diff --git a/lib/api.js b/lib/api.js index bf0cb2498..1ebe27ce5 100644 --- a/lib/api.js +++ b/lib/api.js @@ -177,13 +177,19 @@ export default class Api extends Emittery { const fileCount = selectedFiles.length; // The files must be in the same order across all runs, so sort them. - selectedFiles = selectedFiles.sort((a, b) => a.localeCompare(b, [], {numeric: true})); + const defaultComparator = (a, b) => a.localeCompare(b, [], {numeric: true}); + selectedFiles = selectedFiles.sort(this.options.sortTestFiles || defaultComparator); selectedFiles = chunkd(selectedFiles, currentIndex, totalRuns); const currentFileCount = selectedFiles.length; runStatus = new RunStatus(fileCount, {currentFileCount, currentIndex, totalRuns}, selectionInsights); } else { + // If a custom sorter was configured, use it. + if (this.options.sortTestFiles) { + selectedFiles = selectedFiles.sort(this.options.sortTestFiles); + } + runStatus = new RunStatus(selectedFiles.length, null, selectionInsights); } @@ -261,8 +267,8 @@ export default class Api extends Emittery { } const lineNumbers = getApplicableLineNumbers(globs.normalizeFileForMatching(apiOptions.projectDir, file), filter); - // Removing `providers` field because they cannot be transfered to the worker threads. - const {providers, ...forkOptions} = apiOptions; + // Removing `providers` and `sortTestFiles` fields because they cannot be transferred to the worker threads. + const {providers, sortTestFiles, ...forkOptions} = apiOptions; const options = { ...forkOptions, providerStates, diff --git a/lib/cli.js b/lib/cli.js index 48c87caa5..80d49a69f 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -321,6 +321,10 @@ export default async function loadCli() { // eslint-disable-line complexity exit('’sources’ has been removed. Use ’ignoredByWatcher’ to provide glob patterns of files that the watcher should ignore.'); } + if (Reflect.has(conf, 'sortTestFiles') && typeof conf.sortTestFiles !== 'function') { + exit('’sortTestFiles’ must be a comparator function.'); + } + let projectPackageObject; try { projectPackageObject = JSON.parse(fs.readFileSync(path.resolve(projectDir, 'package.json'))); @@ -413,6 +417,7 @@ export default async function loadCli() { // eslint-disable-line complexity moduleTypes, nodeArguments, parallelRuns, + sortTestFiles: conf.sortTestFiles, projectDir, providers, ranFromCli: true, diff --git a/readme.md b/readme.md index ab865265c..78b4a9cd9 100644 --- a/readme.md +++ b/readme.md @@ -139,15 +139,16 @@ We have a growing list of [common pitfalls](docs/08-common-pitfalls.md) you may ### Recipes -- [Shared workers](docs/recipes/shared-workers.md) - [Test setup](docs/recipes/test-setup.md) -- [Code coverage](docs/recipes/code-coverage.md) +- [TypeScript](docs/recipes/typescript.md) +- [Shared workers](docs/recipes/shared-workers.md) - [Watch mode](docs/recipes/watch-mode.md) -- [Endpoint testing](docs/recipes/endpoint-testing.md) - [When to use `t.plan()`](docs/recipes/when-to-use-plan.md) -- [Browser testing](docs/recipes/browser-testing.md) -- [TypeScript](docs/recipes/typescript.md) - [Passing arguments to your test files](docs/recipes/passing-arguments-to-your-test-files.md) +- [Splitting tests in CI](docs/recipes/splitting-tests-ci.md) +- [Code coverage](docs/recipes/code-coverage.md) +- [Endpoint testing](docs/recipes/endpoint-testing.md) +- [Browser testing](docs/recipes/browser-testing.md) - [Testing Vue.js components](docs/recipes/vue.md) - [Debugging tests with Chrome DevTools](docs/recipes/debugging-with-chrome-devtools.md) - [Debugging tests with VSCode](docs/recipes/debugging-with-vscode.md) diff --git a/test-tap/fixture/parallel-runs/custom-comparator/0-1.cjs b/test-tap/fixture/parallel-runs/custom-comparator/0-1.cjs new file mode 100644 index 000000000..5b1c05662 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/0-1.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '2'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/0-2.cjs b/test-tap/fixture/parallel-runs/custom-comparator/0-2.cjs new file mode 100644 index 000000000..5b1c05662 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/0-2.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '2'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/0-3.cjs b/test-tap/fixture/parallel-runs/custom-comparator/0-3.cjs new file mode 100644 index 000000000..5b1c05662 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/0-3.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '2'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/1-1.cjs b/test-tap/fixture/parallel-runs/custom-comparator/1-1.cjs new file mode 100644 index 000000000..ccb999168 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/1-1.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '1'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/1-2.cjs b/test-tap/fixture/parallel-runs/custom-comparator/1-2.cjs new file mode 100644 index 000000000..ccb999168 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/1-2.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '1'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/1-3.cjs b/test-tap/fixture/parallel-runs/custom-comparator/1-3.cjs new file mode 100644 index 000000000..ccb999168 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/1-3.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '1'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/2-1.cjs b/test-tap/fixture/parallel-runs/custom-comparator/2-1.cjs new file mode 100644 index 000000000..a40e62fb8 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/2-1.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '0'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/2-2.cjs b/test-tap/fixture/parallel-runs/custom-comparator/2-2.cjs new file mode 100644 index 000000000..a40e62fb8 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/2-2.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '0'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/2-3.cjs b/test-tap/fixture/parallel-runs/custom-comparator/2-3.cjs new file mode 100644 index 000000000..a40e62fb8 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/2-3.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '0'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/ava.config.js b/test-tap/fixture/parallel-runs/custom-comparator/ava.config.js new file mode 100644 index 000000000..935e14cf2 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/ava.config.js @@ -0,0 +1,5 @@ +export default { + files: ['*.cjs'], + // Descending order + sortTestFiles: (a, b) => b.localeCompare(a, [], {numeric: true}), +}; diff --git a/test-tap/fixture/parallel-runs/custom-comparator/package.json b/test-tap/fixture/parallel-runs/custom-comparator/package.json new file mode 100644 index 000000000..bedb411a9 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/test-tap/fixture/sort-tests/0.cjs b/test-tap/fixture/sort-tests/0.cjs new file mode 100644 index 000000000..1af77b598 --- /dev/null +++ b/test-tap/fixture/sort-tests/0.cjs @@ -0,0 +1,5 @@ +const test = require('../../../entrypoints/main.cjs'); + +test('should run third', t => { + t.pass(); +}); diff --git a/test-tap/fixture/sort-tests/1.cjs b/test-tap/fixture/sort-tests/1.cjs new file mode 100644 index 000000000..e1e0ff47b --- /dev/null +++ b/test-tap/fixture/sort-tests/1.cjs @@ -0,0 +1,5 @@ +const test = require('../../../entrypoints/main.cjs'); + +test('should run second', t => { + t.pass(); +}); diff --git a/test-tap/fixture/sort-tests/2.cjs b/test-tap/fixture/sort-tests/2.cjs new file mode 100644 index 000000000..9ad7df09f --- /dev/null +++ b/test-tap/fixture/sort-tests/2.cjs @@ -0,0 +1,5 @@ +const test = require('../../../entrypoints/main.cjs'); + +test('should run first', t => { + t.pass(); +}); diff --git a/test-tap/fixture/sort-tests/ava.config.js b/test-tap/fixture/sort-tests/ava.config.js new file mode 100644 index 000000000..58d098288 --- /dev/null +++ b/test-tap/fixture/sort-tests/ava.config.js @@ -0,0 +1,7 @@ +export default { + files: ['*.cjs'], + // Descending order + sortTestFiles: (a, b) => b.localeCompare(a, [], {numeric: true}), + concurrency: 1, + verbose: true, +}; diff --git a/test-tap/fixture/sort-tests/package.json b/test-tap/fixture/sort-tests/package.json new file mode 100644 index 000000000..bedb411a9 --- /dev/null +++ b/test-tap/fixture/sort-tests/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/test-tap/integration/assorted.js b/test-tap/integration/assorted.js index e866824c6..db292e8ac 100644 --- a/test-tap/integration/assorted.js +++ b/test-tap/integration/assorted.js @@ -152,3 +152,11 @@ test('load .js test files as ESM modules', t => { t.end(); }); }); + +test('uses sortTestFiles to sort test files', t => { + execCli([], {dirname: 'fixture/sort-tests'}, (error, stdout) => { + t.error(error); + t.match(stdout, /should run first[\s\S]+?should run second[\s\S]+?should run third/); + t.end(); + }); +}); diff --git a/test-tap/integration/parallel-runs.js b/test-tap/integration/parallel-runs.js index fe30e38c6..47f4c0781 100644 --- a/test-tap/integration/parallel-runs.js +++ b/test-tap/integration/parallel-runs.js @@ -43,3 +43,17 @@ test('fail when there are no files', t => { }, error => t.ok(error)); } }); + +test('correctly applies custom comparator', t => { + t.plan(3); + for (let i = 0; i < 3; i++) { + execCli([], { + dirname: 'fixture/parallel-runs/custom-comparator', + env: { + AVA_FORCE_CI: 'ci', + CI_NODE_INDEX: String(i), + CI_NODE_TOTAL: '3', + }, + }, error => t.error(error)); + } +});