From 6d61e62304d50318ca8df2bcb92f49ae97eb45c9 Mon Sep 17 00:00:00 2001 From: erezrokah Date: Fri, 11 Feb 2022 21:37:27 +0100 Subject: [PATCH 01/13] feat(parallel-runs): allow custom comparator --- lib/api.js | 5 ++++- lib/cli.js | 12 ++++++++++++ .../parallel-runs/custom-comparator/0-1.cjs | 5 +++++ .../parallel-runs/custom-comparator/0-2.cjs | 5 +++++ .../parallel-runs/custom-comparator/0-3.cjs | 5 +++++ .../parallel-runs/custom-comparator/1-1.cjs | 5 +++++ .../parallel-runs/custom-comparator/1-2.cjs | 5 +++++ .../parallel-runs/custom-comparator/1-3.cjs | 5 +++++ .../parallel-runs/custom-comparator/2-1.cjs | 5 +++++ .../parallel-runs/custom-comparator/2-2.cjs | 5 +++++ .../parallel-runs/custom-comparator/2-3.cjs | 5 +++++ .../parallel-runs/custom-comparator/ava.config.js | 5 +++++ .../parallel-runs/custom-comparator/package.json | 3 +++ test-tap/integration/parallel-runs.js | 14 ++++++++++++++ 14 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/0-1.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/0-2.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/0-3.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/1-1.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/1-2.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/1-3.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/2-1.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/2-2.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/2-3.cjs create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/ava.config.js create mode 100644 test-tap/fixture/parallel-runs/custom-comparator/package.json diff --git a/lib/api.js b/lib/api.js index bf0cb2498..2ee2bbae3 100644 --- a/lib/api.js +++ b/lib/api.js @@ -177,7 +177,10 @@ export default class Api extends Emittery { const fileCount = selectedFiles.length; // The files must be in the same order across all runs, so sort them. - selectedFiles = selectedFiles.sort((a, b) => a.localeCompare(b, [], {numeric: true})); + // The sorting function is a string representation of the function, so we need to deserialize it + // eslint-disable-next-line no-new-func + const comparator = new Function(`return ${this.options.parallelRunsComparator}`)(); + selectedFiles = selectedFiles.sort(comparator); selectedFiles = chunkd(selectedFiles, currentIndex, totalRuns); const currentFileCount = selectedFiles.length; diff --git a/lib/cli.js b/lib/cli.js index f1597950e..7f9b25845 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -382,11 +382,22 @@ export default async function loadCli() { // eslint-disable-line complexity } let parallelRuns = null; + let parallelRunsComparator = null; if (isCi && ciParallelVars) { const {index: currentIndex, total: totalRuns} = ciParallelVars; parallelRuns = {currentIndex, totalRuns}; } + if (parallelRuns) { + if (Reflect.has(conf, 'ciParallelRunsComparator') && typeof conf.ciParallelRunsComparator !== 'function') { + exit('ciParallelRunsComparator must be a comparator function.'); + } + + const defaultComparator = (a, b) => a.localeCompare(b, [], {numeric: true}); + // The function needs to be serializable to support worker threads + parallelRunsComparator = (conf.ciParallelRunsComparator || defaultComparator).toString(); + } + const match = combined.match === '' ? [] : arrify(combined.match); const input = debug ? debug.files : (argv.pattern || []); @@ -413,6 +424,7 @@ export default async function loadCli() { // eslint-disable-line complexity moduleTypes, nodeArguments, parallelRuns, + parallelRunsComparator, projectDir, providers, ranFromCli: true, diff --git a/test-tap/fixture/parallel-runs/custom-comparator/0-1.cjs b/test-tap/fixture/parallel-runs/custom-comparator/0-1.cjs new file mode 100644 index 000000000..5b1c05662 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/0-1.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '2'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/0-2.cjs b/test-tap/fixture/parallel-runs/custom-comparator/0-2.cjs new file mode 100644 index 000000000..5b1c05662 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/0-2.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '2'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/0-3.cjs b/test-tap/fixture/parallel-runs/custom-comparator/0-3.cjs new file mode 100644 index 000000000..5b1c05662 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/0-3.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '2'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/1-1.cjs b/test-tap/fixture/parallel-runs/custom-comparator/1-1.cjs new file mode 100644 index 000000000..ccb999168 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/1-1.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '1'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/1-2.cjs b/test-tap/fixture/parallel-runs/custom-comparator/1-2.cjs new file mode 100644 index 000000000..ccb999168 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/1-2.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '1'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/1-3.cjs b/test-tap/fixture/parallel-runs/custom-comparator/1-3.cjs new file mode 100644 index 000000000..ccb999168 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/1-3.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '1'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/2-1.cjs b/test-tap/fixture/parallel-runs/custom-comparator/2-1.cjs new file mode 100644 index 000000000..a40e62fb8 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/2-1.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '0'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/2-2.cjs b/test-tap/fixture/parallel-runs/custom-comparator/2-2.cjs new file mode 100644 index 000000000..a40e62fb8 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/2-2.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '0'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/2-3.cjs b/test-tap/fixture/parallel-runs/custom-comparator/2-3.cjs new file mode 100644 index 000000000..a40e62fb8 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/2-3.cjs @@ -0,0 +1,5 @@ +const test = require('../../../../entrypoints/main.cjs'); + +test('at expected index', t => { + t.is(process.env.CI_NODE_INDEX, '0'); +}); diff --git a/test-tap/fixture/parallel-runs/custom-comparator/ava.config.js b/test-tap/fixture/parallel-runs/custom-comparator/ava.config.js new file mode 100644 index 000000000..08fc05d97 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/ava.config.js @@ -0,0 +1,5 @@ +export default { + files: ['*.cjs'], + // Descending order + ciParallelRunsComparator: (a, b) => b.localeCompare(a, [], {numeric: true}), +}; diff --git a/test-tap/fixture/parallel-runs/custom-comparator/package.json b/test-tap/fixture/parallel-runs/custom-comparator/package.json new file mode 100644 index 000000000..bedb411a9 --- /dev/null +++ b/test-tap/fixture/parallel-runs/custom-comparator/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/test-tap/integration/parallel-runs.js b/test-tap/integration/parallel-runs.js index fe30e38c6..47f4c0781 100644 --- a/test-tap/integration/parallel-runs.js +++ b/test-tap/integration/parallel-runs.js @@ -43,3 +43,17 @@ test('fail when there are no files', t => { }, error => t.ok(error)); } }); + +test('correctly applies custom comparator', t => { + t.plan(3); + for (let i = 0; i < 3; i++) { + execCli([], { + dirname: 'fixture/parallel-runs/custom-comparator', + env: { + AVA_FORCE_CI: 'ci', + CI_NODE_INDEX: String(i), + CI_NODE_TOTAL: '3', + }, + }, error => t.error(error)); + } +}); From c9fd0388944fa80318f246e728c4e8cc69c4cf64 Mon Sep 17 00:00:00 2001 From: erezrokah Date: Sat, 12 Feb 2022 17:06:46 +0100 Subject: [PATCH 02/13] docs: add docs an recipe --- docs/06-configuration.md | 1 + docs/recipes/splitting-tests-ci.md | 25 +++++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 docs/recipes/splitting-tests-ci.md diff --git a/docs/06-configuration.md b/docs/06-configuration.md index 6140ddec3..c97b200f3 100644 --- a/docs/06-configuration.md +++ b/docs/06-configuration.md @@ -58,6 +58,7 @@ Arguments passed to the CLI will always take precedence over the CLI options con - `require`: extra modules to require before tests are run. Modules are required in the [worker processes](./01-writing-tests.md#process-isolation) - `timeout`: Timeouts in AVA behave differently than in other test frameworks. AVA resets a timer after each test, forcing tests to quit if no new test results were received within the specified timeout. This can be used to handle stalled tests. See our [timeout documentation](./07-test-timeouts.md) for more options. - `nodeArguments`: Configure Node.js arguments used to launch worker processes. +- `ciParallelRunsComparator`: A comparator function to use when [splitting tests across parallel CI builds](../readme.md#parallel-runs-in-ci). Available only when using a `ava.config.*` file. See example [here](recipes/parallel-runs-in-ci.md). Note that providing files on the CLI overrides the `files` option. diff --git a/docs/recipes/splitting-tests-ci.md b/docs/recipes/splitting-tests-ci.md new file mode 100644 index 000000000..6d57004b1 --- /dev/null +++ b/docs/recipes/splitting-tests-ci.md @@ -0,0 +1,25 @@ +# Splitting tests in CI + +AVA automatically detects whether your CI environment supports parallel builds using [ci-parallel-vars](https://www.npmjs.com/package/ci-parallel-vars). +When parallel builds support is detected, AVA sorts the all detected tests files, and splits them into chunks. +Each CI machine is assigned a chunk of the tests, and then each chunk is run in parallel. + +To better distribute the tests across the machines, you can configure a custom comparator function. +For example: + +**`ava.config.js`:** + +```js +import fs from 'node:fs'; + +// Assuming 'test-data.json' structure is: +// { +// 'tests/test1.js': { order: 1 }, +// 'tests/test2.js': { order: 0 } +// } +const testData = JSON.parse(fs.readFileSync('test-data.json', 'utf8')); + +export default { + ciParallelRunsComparator: (file1, file2) => testData[file1].order - testData[file2].order, +}; +``` From d915f787f113d644e4bcc89ad86321362be2b59a Mon Sep 17 00:00:00 2001 From: erezrokah Date: Sat, 12 Feb 2022 17:09:00 +0100 Subject: [PATCH 03/13] fix(docs): use correct link to recipe --- docs/06-configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/06-configuration.md b/docs/06-configuration.md index c97b200f3..6cedb3159 100644 --- a/docs/06-configuration.md +++ b/docs/06-configuration.md @@ -58,7 +58,7 @@ Arguments passed to the CLI will always take precedence over the CLI options con - `require`: extra modules to require before tests are run. Modules are required in the [worker processes](./01-writing-tests.md#process-isolation) - `timeout`: Timeouts in AVA behave differently than in other test frameworks. AVA resets a timer after each test, forcing tests to quit if no new test results were received within the specified timeout. This can be used to handle stalled tests. See our [timeout documentation](./07-test-timeouts.md) for more options. - `nodeArguments`: Configure Node.js arguments used to launch worker processes. -- `ciParallelRunsComparator`: A comparator function to use when [splitting tests across parallel CI builds](../readme.md#parallel-runs-in-ci). Available only when using a `ava.config.*` file. See example [here](recipes/parallel-runs-in-ci.md). +- `ciParallelRunsComparator`: A comparator function to use when [splitting tests across parallel CI builds](../readme.md#parallel-runs-in-ci). Available only when using a `ava.config.*` file. See example [here](recipes/splitting-tests-ci.md). Note that providing files on the CLI overrides the `files` option. From 3fe59a74b029b35103d834bf091cd0834e75abd3 Mon Sep 17 00:00:00 2001 From: erezrokah Date: Sat, 12 Feb 2022 17:10:35 +0100 Subject: [PATCH 04/13] fix(docs): 'tests -> test' typo --- docs/recipes/splitting-tests-ci.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/recipes/splitting-tests-ci.md b/docs/recipes/splitting-tests-ci.md index 6d57004b1..1483f2e3f 100644 --- a/docs/recipes/splitting-tests-ci.md +++ b/docs/recipes/splitting-tests-ci.md @@ -1,7 +1,7 @@ # Splitting tests in CI AVA automatically detects whether your CI environment supports parallel builds using [ci-parallel-vars](https://www.npmjs.com/package/ci-parallel-vars). -When parallel builds support is detected, AVA sorts the all detected tests files, and splits them into chunks. +When parallel builds support is detected, AVA sorts the all detected test files, and splits them into chunks. Each CI machine is assigned a chunk of the tests, and then each chunk is run in parallel. To better distribute the tests across the machines, you can configure a custom comparator function. From 3b0c1e28020f0ce59930bd2ce388e56da1f6b68d Mon Sep 17 00:00:00 2001 From: erezrokah Date: Sat, 12 Feb 2022 17:11:31 +0100 Subject: [PATCH 05/13] fix(docs): clarify default sorting method --- docs/recipes/splitting-tests-ci.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/recipes/splitting-tests-ci.md b/docs/recipes/splitting-tests-ci.md index 1483f2e3f..5713447df 100644 --- a/docs/recipes/splitting-tests-ci.md +++ b/docs/recipes/splitting-tests-ci.md @@ -1,7 +1,7 @@ # Splitting tests in CI AVA automatically detects whether your CI environment supports parallel builds using [ci-parallel-vars](https://www.npmjs.com/package/ci-parallel-vars). -When parallel builds support is detected, AVA sorts the all detected test files, and splits them into chunks. +When parallel builds support is detected, AVA sorts the all detected test files by name, and splits them into chunks. Each CI machine is assigned a chunk of the tests, and then each chunk is run in parallel. To better distribute the tests across the machines, you can configure a custom comparator function. From 028d632ab3baf44e3396a27a68bd1113b2e2f1ea Mon Sep 17 00:00:00 2001 From: Erez Rokah Date: Sun, 13 Feb 2022 20:13:27 +0100 Subject: [PATCH 06/13] Update docs/recipes/splitting-tests-ci.md Co-authored-by: Mark Wubben --- docs/recipes/splitting-tests-ci.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/recipes/splitting-tests-ci.md b/docs/recipes/splitting-tests-ci.md index 5713447df..fa2546236 100644 --- a/docs/recipes/splitting-tests-ci.md +++ b/docs/recipes/splitting-tests-ci.md @@ -4,8 +4,7 @@ AVA automatically detects whether your CI environment supports parallel builds u When parallel builds support is detected, AVA sorts the all detected test files by name, and splits them into chunks. Each CI machine is assigned a chunk of the tests, and then each chunk is run in parallel. -To better distribute the tests across the machines, you can configure a custom comparator function. -For example: +To better distribute the tests across the machines, you can configure a custom comparator function: **`ava.config.js`:** From 69069e3918604471a86488ddb083a0591b05e9db Mon Sep 17 00:00:00 2001 From: Erez Rokah Date: Sun, 13 Feb 2022 20:13:33 +0100 Subject: [PATCH 07/13] Update docs/recipes/splitting-tests-ci.md Co-authored-by: Mark Wubben --- docs/recipes/splitting-tests-ci.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/recipes/splitting-tests-ci.md b/docs/recipes/splitting-tests-ci.md index fa2546236..1bada6aa3 100644 --- a/docs/recipes/splitting-tests-ci.md +++ b/docs/recipes/splitting-tests-ci.md @@ -1,8 +1,6 @@ # Splitting tests in CI -AVA automatically detects whether your CI environment supports parallel builds using [ci-parallel-vars](https://www.npmjs.com/package/ci-parallel-vars). -When parallel builds support is detected, AVA sorts the all detected test files by name, and splits them into chunks. -Each CI machine is assigned a chunk of the tests, and then each chunk is run in parallel. +AVA automatically detects whether your CI environment supports parallel builds using [ci-parallel-vars](https://www.npmjs.com/package/ci-parallel-vars). When parallel builds support is detected, AVA sorts the all detected test files by name, and splits them into chunks. Each CI machine is assigned a chunk (subset) of the tests, and then each chunk is run in parallel. To better distribute the tests across the machines, you can configure a custom comparator function: From 07c3eff3a3c6c6857992b5c47942ebef8a8e034c Mon Sep 17 00:00:00 2001 From: erezrokah Date: Sun, 13 Feb 2022 20:30:24 +0100 Subject: [PATCH 08/13] fix: remove comparator serialization --- lib/api.js | 9 +++------ lib/cli.js | 3 +-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/api.js b/lib/api.js index 2ee2bbae3..a44657a8a 100644 --- a/lib/api.js +++ b/lib/api.js @@ -177,10 +177,7 @@ export default class Api extends Emittery { const fileCount = selectedFiles.length; // The files must be in the same order across all runs, so sort them. - // The sorting function is a string representation of the function, so we need to deserialize it - // eslint-disable-next-line no-new-func - const comparator = new Function(`return ${this.options.parallelRunsComparator}`)(); - selectedFiles = selectedFiles.sort(comparator); + selectedFiles = selectedFiles.sort(this.options.parallelRunsComparator); selectedFiles = chunkd(selectedFiles, currentIndex, totalRuns); const currentFileCount = selectedFiles.length; @@ -264,8 +261,8 @@ export default class Api extends Emittery { } const lineNumbers = getApplicableLineNumbers(globs.normalizeFileForMatching(apiOptions.projectDir, file), filter); - // Removing `providers` field because they cannot be transfered to the worker threads. - const {providers, ...forkOptions} = apiOptions; + // Removing `providers` and `parallelRunsComparator` fields because they cannot be transferred to the worker threads. + const {providers, parallelRunsComparator, ...forkOptions} = apiOptions; const options = { ...forkOptions, providerStates, diff --git a/lib/cli.js b/lib/cli.js index 7f9b25845..dfe67b749 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -394,8 +394,7 @@ export default async function loadCli() { // eslint-disable-line complexity } const defaultComparator = (a, b) => a.localeCompare(b, [], {numeric: true}); - // The function needs to be serializable to support worker threads - parallelRunsComparator = (conf.ciParallelRunsComparator || defaultComparator).toString(); + parallelRunsComparator = conf.ciParallelRunsComparator || defaultComparator; } const match = combined.match === '' ? [] : arrify(combined.match); From 9836cd7ae8b6083a2c0b4a2dc289809d5b10f6dd Mon Sep 17 00:00:00 2001 From: erezrokah Date: Tue, 15 Feb 2022 10:40:54 +0100 Subject: [PATCH 09/13] feat: make sorter available to all runs --- docs/06-configuration.md | 2 +- lib/api.js | 12 +++++++++--- lib/cli.js | 16 +++++----------- .../custom-comparator/ava.config.js | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/06-configuration.md b/docs/06-configuration.md index 6cedb3159..7985b078e 100644 --- a/docs/06-configuration.md +++ b/docs/06-configuration.md @@ -58,7 +58,7 @@ Arguments passed to the CLI will always take precedence over the CLI options con - `require`: extra modules to require before tests are run. Modules are required in the [worker processes](./01-writing-tests.md#process-isolation) - `timeout`: Timeouts in AVA behave differently than in other test frameworks. AVA resets a timer after each test, forcing tests to quit if no new test results were received within the specified timeout. This can be used to handle stalled tests. See our [timeout documentation](./07-test-timeouts.md) for more options. - `nodeArguments`: Configure Node.js arguments used to launch worker processes. -- `ciParallelRunsComparator`: A comparator function to use when [splitting tests across parallel CI builds](../readme.md#parallel-runs-in-ci). Available only when using a `ava.config.*` file. See example [here](recipes/splitting-tests-ci.md). +- `sortTestFiles`: A comparator function to sort test files with. Available only when using a `ava.config.*` file. See an example use case [here](recipes/splitting-tests-ci.md). Note that providing files on the CLI overrides the `files` option. diff --git a/lib/api.js b/lib/api.js index a44657a8a..1ebe27ce5 100644 --- a/lib/api.js +++ b/lib/api.js @@ -177,13 +177,19 @@ export default class Api extends Emittery { const fileCount = selectedFiles.length; // The files must be in the same order across all runs, so sort them. - selectedFiles = selectedFiles.sort(this.options.parallelRunsComparator); + const defaultComparator = (a, b) => a.localeCompare(b, [], {numeric: true}); + selectedFiles = selectedFiles.sort(this.options.sortTestFiles || defaultComparator); selectedFiles = chunkd(selectedFiles, currentIndex, totalRuns); const currentFileCount = selectedFiles.length; runStatus = new RunStatus(fileCount, {currentFileCount, currentIndex, totalRuns}, selectionInsights); } else { + // If a custom sorter was configured, use it. + if (this.options.sortTestFiles) { + selectedFiles = selectedFiles.sort(this.options.sortTestFiles); + } + runStatus = new RunStatus(selectedFiles.length, null, selectionInsights); } @@ -261,8 +267,8 @@ export default class Api extends Emittery { } const lineNumbers = getApplicableLineNumbers(globs.normalizeFileForMatching(apiOptions.projectDir, file), filter); - // Removing `providers` and `parallelRunsComparator` fields because they cannot be transferred to the worker threads. - const {providers, parallelRunsComparator, ...forkOptions} = apiOptions; + // Removing `providers` and `sortTestFiles` fields because they cannot be transferred to the worker threads. + const {providers, sortTestFiles, ...forkOptions} = apiOptions; const options = { ...forkOptions, providerStates, diff --git a/lib/cli.js b/lib/cli.js index dfe67b749..c0464a057 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -321,6 +321,10 @@ export default async function loadCli() { // eslint-disable-line complexity exit('’sources’ has been removed. Use ’ignoredByWatcher’ to provide glob patterns of files that the watcher should ignore.'); } + if (Reflect.has(conf, 'sortTestFiles') && typeof conf.sortTestFiles !== 'function') { + exit('’sortTestFiles’ must be a comparator function.'); + } + let projectPackageObject; try { projectPackageObject = JSON.parse(fs.readFileSync(path.resolve(projectDir, 'package.json'))); @@ -382,21 +386,11 @@ export default async function loadCli() { // eslint-disable-line complexity } let parallelRuns = null; - let parallelRunsComparator = null; if (isCi && ciParallelVars) { const {index: currentIndex, total: totalRuns} = ciParallelVars; parallelRuns = {currentIndex, totalRuns}; } - if (parallelRuns) { - if (Reflect.has(conf, 'ciParallelRunsComparator') && typeof conf.ciParallelRunsComparator !== 'function') { - exit('ciParallelRunsComparator must be a comparator function.'); - } - - const defaultComparator = (a, b) => a.localeCompare(b, [], {numeric: true}); - parallelRunsComparator = conf.ciParallelRunsComparator || defaultComparator; - } - const match = combined.match === '' ? [] : arrify(combined.match); const input = debug ? debug.files : (argv.pattern || []); @@ -423,7 +417,7 @@ export default async function loadCli() { // eslint-disable-line complexity moduleTypes, nodeArguments, parallelRuns, - parallelRunsComparator, + sortTestFiles: conf.sortTestFiles, projectDir, providers, ranFromCli: true, diff --git a/test-tap/fixture/parallel-runs/custom-comparator/ava.config.js b/test-tap/fixture/parallel-runs/custom-comparator/ava.config.js index 08fc05d97..935e14cf2 100644 --- a/test-tap/fixture/parallel-runs/custom-comparator/ava.config.js +++ b/test-tap/fixture/parallel-runs/custom-comparator/ava.config.js @@ -1,5 +1,5 @@ export default { files: ['*.cjs'], // Descending order - ciParallelRunsComparator: (a, b) => b.localeCompare(a, [], {numeric: true}), + sortTestFiles: (a, b) => b.localeCompare(a, [], {numeric: true}), }; From 44cdb863c1efdde52e4fec3fcc62fc1c6dced5b2 Mon Sep 17 00:00:00 2001 From: erezrokah Date: Tue, 15 Feb 2022 11:26:38 +0100 Subject: [PATCH 10/13] test: add non-ci sorting test case --- test-tap/fixture/sort-tests/0.cjs | 5 +++++ test-tap/fixture/sort-tests/1.cjs | 5 +++++ test-tap/fixture/sort-tests/2.cjs | 5 +++++ test-tap/fixture/sort-tests/ava.config.js | 7 +++++++ test-tap/fixture/sort-tests/package.json | 3 +++ test-tap/integration/assorted.js | 8 ++++++++ 6 files changed, 33 insertions(+) create mode 100644 test-tap/fixture/sort-tests/0.cjs create mode 100644 test-tap/fixture/sort-tests/1.cjs create mode 100644 test-tap/fixture/sort-tests/2.cjs create mode 100644 test-tap/fixture/sort-tests/ava.config.js create mode 100644 test-tap/fixture/sort-tests/package.json diff --git a/test-tap/fixture/sort-tests/0.cjs b/test-tap/fixture/sort-tests/0.cjs new file mode 100644 index 000000000..1af77b598 --- /dev/null +++ b/test-tap/fixture/sort-tests/0.cjs @@ -0,0 +1,5 @@ +const test = require('../../../entrypoints/main.cjs'); + +test('should run third', t => { + t.pass(); +}); diff --git a/test-tap/fixture/sort-tests/1.cjs b/test-tap/fixture/sort-tests/1.cjs new file mode 100644 index 000000000..e1e0ff47b --- /dev/null +++ b/test-tap/fixture/sort-tests/1.cjs @@ -0,0 +1,5 @@ +const test = require('../../../entrypoints/main.cjs'); + +test('should run second', t => { + t.pass(); +}); diff --git a/test-tap/fixture/sort-tests/2.cjs b/test-tap/fixture/sort-tests/2.cjs new file mode 100644 index 000000000..9ad7df09f --- /dev/null +++ b/test-tap/fixture/sort-tests/2.cjs @@ -0,0 +1,5 @@ +const test = require('../../../entrypoints/main.cjs'); + +test('should run first', t => { + t.pass(); +}); diff --git a/test-tap/fixture/sort-tests/ava.config.js b/test-tap/fixture/sort-tests/ava.config.js new file mode 100644 index 000000000..58d098288 --- /dev/null +++ b/test-tap/fixture/sort-tests/ava.config.js @@ -0,0 +1,7 @@ +export default { + files: ['*.cjs'], + // Descending order + sortTestFiles: (a, b) => b.localeCompare(a, [], {numeric: true}), + concurrency: 1, + verbose: true, +}; diff --git a/test-tap/fixture/sort-tests/package.json b/test-tap/fixture/sort-tests/package.json new file mode 100644 index 000000000..bedb411a9 --- /dev/null +++ b/test-tap/fixture/sort-tests/package.json @@ -0,0 +1,3 @@ +{ + "type": "module" +} diff --git a/test-tap/integration/assorted.js b/test-tap/integration/assorted.js index e866824c6..db292e8ac 100644 --- a/test-tap/integration/assorted.js +++ b/test-tap/integration/assorted.js @@ -152,3 +152,11 @@ test('load .js test files as ESM modules', t => { t.end(); }); }); + +test('uses sortTestFiles to sort test files', t => { + execCli([], {dirname: 'fixture/sort-tests'}, (error, stdout) => { + t.error(error); + t.match(stdout, /should run first[\s\S]+?should run second[\s\S]+?should run third/); + t.end(); + }); +}); From 1ba1feb980313960325b46761a7acbc1e0cf5b74 Mon Sep 17 00:00:00 2001 From: Mark Wubben Date: Tue, 1 Mar 2022 19:02:07 +0100 Subject: [PATCH 11/13] Fix function name and indentation in recipe --- docs/recipes/splitting-tests-ci.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/recipes/splitting-tests-ci.md b/docs/recipes/splitting-tests-ci.md index 1bada6aa3..95849d3dd 100644 --- a/docs/recipes/splitting-tests-ci.md +++ b/docs/recipes/splitting-tests-ci.md @@ -11,12 +11,12 @@ import fs from 'node:fs'; // Assuming 'test-data.json' structure is: // { -// 'tests/test1.js': { order: 1 }, -// 'tests/test2.js': { order: 0 } +// 'tests/test1.js': { order: 1 }, +// 'tests/test2.js': { order: 0 } // } const testData = JSON.parse(fs.readFileSync('test-data.json', 'utf8')); export default { - ciParallelRunsComparator: (file1, file2) => testData[file1].order - testData[file2].order, + sortTestFiles: (file1, file2) => testData[file1].order - testData[file2].order, }; ``` From 2bb174cf16dd81926420a57063816bec42419503 Mon Sep 17 00:00:00 2001 From: Mark Wubben Date: Tue, 1 Mar 2022 19:02:23 +0100 Subject: [PATCH 12/13] Reorder recipe links --- readme.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/readme.md b/readme.md index 682315ce0..e2c0e6ad9 100644 --- a/readme.md +++ b/readme.md @@ -139,15 +139,15 @@ We have a growing list of [common pitfalls](docs/08-common-pitfalls.md) you may ### Recipes -- [Shared workers](docs/recipes/shared-workers.md) - [Test setup](docs/recipes/test-setup.md) -- [Code coverage](docs/recipes/code-coverage.md) +- [TypeScript](docs/recipes/typescript.md) +- [Shared workers](docs/recipes/shared-workers.md) - [Watch mode](docs/recipes/watch-mode.md) -- [Endpoint testing](docs/recipes/endpoint-testing.md) - [When to use `t.plan()`](docs/recipes/when-to-use-plan.md) -- [Browser testing](docs/recipes/browser-testing.md) -- [TypeScript](docs/recipes/typescript.md) - [Passing arguments to your test files](docs/recipes/passing-arguments-to-your-test-files.md) +- [Code coverage](docs/recipes/code-coverage.md) +- [Endpoint testing](docs/recipes/endpoint-testing.md) +- [Browser testing](docs/recipes/browser-testing.md) - [Testing Vue.js components](docs/recipes/vue.md) - [Debugging tests with Chrome DevTools](docs/recipes/debugging-with-chrome-devtools.md) - [Debugging tests with VSCode](docs/recipes/debugging-with-vscode.md) From 4aacf96508301cf4afc53b9d3fac4294429e7e62 Mon Sep 17 00:00:00 2001 From: Mark Wubben Date: Tue, 1 Mar 2022 19:02:30 +0100 Subject: [PATCH 13/13] Link to new recipe --- readme.md | 1 + 1 file changed, 1 insertion(+) diff --git a/readme.md b/readme.md index e2c0e6ad9..2adacc3e2 100644 --- a/readme.md +++ b/readme.md @@ -145,6 +145,7 @@ We have a growing list of [common pitfalls](docs/08-common-pitfalls.md) you may - [Watch mode](docs/recipes/watch-mode.md) - [When to use `t.plan()`](docs/recipes/when-to-use-plan.md) - [Passing arguments to your test files](docs/recipes/passing-arguments-to-your-test-files.md) +- [Splitting tests in CI](docs/recipes/splitting-tests-ci.md) - [Code coverage](docs/recipes/code-coverage.md) - [Endpoint testing](docs/recipes/endpoint-testing.md) - [Browser testing](docs/recipes/browser-testing.md)