Allow customized sorting of test files prior to execution

docs/06-configuration.md

@@ -58,6 +58,7 @@ Arguments passed to the CLI will always take precedence over the CLI options con
 - `require`: extra modules to require before tests are run. Modules are required in the [worker processes](./01-writing-tests.md#process-isolation)
 - `timeout`: Timeouts in AVA behave differently than in other test frameworks. AVA resets a timer after each test, forcing tests to quit if no new test results were received within the specified timeout. This can be used to handle stalled tests. See our [timeout documentation](./07-test-timeouts.md) for more options.
 - `nodeArguments`: Configure Node.js arguments used to launch worker processes.
+- `sortTestFiles`: A comparator function to sort test files with. Available only when using a `ava.config.*` file. See an example use case [here](recipes/splitting-tests-ci.md).
 
 Note that providing files on the CLI overrides the `files` option.
 

docs/recipes/splitting-tests-ci.md

@@ -0,0 +1,22 @@
+# Splitting tests in CI
+
+AVA automatically detects whether your CI environment supports parallel builds using [ci-parallel-vars](https://www.npmjs.com/package/ci-parallel-vars). When parallel builds support is detected, AVA sorts the all detected test files by name, and splits them into chunks. Each CI machine is assigned a chunk (subset) of the tests, and then each chunk is run in parallel.
+
+To better distribute the tests across the machines, you can configure a custom comparator function:
+
+**`ava.config.js`:**
+
+```js
+import fs from 'node:fs';
+
+// Assuming 'test-data.json' structure is:
+// {
+// 	'tests/test1.js': { order: 1 },
+// 	'tests/test2.js': { order: 0 }
+// }
+const testData = JSON.parse(fs.readFileSync('test-data.json', 'utf8'));
+
+export default {
+	sortTestFiles: (file1, file2) => testData[file1].order - testData[file2].order,
+};
+```

lib/api.js

@@ -177,13 +177,19 @@ export default class Api extends Emittery {
 				const fileCount = selectedFiles.length;
 
 				// The files must be in the same order across all runs, so sort them.
-				selectedFiles = selectedFiles.sort((a, b) => a.localeCompare(b, [], {numeric: true}));
+				const defaultComparator = (a, b) => a.localeCompare(b, [], {numeric: true});
+				selectedFiles = selectedFiles.sort(this.options.sortTestFiles || defaultComparator);
 				selectedFiles = chunkd(selectedFiles, currentIndex, totalRuns);
 
 				const currentFileCount = selectedFiles.length;
 
 				runStatus = new RunStatus(fileCount, {currentFileCount, currentIndex, totalRuns}, selectionInsights);
 			} else {
+				// If a custom sorter was configured, use it.
+				if (this.options.sortTestFiles) {
+					selectedFiles = selectedFiles.sort(this.options.sortTestFiles);
+				}
+
 				runStatus = new RunStatus(selectedFiles.length, null, selectionInsights);
 			}
 
@@ -261,8 +267,8 @@ export default class Api extends Emittery {
 				}
 
 				const lineNumbers = getApplicableLineNumbers(globs.normalizeFileForMatching(apiOptions.projectDir, file), filter);
-				// Removing `providers` field because they cannot be transfered to the worker threads.
-				const {providers, ...forkOptions} = apiOptions;
+				// Removing `providers` and `sortTestFiles` fields because they cannot be transferred to the worker threads.
+				const {providers, sortTestFiles, ...forkOptions} = apiOptions;
 				const options = {
 					...forkOptions,
 					providerStates,

lib/cli.js

@@ -321,6 +321,10 @@ export default async function loadCli() { // eslint-disable-line complexity
 		exit('’sources’ has been removed. Use ’ignoredByWatcher’ to provide glob patterns of files that the watcher should ignore.');
 	}
 
+	if (Reflect.has(conf, 'sortTestFiles') && typeof conf.sortTestFiles !== 'function') {
+		exit('’sortTestFiles’ must be a comparator function.');
+	}
+
 	let projectPackageObject;
 	try {
 		projectPackageObject = JSON.parse(fs.readFileSync(path.resolve(projectDir, 'package.json')));
@@ -413,6 +417,7 @@ export default async function loadCli() { // eslint-disable-line complexity
 		moduleTypes,
 		nodeArguments,
 		parallelRuns,
+		sortTestFiles: conf.sortTestFiles,
 		projectDir,
 		providers,
 		ranFromCli: true,

readme.md

@@ -139,15 +139,16 @@ We have a growing list of [common pitfalls](docs/08-common-pitfalls.md) you may
 
 ### Recipes
 
-- [Shared workers](docs/recipes/shared-workers.md)
 - [Test setup](docs/recipes/test-setup.md)
-- [Code coverage](docs/recipes/code-coverage.md)
+- [TypeScript](docs/recipes/typescript.md)
+- [Shared workers](docs/recipes/shared-workers.md)
 - [Watch mode](docs/recipes/watch-mode.md)
-- [Endpoint testing](docs/recipes/endpoint-testing.md)
 - [When to use `t.plan()`](docs/recipes/when-to-use-plan.md)
-- [Browser testing](docs/recipes/browser-testing.md)
-- [TypeScript](docs/recipes/typescript.md)
 - [Passing arguments to your test files](docs/recipes/passing-arguments-to-your-test-files.md)
+- [Splitting tests in CI](docs/recipes/splitting-tests-ci.md)
+- [Code coverage](docs/recipes/code-coverage.md)
+- [Endpoint testing](docs/recipes/endpoint-testing.md)
+- [Browser testing](docs/recipes/browser-testing.md)
 - [Testing Vue.js components](docs/recipes/vue.md)
 - [Debugging tests with Chrome DevTools](docs/recipes/debugging-with-chrome-devtools.md)
 - [Debugging tests with VSCode](docs/recipes/debugging-with-vscode.md)

test-tap/fixture/parallel-runs/custom-comparator/0-1.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '2');
+});

test-tap/fixture/parallel-runs/custom-comparator/0-2.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '2');
+});

test-tap/fixture/parallel-runs/custom-comparator/0-3.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '2');
+});

test-tap/fixture/parallel-runs/custom-comparator/1-1.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '1');
+});

test-tap/fixture/parallel-runs/custom-comparator/1-2.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '1');
+});

test-tap/fixture/parallel-runs/custom-comparator/1-3.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '1');
+});

test-tap/fixture/parallel-runs/custom-comparator/2-1.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '0');
+});

test-tap/fixture/parallel-runs/custom-comparator/2-2.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '0');
+});

test-tap/fixture/parallel-runs/custom-comparator/2-3.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '0');
+});

test-tap/fixture/parallel-runs/custom-comparator/ava.config.js

@@ -0,0 +1,5 @@
+export default {
+	files: ['*.cjs'],
+	// Descending order
+	sortTestFiles: (a, b) => b.localeCompare(a, [], {numeric: true}),
+};

test-tap/fixture/parallel-runs/custom-comparator/package.json

@@ -0,0 +1,3 @@
+{
+	"type": "module"
+}

test-tap/fixture/sort-tests/0.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../entrypoints/main.cjs');
+
+test('should run third', t => {
+	t.pass();
+});

test-tap/fixture/sort-tests/1.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../entrypoints/main.cjs');
+
+test('should run second', t => {
+	t.pass();
+});

test-tap/fixture/sort-tests/2.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../entrypoints/main.cjs');
+
+test('should run first', t => {
+	t.pass();
+});

test-tap/fixture/sort-tests/ava.config.js

@@ -0,0 +1,7 @@
+export default {
+	files: ['*.cjs'],
+	// Descending order
+	sortTestFiles: (a, b) => b.localeCompare(a, [], {numeric: true}),
+	concurrency: 1,
+	verbose: true,
+};

test-tap/fixture/sort-tests/package.json

@@ -0,0 +1,3 @@
+{
+	"type": "module"
+}

test-tap/integration/assorted.js

@@ -152,3 +152,11 @@ test('load .js test files as ESM modules', t => {
 		t.end();
 	});
 });
+
+test('uses sortTestFiles to sort test files', t => {
+	execCli([], {dirname: 'fixture/sort-tests'}, (error, stdout) => {
+		t.error(error);
+		t.match(stdout, /should run first[\s\S]+?should run second[\s\S]+?should run third/);
+		t.end();
+	});
+});

test-tap/integration/parallel-runs.js

@@ -43,3 +43,17 @@ test('fail when there are no files', t => {
 		}, error => t.ok(error));
 	}
 });
+
+test('correctly applies custom comparator', t => {
+	t.plan(3);
+	for (let i = 0; i < 3; i++) {
+		execCli([], {
+			dirname: 'fixture/parallel-runs/custom-comparator',
+			env: {
+				AVA_FORCE_CI: 'ci',
+				CI_NODE_INDEX: String(i),
+				CI_NODE_TOTAL: '3',
+			},
+		}, error => t.error(error));
+	}
+});