Allow customized sorting of test files prior to execution

docs/06-configuration.md

@@ -58,6 +58,7 @@ Arguments passed to the CLI will always take precedence over the CLI options con
 - `require`: extra modules to require before tests are run. Modules are required in the [worker processes](./01-writing-tests.md#process-isolation)
 - `timeout`: Timeouts in AVA behave differently than in other test frameworks. AVA resets a timer after each test, forcing tests to quit if no new test results were received within the specified timeout. This can be used to handle stalled tests. See our [timeout documentation](./07-test-timeouts.md) for more options.
 - `nodeArguments`: Configure Node.js arguments used to launch worker processes.
+- `ciParallelRunsComparator`: A comparator function to use when [splitting tests across parallel CI builds](../readme.md#parallel-runs-in-ci). Available only when using a `ava.config.*` file. See example [here](recipes/splitting-tests-ci.md).
 
 Note that providing files on the CLI overrides the `files` option.
 

docs/recipes/splitting-tests-ci.md

@@ -0,0 +1,25 @@
+# Splitting tests in CI
+
+AVA automatically detects whether your CI environment supports parallel builds using [ci-parallel-vars](https://www.npmjs.com/package/ci-parallel-vars).
+When parallel builds support is detected, AVA sorts the all detected test files by name, and splits them into chunks.
+Each CI machine is assigned a chunk of the tests, and then each chunk is run in parallel.
+
+To better distribute the tests across the machines, you can configure a custom comparator function.
+For example:
+
+**`ava.config.js`:**
+
+```js
+import fs from 'node:fs';
+
+// Assuming 'test-data.json' structure is:
+// {
+//    'tests/test1.js': { order: 1 },
+//    'tests/test2.js': { order: 0 }
+// }
+const testData = JSON.parse(fs.readFileSync('test-data.json', 'utf8'));
+
+export default {
+    ciParallelRunsComparator: (file1, file2) => testData[file1].order - testData[file2].order,
+};
+```

lib/api.js

@@ -177,7 +177,10 @@ export default class Api extends Emittery {
 				const fileCount = selectedFiles.length;
 
 				// The files must be in the same order across all runs, so sort them.
-				selectedFiles = selectedFiles.sort((a, b) => a.localeCompare(b, [], {numeric: true}));
+				// The sorting function is a string representation of the function, so we need to deserialize it
+				// eslint-disable-next-line no-new-func
+				const comparator = new Function(`return ${this.options.parallelRunsComparator}`)();
+				selectedFiles = selectedFiles.sort(comparator);
 				selectedFiles = chunkd(selectedFiles, currentIndex, totalRuns);
 
 				const currentFileCount = selectedFiles.length;

lib/cli.js

@@ -382,11 +382,22 @@ export default async function loadCli() { // eslint-disable-line complexity
 	}
 
 	let parallelRuns = null;
+	let parallelRunsComparator = null;
 	if (isCi && ciParallelVars) {
 		const {index: currentIndex, total: totalRuns} = ciParallelVars;
 		parallelRuns = {currentIndex, totalRuns};
 	}
 
+	if (parallelRuns) {
+		if (Reflect.has(conf, 'ciParallelRunsComparator') && typeof conf.ciParallelRunsComparator !== 'function') {
+			exit('ciParallelRunsComparator must be a comparator function.');
+		}
+
+		const defaultComparator = (a, b) => a.localeCompare(b, [], {numeric: true});
+		// The function needs to be serializable to support worker threads
+		parallelRunsComparator = (conf.ciParallelRunsComparator || defaultComparator).toString();
+	}
+
 	const match = combined.match === '' ? [] : arrify(combined.match);
 
 	const input = debug ? debug.files : (argv.pattern || []);
@@ -413,6 +424,7 @@ export default async function loadCli() { // eslint-disable-line complexity
 		moduleTypes,
 		nodeArguments,
 		parallelRuns,
+		parallelRunsComparator,
 		projectDir,
 		providers,
 		ranFromCli: true,

test-tap/fixture/parallel-runs/custom-comparator/0-1.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '2');
+});

test-tap/fixture/parallel-runs/custom-comparator/0-2.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '2');
+});

test-tap/fixture/parallel-runs/custom-comparator/0-3.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '2');
+});

test-tap/fixture/parallel-runs/custom-comparator/1-1.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '1');
+});

test-tap/fixture/parallel-runs/custom-comparator/1-2.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '1');
+});

test-tap/fixture/parallel-runs/custom-comparator/1-3.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '1');
+});

test-tap/fixture/parallel-runs/custom-comparator/2-1.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '0');
+});

test-tap/fixture/parallel-runs/custom-comparator/2-2.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '0');
+});

test-tap/fixture/parallel-runs/custom-comparator/2-3.cjs

@@ -0,0 +1,5 @@
+const test = require('../../../../entrypoints/main.cjs');
+
+test('at expected index', t => {
+	t.is(process.env.CI_NODE_INDEX, '0');
+});

test-tap/fixture/parallel-runs/custom-comparator/ava.config.js

@@ -0,0 +1,5 @@
+export default {
+	files: ['*.cjs'],
+	// Descending order
+	ciParallelRunsComparator: (a, b) => b.localeCompare(a, [], {numeric: true}),
+};

test-tap/fixture/parallel-runs/custom-comparator/package.json

@@ -0,0 +1,3 @@
+{
+	"type": "module"
+}

test-tap/integration/parallel-runs.js

@@ -43,3 +43,17 @@ test('fail when there are no files', t => {
 		}, error => t.ok(error));
 	}
 });
+
+test('correctly applies custom comparator', t => {
+	t.plan(3);
+	for (let i = 0; i < 3; i++) {
+		execCli([], {
+			dirname: 'fixture/parallel-runs/custom-comparator',
+			env: {
+				AVA_FORCE_CI: 'ci',
+				CI_NODE_INDEX: String(i),
+				CI_NODE_TOTAL: '3',
+			},
+		}, error => t.error(error));
+	}
+});