From 7014ca44d409e05d84a18eeff287f021907d3992 Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Mon, 13 Jun 2022 20:19:05 +0300 Subject: [PATCH 1/4] feat: add --shard command --- docs/guide/cli.md | 29 +++++++++++++++-- packages/vitest/src/node/cli.ts | 1 + packages/vitest/src/node/config.ts | 27 ++++++++++++++++ packages/vitest/src/node/pool.ts | 21 ++++++++++++ packages/vitest/src/types/config.ts | 14 +++++++- pnpm-lock.yaml | 8 +++++ test/shard/package.json | 11 +++++++ test/shard/shard-test.test.ts | 50 +++++++++++++++++++++++++++++ test/shard/test/1.test.js | 3 ++ test/shard/test/2.test.js | 3 ++ test/shard/test/3.test.js | 3 ++ 11 files changed, 166 insertions(+), 4 deletions(-) create mode 100644 test/shard/package.json create mode 100644 test/shard/shard-test.test.ts create mode 100644 test/shard/test/1.test.js create mode 100644 test/shard/test/2.test.js create mode 100644 test/shard/test/3.test.js diff --git a/docs/guide/cli.md b/docs/guide/cli.md index 2ff00fd5fa41..0699d1535fea 100644 --- a/docs/guide/cli.md +++ b/docs/guide/cli.md @@ -66,7 +66,8 @@ vitest related /src/index.ts /src/hello-world.js | `--environment ` | Runner environment (default: `node`) | | `--passWithNoTests` | Pass when no tests found | | `--allowOnly` | Allow tests and suites that are marked as `only` (default: false in CI, true otherwise) | -| `--changed [since]` | Run tests that are affected by the changed files (default: false). See [docs](#changed) +| `--changed [since]` | Run tests that are affected by the changed files (default: false). See [docs](#changed) | +| `--shard ` | Execute tests in a specified shard | | `-h, --help` | Display available CLI options | ### changed @@ -74,6 +75,28 @@ vitest related /src/index.ts /src/hello-world.js - **Type**: `boolean | string` - **Default**: false -Run tests only against changed files. If no value is provided, it will run tests against uncommitted changes (including staged and unstaged). + Run tests only against changed files. If no value is provided, it will run tests against uncommitted changes (including staged and unstaged). -To run tests against changes made in the last commit, you can use `--changed HEAD~1`. You can also pass commit hash or branch name. + To run tests against changes made in the last commit, you can use `--changed HEAD~1`. You can also pass commit hash or branch name. + +### shard + +- **Type**: `string` +- **Default**: disabled + + Test suite shard to execute in a format of ``/``, where + + - `count` is a positive integer, count of divided parts + - `index` is a positive integer, index of divided part + + This command will divide all tests into `count` equal parts, and will run only those that happen to be in an `index` part. For example, to split your tests suite into three parts, use this: + + ```sh + vitest run --shard=1/3 + vitest run --shard=2/3 + vitest run --shard=3/3 + ``` + +:::warning +You cannot use this option with `--watch` enabled (enabled in dev by default). +::: diff --git a/packages/vitest/src/node/cli.ts b/packages/vitest/src/node/cli.ts index d258d43ed042..496f4ce88f28 100644 --- a/packages/vitest/src/node/cli.ts +++ b/packages/vitest/src/node/cli.ts @@ -33,6 +33,7 @@ cli .option('--environment ', 'runner environment (default: node)') .option('--passWithNoTests', 'pass when no tests found') .option('--allowOnly', 'Allow tests and suites that are marked as only (default: !process.env.CI)') + .option('--shard ', 'Test suite shard to execute in a format of /') .option('--changed [since]', 'Run tests that are affected by the changed files (default: false)') .help() diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index ecacba036a53..b1e055a6cb86 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -56,6 +56,16 @@ export function resolveApiConfig( return api } +const configError = (error: string): never => { + console.warn( + c.yellow( + `${c.inverse(c.red(' VITEST '))} ${error}\n`, + ), + ) + + process.exit(1) +} + export function resolveConfig( options: UserConfig, viteConfig: ResolvedViteConfig, @@ -90,6 +100,23 @@ export function resolveConfig( resolved.coverage = resolveC8Options(options.coverage || {}, resolved.root) + if (options.shard) { + if (resolved.watch) + configError('You cannot use --shard option with enabled watch') + + const [indexString, countString] = options.shard.split('/') + const index = Math.abs(parseInt(indexString, 10)) + const count = Math.abs(parseInt(countString, 10)) + + if (isNaN(count) || count <= 0) + configError('--shard must be a positive number') + + if (isNaN(index) || index <= 0 || index > count) + configError('--shard must be a positive number less then ') + + resolved.shard = { index, count } + } + resolved.deps = resolved.deps || {} // vitenode will try to import such file with native node, // but then our mocker will not work properly diff --git a/packages/vitest/src/node/pool.ts b/packages/vitest/src/node/pool.ts index 0843fe9f47a4..6fa99d7c9e79 100644 --- a/packages/vitest/src/node/pool.ts +++ b/packages/vitest/src/node/pool.ts @@ -1,6 +1,7 @@ import { MessageChannel } from 'worker_threads' import { pathToFileURL } from 'url' import { cpus } from 'os' +import crypto from 'crypto' import { resolve } from 'pathe' import type { Options as TinypoolOptions } from 'tinypool' import { Tinypool } from 'tinypool' @@ -88,6 +89,26 @@ export function createPool(ctx: Vitest): WorkerPool { } return async (files, invalidates) => { + if (config.shard) { + const { index, count } = config.shard + const shardSize = Math.ceil(files.length / count) + const shardStart = shardSize * (index - 1) + const shardEnd = shardSize * index + files = files + .map((file) => { + return { + file, + hash: crypto + .createHash('sha1') + .update(file) + .digest('hex'), + } + }) + .sort((a, b) => (a.hash < b.hash ? -1 : a.hash > b.hash ? 1 : 0)) + .slice(shardStart, shardEnd) + .map(({ file }) => file) + } + if (!ctx.config.threads) { await runFiles(files) } diff --git a/packages/vitest/src/types/config.ts b/packages/vitest/src/types/config.ts index 1c133f281283..faa3af1eb275 100644 --- a/packages/vitest/src/types/config.ts +++ b/packages/vitest/src/types/config.ts @@ -380,9 +380,17 @@ export interface UserConfig extends InlineConfig { * @default false */ changed?: boolean | string + + /** + * Test suite shard to execute in a format of /. + * Will divide tests into a `count` numbers, and run only the `indexed` part. + * Cannot be used with enabled watch. + * @example --shard=2/3 + */ + shard?: string } -export interface ResolvedConfig extends Omit, 'config' | 'filters' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath'> { +export interface ResolvedConfig extends Omit, 'config' | 'filters' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'shard'> { base?: string config?: string @@ -398,4 +406,8 @@ export interface ResolvedConfig extends Omit, 'config' | 'f defines: Record api?: ApiConfig + shard?: { + index: number + count: number + } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8a4b361e0f4b..f20e98079465 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -847,6 +847,14 @@ importers: devDependencies: vitest: link:../../packages/vitest + test/shard: + specifiers: + execa: ^6.1.0 + vitest: workspace:* + devDependencies: + execa: 6.1.0 + vitest: link:../../packages/vitest + test/single-thread: specifiers: vitest: workspace:* diff --git a/test/shard/package.json b/test/shard/package.json new file mode 100644 index 000000000000..8a28b5035add --- /dev/null +++ b/test/shard/package.json @@ -0,0 +1,11 @@ +{ + "name": "@vitest/test-shard", + "private": true, + "scripts": { + "test": "vitest run shard-test.test.ts" + }, + "devDependencies": { + "execa": "^6.1.0", + "vitest": "workspace:*" + } +} diff --git a/test/shard/shard-test.test.ts b/test/shard/shard-test.test.ts new file mode 100644 index 000000000000..9f185c24cec1 --- /dev/null +++ b/test/shard/shard-test.test.ts @@ -0,0 +1,50 @@ +import { expect, test } from 'vitest' +import { basename } from 'pathe' +import { execa } from 'execa' + +const runVitest = async (args: string[]) => { + const { stdout } = await execa('vitest', ['--run', '--dir', './test', ...args]) + return stdout +} + +const parsePaths = (stdout: string) => { + return stdout + .split('\n') + .filter(line => line && line.includes('.test.js')) + .map(file => basename(file.trim().split(' ')[1])) + .sort() +} + +test.concurrent('--shard=1/1', async () => { + const stdout = await runVitest(['--shard=1/1']) + + const paths = parsePaths(stdout) + + expect(paths).toEqual(['1.test.js', '2.test.js', '3.test.js']) +}) + +test.concurrent('--shard=1/2', async () => { + const stdout = await runVitest(['--shard=1/2']) + + const paths = parsePaths(stdout) + + expect(paths).toEqual(['2.test.js', '3.test.js']) +}) + +test.concurrent('--shard=2/2', async () => { + const stdout = await runVitest(['--shard=2/2']) + + const paths = parsePaths(stdout) + + expect(paths).toEqual(['1.test.js']) +}) + +test.concurrent('--shard=4/4', async () => { + const stdout = await runVitest(['--shard=4/4']) + + const paths = parsePaths(stdout) + + // project only has 3 files + // shards > 3 are empty + expect(paths).toEqual([]) +}) diff --git a/test/shard/test/1.test.js b/test/shard/test/1.test.js new file mode 100644 index 000000000000..f8d17d1d1108 --- /dev/null +++ b/test/shard/test/1.test.js @@ -0,0 +1,3 @@ +import { expect, test } from 'vitest' + +test('1', () => expect(1).toBe(1)) diff --git a/test/shard/test/2.test.js b/test/shard/test/2.test.js new file mode 100644 index 000000000000..786139928b08 --- /dev/null +++ b/test/shard/test/2.test.js @@ -0,0 +1,3 @@ +import { expect, test } from 'vitest' + +test('2', () => expect(1).toBe(1)) diff --git a/test/shard/test/3.test.js b/test/shard/test/3.test.js new file mode 100644 index 000000000000..e55f550a4f06 --- /dev/null +++ b/test/shard/test/3.test.js @@ -0,0 +1,3 @@ +import { expect, test } from 'vitest' + +test('3', () => expect(1).toBe(1)) From 0a4c83c82150fcac757fb5afc4a3d33c6fb3ddf6 Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Mon, 13 Jun 2022 20:35:45 +0300 Subject: [PATCH 2/4] chore: increase timeout for shard tests --- test/shard/shard-test.test.ts | 8 ++++---- test/shard/vitest.config.ts | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) create mode 100644 test/shard/vitest.config.ts diff --git a/test/shard/shard-test.test.ts b/test/shard/shard-test.test.ts index 9f185c24cec1..41ea0d0fbbf6 100644 --- a/test/shard/shard-test.test.ts +++ b/test/shard/shard-test.test.ts @@ -15,7 +15,7 @@ const parsePaths = (stdout: string) => { .sort() } -test.concurrent('--shard=1/1', async () => { +test('--shard=1/1', async () => { const stdout = await runVitest(['--shard=1/1']) const paths = parsePaths(stdout) @@ -23,7 +23,7 @@ test.concurrent('--shard=1/1', async () => { expect(paths).toEqual(['1.test.js', '2.test.js', '3.test.js']) }) -test.concurrent('--shard=1/2', async () => { +test('--shard=1/2', async () => { const stdout = await runVitest(['--shard=1/2']) const paths = parsePaths(stdout) @@ -31,7 +31,7 @@ test.concurrent('--shard=1/2', async () => { expect(paths).toEqual(['2.test.js', '3.test.js']) }) -test.concurrent('--shard=2/2', async () => { +test('--shard=2/2', async () => { const stdout = await runVitest(['--shard=2/2']) const paths = parsePaths(stdout) @@ -39,7 +39,7 @@ test.concurrent('--shard=2/2', async () => { expect(paths).toEqual(['1.test.js']) }) -test.concurrent('--shard=4/4', async () => { +test('--shard=4/4', async () => { const stdout = await runVitest(['--shard=4/4']) const paths = parsePaths(stdout) diff --git a/test/shard/vitest.config.ts b/test/shard/vitest.config.ts new file mode 100644 index 000000000000..99017865e5bd --- /dev/null +++ b/test/shard/vitest.config.ts @@ -0,0 +1,7 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + testTimeout: 50_000, + }, +}) From ed16364617eca7bda5415f66e78e571ceb3dbd9c Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Mon, 13 Jun 2022 21:12:08 +0300 Subject: [PATCH 3/4] chore: use relative path for spec --- packages/vitest/src/node/pool.ts | 11 ++++++----- test/shard/shard-test.test.ts | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/vitest/src/node/pool.ts b/packages/vitest/src/node/pool.ts index 6fa99d7c9e79..e151cdbb3401 100644 --- a/packages/vitest/src/node/pool.ts +++ b/packages/vitest/src/node/pool.ts @@ -1,7 +1,7 @@ import { MessageChannel } from 'worker_threads' import { pathToFileURL } from 'url' import { cpus } from 'os' -import crypto from 'crypto' +import { createHash } from 'crypto' import { resolve } from 'pathe' import type { Options as TinypoolOptions } from 'tinypool' import { Tinypool } from 'tinypool' @@ -9,7 +9,7 @@ import { createBirpc } from 'birpc' import type { RawSourceMap } from 'vite-node' import type { WorkerContext, WorkerRPC } from '../types' import { distDir } from '../constants' -import { AggregateError } from '../utils' +import { AggregateError, slash } from '../utils' import type { Vitest } from './core' export type RunWithFiles = (files: string[], invalidates?: string[]) => Promise @@ -96,11 +96,12 @@ export function createPool(ctx: Vitest): WorkerPool { const shardEnd = shardSize * index files = files .map((file) => { + const fullPath = resolve(slash(config.root), slash(file)) + const specPath = fullPath.slice(config.root.length) return { file, - hash: crypto - .createHash('sha1') - .update(file) + hash: createHash('sha1') + .update(specPath) .digest('hex'), } }) diff --git a/test/shard/shard-test.test.ts b/test/shard/shard-test.test.ts index 41ea0d0fbbf6..b7099f1b80ea 100644 --- a/test/shard/shard-test.test.ts +++ b/test/shard/shard-test.test.ts @@ -28,7 +28,7 @@ test('--shard=1/2', async () => { const paths = parsePaths(stdout) - expect(paths).toEqual(['2.test.js', '3.test.js']) + expect(paths).toEqual(['1.test.js', '2.test.js']) }) test('--shard=2/2', async () => { @@ -36,7 +36,7 @@ test('--shard=2/2', async () => { const paths = parsePaths(stdout) - expect(paths).toEqual(['1.test.js']) + expect(paths).toEqual(['3.test.js']) }) test('--shard=4/4', async () => { From 631b5a94be3e27ed377539dfd6f60cd89652a1a6 Mon Sep 17 00:00:00 2001 From: Vladimir Sheremet Date: Tue, 14 Jun 2022 18:48:16 +0300 Subject: [PATCH 4/4] chore: add try/catch to plugin --- packages/vitest/src/node/config.ts | 16 +++------------- packages/vitest/src/node/plugins/index.ts | 14 ++++++++++---- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index b1e055a6cb86..d2609b705db8 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -56,16 +56,6 @@ export function resolveApiConfig( return api } -const configError = (error: string): never => { - console.warn( - c.yellow( - `${c.inverse(c.red(' VITEST '))} ${error}\n`, - ), - ) - - process.exit(1) -} - export function resolveConfig( options: UserConfig, viteConfig: ResolvedViteConfig, @@ -102,17 +92,17 @@ export function resolveConfig( if (options.shard) { if (resolved.watch) - configError('You cannot use --shard option with enabled watch') + throw new Error('You cannot use --shard option with enabled watch') const [indexString, countString] = options.shard.split('/') const index = Math.abs(parseInt(indexString, 10)) const count = Math.abs(parseInt(countString, 10)) if (isNaN(count) || count <= 0) - configError('--shard must be a positive number') + throw new Error('--shard must be a positive number') if (isNaN(index) || index <= 0 || index > count) - configError('--shard must be a positive number less then ') + throw new Error('--shard must be a positive number less then ') resolved.shard = { index, count } } diff --git a/packages/vitest/src/node/plugins/index.ts b/packages/vitest/src/node/plugins/index.ts index 8cc0f4239480..750f11e149a0 100644 --- a/packages/vitest/src/node/plugins/index.ts +++ b/packages/vitest/src/node/plugins/index.ts @@ -125,10 +125,16 @@ export async function VitestPlugin(options: UserConfig = {}, ctx = new Vitest()) async configureServer(server) { if (haveStarted) await ctx.report('onServerRestart') - await ctx.setServer(options, server) - haveStarted = true - if (options.api && options.watch) - (await import('../../api/setup')).setup(ctx) + try { + await ctx.setServer(options, server) + haveStarted = true + if (options.api && options.watch) + (await import('../../api/setup')).setup(ctx) + } + catch (err) { + ctx.printError(err, true) + process.exit(1) + } // #415, in run mode we don't need the watcher, close it would improve the performance if (!options.watch)