diff --git a/docs/config/index.md b/docs/config/index.md index 0564e20cec5f..00ed5cd393c3 100644 --- a/docs/config/index.md +++ b/docs/config/index.md @@ -10,7 +10,7 @@ outline: deep - Create `vitest.config.ts`, which will have the higher priority and will override the configuration from `vite.config.ts` - Pass `--config` option to CLI, e.g. `vitest --config ./path/to/vitest.config.ts` -- Use `process.env.VITEST` or `mode` property on `defineConfig` (will be set to `test` if not overridden) to conditionally apply different configuration in `vite.config.ts` +- Use `process.env.VITEST` or `mode` property on `defineConfig` (will be set to `test`/`benchmark` if not overridden) to conditionally apply different configuration in `vite.config.ts` To configure `vitest` itself, add `test` property in your Vite config. You'll also need to add a reference to Vitest types using a [triple slash command](https://www.typescriptlang.org/docs/handbook/triple-slash-directives.html#-reference-types-) at the top of your config file, if you are importing `defineConfig` from `vite` itself. diff --git a/packages/ui/client/composables/summary.ts b/packages/ui/client/composables/summary.ts index 07f1518ac836..aa8eab5e86ad 100644 --- a/packages/ui/client/composables/summary.ts +++ b/packages/ui/client/composables/summary.ts @@ -1,5 +1,5 @@ import { hasFailedSnapshot } from '@vitest/ws-client' -import type { Task, Test } from 'vitest/src' +import type { Benchmark, Task, Test } from 'vitest/src' import { files, testRunState } from '~/composables/client' type Nullable = T | null | undefined @@ -52,7 +52,9 @@ function toArray(array?: Nullable>): Array { return array return [array] } - -function getTests(suite: Arrayable): Test[] { - return toArray(suite).flatMap(s => s.type === 'test' ? [s] : s.tasks.flatMap(c => c.type === 'test' ? [c] : getTests(c))) +function isAtomTest(s: Task): s is Test | Benchmark { + return (s.type === 'test' || s.type === 'benchmark') +} +function getTests(suite: Arrayable): (Test | Benchmark)[] { + return toArray(suite).flatMap(s => isAtomTest(s) ? [s] : s.tasks.flatMap(c => isAtomTest(c) ? [c] : getTests(c))) } diff --git a/packages/vitest/package.json b/packages/vitest/package.json index cf1d256cd8e9..762af70341eb 100644 --- a/packages/vitest/package.json +++ b/packages/vitest/package.json @@ -103,6 +103,7 @@ "debug": "^4.3.4", "local-pkg": "^0.4.2", "strip-literal": "^0.4.0", + "tinybench": "^2.1.3", "tinypool": "^0.2.4", "tinyspy": "^1.0.2", "vite": "^2.9.12 || ^3.0.0-0" @@ -123,6 +124,7 @@ "chai-subset": "^1.6.0", "cli-truncate": "^3.1.0", "diff": "^5.1.0", + "event-target-polyfill": "^0.0.3", "execa": "^6.1.0", "fast-glob": "^3.2.11", "find-up": "^6.3.0", diff --git a/packages/vitest/src/defaults.ts b/packages/vitest/src/defaults.ts index bd29f09802f2..4c717e1c413d 100644 --- a/packages/vitest/src/defaults.ts +++ b/packages/vitest/src/defaults.ts @@ -1,8 +1,15 @@ -import type { ResolvedCoverageOptions, UserConfig } from './types' +import type { BenchmarkUserOptions, ResolvedCoverageOptions, UserConfig } from './types' export const defaultInclude = ['**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'] export const defaultExclude = ['**/node_modules/**', '**/dist/**', '**/cypress/**', '**/.{idea,git,cache,output,temp}/**'] +export const benchmarkConfigDefaults: Required = { + include: ['**/*.{bench,benchmark}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'], + exclude: defaultExclude, + includeSource: [], + reporters: ['default'], +} + const defaultCoverageExcludes = [ 'coverage/**', 'dist/**', diff --git a/packages/vitest/src/index.ts b/packages/vitest/src/index.ts index 608077a511ca..8b8ddedbbc09 100644 --- a/packages/vitest/src/index.ts +++ b/packages/vitest/src/index.ts @@ -1,4 +1,4 @@ -export { suite, test, describe, it } from './runtime/suite' +export { suite, test, describe, it, bench } from './runtime/suite' export * from './runtime/hooks' export * from './runtime/utils' diff --git a/packages/vitest/src/node/cli-api.ts b/packages/vitest/src/node/cli-api.ts index fc19fbfce6d4..2058902e8c5d 100644 --- a/packages/vitest/src/node/cli-api.ts +++ b/packages/vitest/src/node/cli-api.ts @@ -3,7 +3,7 @@ import type { UserConfig as ViteUserConfig } from 'vite' import { EXIT_CODE_RESTART } from '../constants' import { CoverageProviderMap } from '../integrations/coverage' import { getEnvPackageName } from '../integrations/env' -import type { UserConfig } from '../types' +import type { UserConfig, VitestRunMode } from '../types' import { ensurePackageInstalled } from '../utils' import { createVitest } from './create' import { registerConsoleShortcuts } from './stdin' @@ -15,7 +15,7 @@ export interface CliOptions extends UserConfig { run?: boolean } -export async function startVitest(cliFilters: string[], options: CliOptions, viteOverrides?: ViteUserConfig) { +export async function startVitest(mode: VitestRunMode, cliFilters: string[], options: CliOptions, viteOverrides?: ViteUserConfig) { process.env.TEST = 'true' process.env.VITEST = 'true' process.env.NODE_ENV ??= options.mode || 'test' @@ -36,9 +36,9 @@ export async function startVitest(cliFilters: string[], options: CliOptions, vit if (typeof options.coverage === 'boolean') options.coverage = { enabled: options.coverage } - const ctx = await createVitest(options, viteOverrides) + const ctx = await createVitest(mode, options, viteOverrides) - if (ctx.config.coverage.enabled) { + if (mode !== 'benchmark' && ctx.config.coverage.enabled) { const provider = ctx.config.coverage.provider || 'c8' if (typeof provider === 'string') { const requiredPackages = CoverageProviderMap[provider] diff --git a/packages/vitest/src/node/cli.ts b/packages/vitest/src/node/cli.ts index 5a09a7ce1d09..7fd43aeff351 100644 --- a/packages/vitest/src/node/cli.ts +++ b/packages/vitest/src/node/cli.ts @@ -1,6 +1,7 @@ import cac from 'cac' import c from 'picocolors' import { version } from '../../package.json' +import type { VitestRunMode } from '../types' import type { CliOptions } from './cli-api' import { startVitest } from './cli-api' import { divider } from './reporters/renderers/utils' @@ -60,26 +61,35 @@ cli .command('dev [...filters]') .action(start) +cli + .command('bench [...filters]') + .action(benchmark) + cli .command('[...filters]') - .action(start) + .action((filter, options) => start('test', filter, options)) cli.parse() async function runRelated(relatedFiles: string[] | string, argv: CliOptions) { argv.related = relatedFiles argv.passWithNoTests ??= true - await start([], argv) + await start('test', [], argv) } async function run(cliFilters: string[], options: CliOptions) { options.run = true - await start(cliFilters, options) + await start('test', cliFilters, options) +} + +async function benchmark(cliFilters: string[], options: CliOptions) { + console.warn(c.yellow('Benchmarking is an experimental feature. API might change in the future.')) + await start('benchmark', cliFilters, options) } -async function start(cliFilters: string[], options: CliOptions) { +async function start(mode: VitestRunMode, cliFilters: string[], options: CliOptions) { try { - if (await startVitest(cliFilters, options) === false) + if (await startVitest(mode, cliFilters, options) === false) process.exit() } catch (e) { diff --git a/packages/vitest/src/node/config.ts b/packages/vitest/src/node/config.ts index 72999551a87f..791d27ea2299 100644 --- a/packages/vitest/src/node/config.ts +++ b/packages/vitest/src/node/config.ts @@ -3,9 +3,9 @@ import { normalize, resolve } from 'pathe' import c from 'picocolors' import type { ResolvedConfig as ResolvedViteConfig } from 'vite' -import type { ApiConfig, ResolvedConfig, UserConfig } from '../types' +import type { ApiConfig, ResolvedConfig, UserConfig, VitestRunMode } from '../types' import { defaultPort } from '../constants' -import { configDefaults } from '../defaults' +import { benchmarkConfigDefaults, configDefaults } from '../defaults' import { toArray } from '../utils' import { VitestCache } from './cache' import { BaseSequencer } from './sequencers/BaseSequencer' @@ -61,6 +61,7 @@ export function resolveApiConfig( } export function resolveConfig( + mode: VitestRunMode, options: UserConfig, viteConfig: ResolvedViteConfig, ): ResolvedConfig { @@ -87,6 +88,7 @@ export function resolveConfig( ...configDefaults, ...options, root: viteConfig.root, + mode, } as ResolvedConfig if (viteConfig.base !== '/') @@ -157,6 +159,18 @@ export function resolveConfig( if (process.env.VITEST_MIN_THREADS) resolved.minThreads = parseInt(process.env.VITEST_MIN_THREADS) + if (mode === 'benchmark') { + resolved.benchmark = { + ...benchmarkConfigDefaults, + ...resolved.benchmark, + } + // override test config + resolved.coverage.enabled = false + resolved.include = resolved.benchmark.include + resolved.exclude = resolved.benchmark.exclude + resolved.includeSource = resolved.benchmark.includeSource + } + resolved.setupFiles = toArray(resolved.setupFiles || []).map(file => normalize( resolveModule(file, { paths: [resolved.root] }) @@ -175,6 +189,7 @@ export function resolveConfig( // @ts-expect-error from CLI ...toArray(resolved.reporter), ])).filter(Boolean) + if (!resolved.reporters.length) resolved.reporters.push('default') diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts index a53df6817a82..b2c90edca9d6 100644 --- a/packages/vitest/src/node/core.ts +++ b/packages/vitest/src/node/core.ts @@ -7,13 +7,13 @@ import mm from 'micromatch' import c from 'picocolors' import { ViteNodeRunner } from 'vite-node/client' import { ViteNodeServer } from 'vite-node/server' -import type { ArgumentsType, CoverageProvider, OnServerRestartHandler, Reporter, ResolvedConfig, UserConfig } from '../types' +import type { ArgumentsType, CoverageProvider, OnServerRestartHandler, Reporter, ResolvedConfig, UserConfig, VitestRunMode } from '../types' import { SnapshotManager } from '../integrations/snapshot/manager' -import { clearTimeout, deepMerge, hasFailed, noop, setTimeout, slash } from '../utils' +import { clearTimeout, deepMerge, hasFailed, noop, setTimeout, slash, toArray } from '../utils' import { getCoverageProvider } from '../integrations/coverage' import { createPool } from './pool' import type { WorkerPool } from './pool' -import { createReporters } from './reporters/utils' +import { createBenchmarkReporters, createReporters } from './reporters/utils' import { StateManager } from './state' import { resolveConfig } from './config' import { Logger } from './logger' @@ -45,7 +45,9 @@ export class Vitest { restartsCount = 0 runner: ViteNodeRunner = undefined! - constructor() { + constructor( + public readonly mode: VitestRunMode, + ) { this.logger = new Logger(this) } @@ -58,7 +60,7 @@ export class Vitest { this.pool?.close() this.pool = undefined - const resolved = resolveConfig(options, server.config) + const resolved = resolveConfig(this.mode, options, server.config) this.server = server this.config = resolved @@ -101,7 +103,9 @@ export class Vitest { }) } - this.reporters = await createReporters(resolved.reporters, this.runner) + this.reporters = resolved.mode === 'benchmark' + ? await createBenchmarkReporters(toArray(resolved.benchmark?.reporters), this.runner) + : await createReporters(resolved.reporters, this.runner) this.runningPromise = undefined @@ -490,13 +494,15 @@ export class Vitest { } async globTestFiles(filters: string[] = []) { + const { include, exclude, includeSource } = this.config + const globOptions = { absolute: true, cwd: this.config.dir || this.config.root, - ignore: this.config.exclude, + ignore: exclude, } - let testFiles = await fg(this.config.include, globOptions) + let testFiles = await fg(include, globOptions) if (filters.length && process.platform === 'win32') filters = filters.map(f => toNamespacedPath(f)) @@ -504,8 +510,8 @@ export class Vitest { if (filters.length) testFiles = testFiles.filter(i => filters.some(f => i.includes(f))) - if (this.config.includeSource) { - let files = await fg(this.config.includeSource, globOptions) + if (includeSource) { + let files = await fg(includeSource, globOptions) if (filters.length) files = files.filter(i => filters.some(f => i.includes(f))) diff --git a/packages/vitest/src/node/create.ts b/packages/vitest/src/node/create.ts index 8f18080f7c3c..707248374773 100644 --- a/packages/vitest/src/node/create.ts +++ b/packages/vitest/src/node/create.ts @@ -2,13 +2,13 @@ import { resolve } from 'pathe' import { createServer, mergeConfig } from 'vite' import type { InlineConfig as ViteInlineConfig, UserConfig as ViteUserConfig } from 'vite' import { findUp } from 'find-up' -import type { UserConfig } from '../types' +import type { UserConfig, VitestRunMode } from '../types' import { configFiles } from '../constants' import { Vitest } from './core' import { VitestPlugin } from './plugins' -export async function createVitest(options: UserConfig, viteOverrides: ViteUserConfig = {}) { - const ctx = new Vitest() +export async function createVitest(mode: VitestRunMode, options: UserConfig, viteOverrides: ViteUserConfig = {}) { + const ctx = new Vitest(mode) const root = resolve(options.root || process.cwd()) const configPath = options.config @@ -19,7 +19,7 @@ export async function createVitest(options: UserConfig, viteOverrides: ViteUserC logLevel: 'error', configFile: configPath, // this will make "mode" = "test" inside defineConfig - mode: options.mode || process.env.NODE_ENV || 'test', + mode: options.mode || process.env.NODE_ENV || mode, plugins: await VitestPlugin(options, ctx), } diff --git a/packages/vitest/src/node/logger.ts b/packages/vitest/src/node/logger.ts index 09716b5f3102..a651f1256ad8 100644 --- a/packages/vitest/src/node/logger.ts +++ b/packages/vitest/src/node/logger.ts @@ -78,9 +78,9 @@ export class Logger { this.console.error(c.dim('watch exclude: ') + c.yellow(config.watchExclude.join(comma))) if (config.passWithNoTests) - this.log('No test files found, exiting with code 0\n') + this.log(`No ${config.mode} files found, exiting with code 0\n`) else - this.error(c.red('\nNo test files found, exiting with code 1')) + this.error(c.red(`\nNo ${config.mode} files found, exiting with code 1`)) } printBanner() { diff --git a/packages/vitest/src/node/plugins/index.ts b/packages/vitest/src/node/plugins/index.ts index cc4ab2974cd7..f981a41ec787 100644 --- a/packages/vitest/src/node/plugins/index.ts +++ b/packages/vitest/src/node/plugins/index.ts @@ -12,7 +12,7 @@ import { MocksPlugin } from './mock' import { CSSEnablerPlugin } from './cssEnabler' import { CoverageTransform } from './coverageTransform' -export async function VitestPlugin(options: UserConfig = {}, ctx = new Vitest()): Promise { +export async function VitestPlugin(options: UserConfig = {}, ctx = new Vitest('test')): Promise { const getRoot = () => ctx.config?.root || options.root || process.cwd() async function UIPlugin() { diff --git a/packages/vitest/src/node/reporters/benchmark/index.ts b/packages/vitest/src/node/reporters/benchmark/index.ts new file mode 100644 index 000000000000..95a76d5ad001 --- /dev/null +++ b/packages/vitest/src/node/reporters/benchmark/index.ts @@ -0,0 +1,7 @@ +import { VerboseReporter } from '../verbose' +import { JsonReporter } from './json' +export const BenchmarkReportsMap = { + default: VerboseReporter, + json: JsonReporter, +} +export type BenchmarkBuiltinReporters = keyof typeof BenchmarkReportsMap diff --git a/packages/vitest/src/node/reporters/benchmark/json.ts b/packages/vitest/src/node/reporters/benchmark/json.ts new file mode 100644 index 000000000000..d85735666adc --- /dev/null +++ b/packages/vitest/src/node/reporters/benchmark/json.ts @@ -0,0 +1,82 @@ +import { existsSync, promises as fs } from 'fs' +import { dirname, resolve } from 'pathe' +import type { Vitest } from '../../../node' +import type { BenchTaskResult, File, Reporter } from '../../../types' +import { getSuites, getTests } from '../../../utils' +import { getOutputFile } from '../../../utils/config-helpers' + +interface FormattedTestResults { + numTotalTests: number + numTotalTestSuites: number + testResults: Record +} + +export class JsonReporter implements Reporter { + start = 0 + ctx!: Vitest + + onInit(ctx: Vitest): void { + this.ctx = ctx + } + + protected async logTasks(files: File[]) { + const suites = getSuites(files) + const numTotalTestSuites = suites.length + const tests = getTests(files) + const numTotalTests = tests.length + const testResults: Record = {} + const outputFile = getOutputFile(this.ctx, 'json') + for (const file of files) { + const tests = getTests([file]) + for (const test of tests) { + const res = test.result!.benchmark! + if (!outputFile) + res.samples = 'ignore on terminal' as any + + testResults[test.suite.name] = (testResults[test.suite.name] || []).concat(res) + } + + // test.suite.name + if (tests.some(t => t.result?.state === 'run')) { + this.ctx.logger.warn('WARNING: Some tests are still running when generating the markdown report.' + + 'This is likely an internal bug in Vitest.' + + 'Please report it to https://github.com/vitest-dev/vitest/issues') + } + } + + const result: FormattedTestResults = { + numTotalTestSuites, + numTotalTests, + testResults, + } + + await this.writeReport(JSON.stringify(result, null, 2)) + } + + async onFinished(files = this.ctx.state.getFiles()) { + await this.logTasks(files) + } + + /** + * Writes the report to an output file if specified in the config, + * or logs it to the console otherwise. + * @param report + */ + async writeReport(report: string) { + const outputFile = getOutputFile(this.ctx, 'json') + + if (outputFile) { + const reportFile = resolve(this.ctx.config.root, outputFile) + + const outputDirectory = dirname(reportFile) + if (!existsSync(outputDirectory)) + await fs.mkdir(outputDirectory, { recursive: true }) + + await fs.writeFile(reportFile, report, 'utf-8') + this.ctx.logger.log(`markdown report written to ${reportFile}`) + } + else { + this.ctx.logger.log(report) + } + } +} diff --git a/packages/vitest/src/node/reporters/index.ts b/packages/vitest/src/node/reporters/index.ts index 09c0698566b4..664f2afcad58 100644 --- a/packages/vitest/src/node/reporters/index.ts +++ b/packages/vitest/src/node/reporters/index.ts @@ -19,3 +19,5 @@ export const ReportersMap = { } export type BuiltinReporters = keyof typeof ReportersMap + +export * from './benchmark' diff --git a/packages/vitest/src/node/reporters/json.ts b/packages/vitest/src/node/reporters/json.ts index 13497c42bd9b..3c06f745c13b 100644 --- a/packages/vitest/src/node/reporters/json.ts +++ b/packages/vitest/src/node/reporters/json.ts @@ -1,7 +1,7 @@ import { existsSync, promises as fs } from 'fs' import { dirname, resolve } from 'pathe' import type { Vitest } from '../../node' -import type { File, Reporter, Suite, TaskState, Test } from '../../types' +import type { File, Reporter, Suite, Task, TaskState } from '../../types' import { getSuites, getTests } from '../../utils' import { getOutputFile } from '../../utils/config-helpers' import { interpretSourcePos, parseStacktrace } from '../../utils/source-map' @@ -180,7 +180,7 @@ export class JsonReporter implements Reporter { } } - protected async getFailureLocation(test: Test): Promise { + protected async getFailureLocation(test: Task): Promise { const error = test.result?.error if (!error) return diff --git a/packages/vitest/src/node/reporters/renderers/listRenderer.ts b/packages/vitest/src/node/reporters/renderers/listRenderer.ts index ffea81e29bc2..ec4c2593a9a0 100644 --- a/packages/vitest/src/node/reporters/renderers/listRenderer.ts +++ b/packages/vitest/src/node/reporters/renderers/listRenderer.ts @@ -1,8 +1,8 @@ import c from 'picocolors' import cliTruncate from 'cli-truncate' import stripAnsi from 'strip-ansi' -import type { SuiteHooks, Task } from '../../../types' -import { clearInterval, getTests, setInterval } from '../../../utils' +import type { Benchmark, BenchmarkResult, SuiteHooks, Task } from '../../../types' +import { clearInterval, getTests, notNullish, setInterval } from '../../../utils' import { F_RIGHT } from '../../../utils/figures' import type { Logger } from '../../logger' import { getCols, getHookStateSymbol, getStateSymbol } from './utils' @@ -28,6 +28,12 @@ function formatFilepath(path: string) { return c.dim(path.slice(0, lastSlash)) + path.slice(lastSlash, firstDot) + c.dim(path.slice(firstDot)) } +function formatNumber(number: number) { + const res = String(number.toFixed(number < 100 ? 4 : 2)).split('.') + return res[0].replace(/(?=(?:\d{3})+$)(?!\b)/g, ',') + + (res[1] ? `.${res[1]}` : '') +} + function renderHookState(task: Task, hookName: keyof SuiteHooks, level = 0) { const state = task.result?.hooks?.[hookName] if (state && state === 'run') @@ -36,6 +42,49 @@ function renderHookState(task: Task, hookName: keyof SuiteHooks, level = 0) { return '' } +function renderBenchmarkItems(result: BenchmarkResult) { + return [ + result.name, + formatNumber(result.hz || 0), + formatNumber(result.p99 || 0), + `±${result.rme.toFixed(2)}%`, + result.samples.length.toString(), + ] +} + +function renderBenchmark(task: Benchmark, tasks: Task[]): string { + const result = task.result?.benchmark + if (!result) + return task.name + + const benchs = tasks + .map(i => i.type === 'benchmark' ? i.result?.benchmark : undefined) + .filter(notNullish) + + const allItems = benchs.map(renderBenchmarkItems) + const items = renderBenchmarkItems(result) + const padded = items.map((i, idx) => { + const width = Math.max(...allItems.map(i => i[idx].length)) + return idx + ? i.padStart(width, ' ') + : i.padEnd(width, ' ') // name + }) + + return [ + padded[0], // name + c.dim(' '), + c.blue(padded[1]), + c.dim(' ops/sec '), + c.cyan(padded[3]), + c.dim(` (${padded[4]} samples)`), + result.rank === 1 + ? c.bold(c.green(' fastest')) + : result.rank === benchs.length && benchs.length > 2 + ? c.bold(c.gray(' slowest')) + : '', + ].join('') +} + export function renderTree(tasks: Task[], options: ListRendererOptions, level = 0) { let output: string[] = [] @@ -63,7 +112,13 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level = let name = task.name if (level === 0) name = formatFilepath(name) - output.push(' '.repeat(level) + prefix + name + suffix) + + const padding = ' '.repeat(level) + const body = task.type === 'benchmark' + ? renderBenchmark(task, tasks) + : name + + output.push(padding + prefix + body + suffix) if ((task.result?.state !== 'pass') && outputMap.get(task) != null) { let data: string | undefined = outputMap.get(task) diff --git a/packages/vitest/src/node/reporters/renderers/utils.ts b/packages/vitest/src/node/reporters/renderers/utils.ts index 19cdfeeca946..469e37dfe666 100644 --- a/packages/vitest/src/node/reporters/renderers/utils.ts +++ b/packages/vitest/src/node/reporters/renderers/utils.ts @@ -127,8 +127,11 @@ export function getStateSymbol(task: Task) { return c.yellow(spinner()) } - if (task.result.state === 'pass') - return c.green(F_CHECK) + if (task.result.state === 'pass') { + return task.type === 'benchmark' + ? c.green(F_DOT) + : c.green(F_CHECK) + } if (task.result.state === 'fail') { return task.type === 'suite' @@ -171,6 +174,24 @@ export function elegantSpinner() { } } +export function duration(time: number, locale = 'en-us') { + if (time < 1e0) + return `${Number((time * 1e3).toFixed(2)).toLocaleString(locale)} ps` + + if (time < 1e3) + return `${Number(time.toFixed(2)).toLocaleString(locale)} ns` + if (time < 1e6) + return `${Number((time / 1e3).toFixed(2)).toLocaleString(locale)} µs` + if (time < 1e9) + return `${Number((time / 1e6).toFixed(2)).toLocaleString(locale)} ms` + if (time < 1e12) + return `${Number((time / 1e9).toFixed(2)).toLocaleString(locale)} s` + if (time < 36e11) + return `${Number((time / 60e9).toFixed(2)).toLocaleString(locale)} m` + + return `${Number((time / 36e11).toFixed(2)).toLocaleString(locale)} h` +} + export function formatTimeString(date: Date) { return date.toTimeString().split(' ')[0] } diff --git a/packages/vitest/src/node/reporters/utils.ts b/packages/vitest/src/node/reporters/utils.ts index 1380f2d12904..aad3aa695c88 100644 --- a/packages/vitest/src/node/reporters/utils.ts +++ b/packages/vitest/src/node/reporters/utils.ts @@ -1,7 +1,7 @@ import type { ViteNodeRunner } from 'vite-node/client' import type { Reporter } from '../../types' -import { ReportersMap } from './index' -import type { BuiltinReporters } from './index' +import { BenchmarkReportsMap, ReportersMap } from './index' +import type { BenchmarkBuiltinReporters, BuiltinReporters } from './index' async function loadCustomReporterModule(path: string, runner: ViteNodeRunner): Promise C> { let customReporterModule: { default: new () => C } @@ -35,4 +35,21 @@ function createReporters(reporterReferences: Array, runner: ViteNodeRunner) { + const promisedReporters = reporterReferences.map(async (referenceOrInstance) => { + if (typeof referenceOrInstance === 'string') { + if (referenceOrInstance in BenchmarkReportsMap) { + const BuiltinReporter = BenchmarkReportsMap[referenceOrInstance as BenchmarkBuiltinReporters] + return new BuiltinReporter() + } + else { + const CustomReporter = await loadCustomReporterModule(referenceOrInstance, runner) + return new CustomReporter() + } + } + return referenceOrInstance + }) + return Promise.all(promisedReporters) +} + +export { createReporters, createBenchmarkReporters } diff --git a/packages/vitest/src/runtime/chain.ts b/packages/vitest/src/runtime/chain.ts index fec730089f4f..12db6fbb125b 100644 --- a/packages/vitest/src/runtime/chain.ts +++ b/packages/vitest/src/runtime/chain.ts @@ -2,6 +2,8 @@ export type ChainableFunction +} & { + fn: (this: Record, ...args: Args) => R } & E export function createChainable( diff --git a/packages/vitest/src/runtime/collect.ts b/packages/vitest/src/runtime/collect.ts index 5b48184b935a..1c18abd6aaf3 100644 --- a/packages/vitest/src/runtime/collect.ts +++ b/packages/vitest/src/runtime/collect.ts @@ -20,7 +20,7 @@ function hash(str: string): string { return `${hash}` } -export async function collectTests(paths: string[], config: ResolvedConfig) { +export async function collectTests(paths: string[], config: ResolvedConfig): Promise { const files: File[] = [] const browserHashMap = getWorkerState().browserHashMap! @@ -45,6 +45,7 @@ export async function collectTests(paths: string[], config: ResolvedConfig) { } clearCollectorContext() + try { const setupStart = now() await runSetupFiles(config) @@ -64,10 +65,13 @@ export async function collectTests(paths: string[], config: ResolvedConfig) { if (c.type === 'test') { file.tasks.push(c) } + else if (c.type === 'benchmark') { + file.tasks.push(c) + } else if (c.type === 'suite') { file.tasks.push(c) } - else { + else if (c.type === 'collector') { const suite = await c.collect(file) if (suite.name || suite.tasks.length) file.tasks.push(suite) diff --git a/packages/vitest/src/runtime/map.ts b/packages/vitest/src/runtime/map.ts index 190297243393..6fd2262ef96d 100644 --- a/packages/vitest/src/runtime/map.ts +++ b/packages/vitest/src/runtime/map.ts @@ -1,15 +1,16 @@ -import type { Awaitable, Suite, SuiteHooks, Test } from '../types' +import type { Awaitable, BenchFactory, BenchFunction, Benchmark, Suite, SuiteHooks, Test } from '../types' // use WeakMap here to make the Test and Suite object serializable const fnMap = new WeakMap() const hooksMap = new WeakMap() +const benchmarkMap = new WeakMap() -export function setFn(key: Test, fn: () => Awaitable) { +export function setFn(key: Test | Benchmark, fn: (() => Awaitable) | BenchFunction) { fnMap.set(key, fn) } -export function getFn(key: Test): () => Awaitable { - return fnMap.get(key) +export function getFn(key: Task): Task extends Test ? (() => Awaitable) : BenchFunction { + return fnMap.get(key as any) } export function setHooks(key: Suite, hooks: SuiteHooks) { @@ -19,3 +20,20 @@ export function setHooks(key: Suite, hooks: SuiteHooks) { export function getHooks(key: Suite): SuiteHooks { return hooksMap.get(key) } + +export function isTest(task: Test | Benchmark): task is Test { + return task.type === 'test' +} + +export async function getBenchmarkFactory(key: Suite): Promise { + let benchmark = benchmarkMap.get(key) + if (!benchmark) { + if (!globalThis.EventTarget) + await import('event-target-polyfill' as any) + + const Bench = (await import('tinybench')).default + benchmark = new Bench({}) + benchmarkMap.set(key, benchmark) + } + return benchmark +} diff --git a/packages/vitest/src/runtime/run.ts b/packages/vitest/src/runtime/run.ts index 7466dc7d5f13..55a228150792 100644 --- a/packages/vitest/src/runtime/run.ts +++ b/packages/vitest/src/runtime/run.ts @@ -1,11 +1,12 @@ +import { performance } from 'perf_hooks' import limit from 'p-limit' -import type { File, HookCleanupCallback, HookListener, ResolvedConfig, Suite, SuiteHooks, Task, TaskResult, TaskState, Test } from '../types' +import type { Benchmark, BenchmarkResult, File, HookCleanupCallback, HookListener, ResolvedConfig, Suite, SuiteHooks, Task, TaskResult, TaskState, Test } from '../types' import { vi } from '../integrations/vi' -import { clearTimeout, getFullName, getWorkerState, hasFailed, hasTests, isBrowser, isNode, partitionSuiteChildren, setTimeout, shuffle } from '../utils' +import { clearTimeout, createDefer, getFullName, getWorkerState, hasFailed, hasTests, isBrowser, isNode, isRunningInBenchmark, partitionSuiteChildren, setTimeout, shuffle } from '../utils' import { getState, setState } from '../integrations/chai/jest-expect' import { GLOBAL_EXPECT } from '../integrations/chai/constants' import { takeCoverageInsideWorker } from '../integrations/coverage' -import { getFn, getHooks } from './map' +import { getBenchmarkFactory, getFn, getHooks } from './map' import { rpc } from './rpc' import { collectTests } from './collect' import { processError } from './error' @@ -230,23 +231,27 @@ export async function runSuite(suite: Suite) { else { try { const beforeAllCleanups = await callSuiteHook(suite, suite, 'beforeAll', [suite]) - - for (let tasksGroup of partitionSuiteChildren(suite)) { - if (tasksGroup[0].concurrent === true) { - const mutex = limit(workerState.config.maxConcurrency) - await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c)))) - } - else { - const { sequence } = workerState.config - if (sequence.shuffle || suite.shuffle) { - // run describe block independently from tests - const suites = tasksGroup.filter(group => group.type === 'suite') - const tests = tasksGroup.filter(group => group.type === 'test') - const groups = shuffle([suites, tests], sequence.seed) - tasksGroup = groups.flatMap(group => shuffle(group, sequence.seed)) + if (isRunningInBenchmark()) { + await runBenchmarkSuit(suite) + } + else { + for (let tasksGroup of partitionSuiteChildren(suite)) { + if (tasksGroup[0].concurrent === true) { + const mutex = limit(workerState.config.maxConcurrency) + await Promise.all(tasksGroup.map(c => mutex(() => runSuiteChild(c)))) + } + else { + const { sequence } = workerState.config + if (sequence.shuffle || suite.shuffle) { + // run describe block independently from tests + const suites = tasksGroup.filter(group => group.type === 'suite') + const tests = tasksGroup.filter(group => group.type === 'test') + const groups = shuffle([suites, tests], sequence.seed) + tasksGroup = groups.flatMap(group => shuffle(group, sequence.seed)) + } + for (const c of tasksGroup) + await runSuiteChild(c) } - for (const c of tasksGroup) - await runSuiteChild(c) } } @@ -280,10 +285,97 @@ export async function runSuite(suite: Suite) { updateTask(suite) } +function createBenchmarkResult(name: string): BenchmarkResult { + return { + name, + rank: 0, + rme: 0, + samples: [] as number[], + } as BenchmarkResult +} + +async function runBenchmarkSuit(suite: Suite) { + const start = performance.now() + + const benchmarkGroup = [] + const benchmarkSuiteGroup = [] + for (const task of suite.tasks) { + if (task.type === 'benchmark') + benchmarkGroup.push(task) + else if (task.type === 'suite') + benchmarkSuiteGroup.push(task) + } + + if (benchmarkSuiteGroup.length) + await Promise.all(benchmarkSuiteGroup.map(subSuite => runBenchmarkSuit(subSuite))) + + if (benchmarkGroup.length) { + const benchmarkInstance = await getBenchmarkFactory(suite) + const defer = createDefer() + const benchmarkMap: Record = {} + suite.result = { + state: 'run', + startTime: start, + benchmark: createBenchmarkResult(suite.name), + } + updateTask(suite) + benchmarkGroup.forEach((benchmark, idx) => { + const benchmarkFn = getFn(benchmark) + benchmark.result = { + state: 'run', + startTime: start, + benchmark: createBenchmarkResult(benchmark.name), + } + const id = idx.toString() + benchmarkMap[id] = benchmark + benchmarkInstance.add(id, benchmarkFn) + updateTask(benchmark) + }) + benchmarkInstance.addEventListener('cycle', (e) => { + const task = e.task + const benchmark = benchmarkMap[task.name || ''] + if (benchmark) { + const taskRes = task.result! + const result = benchmark.result!.benchmark! + Object.assign(result, taskRes) + updateTask(benchmark) + } + }) + + benchmarkInstance.addEventListener('complete', () => { + suite.result!.duration = performance.now() - start + suite.result!.state = 'pass' + + benchmarkInstance.tasks + .sort((a, b) => b.result!.mean - a.result!.mean) + .forEach((cycle, idx) => { + const benchmark = benchmarkMap[cycle.name || ''] + benchmark.result!.state = 'pass' + if (benchmark) { + const result = benchmark.result!.benchmark! + result.rank = Number(idx) + 1 + updateTask(benchmark) + } + }) + updateTask(suite) + defer.resolve(null) + }) + + benchmarkInstance.addEventListener('error', (e) => { + defer.reject(e) + }) + + benchmarkInstance.run() + await defer + } +} + async function runSuiteChild(c: Task) { - return c.type === 'test' - ? runTest(c) - : runSuite(c) + if (c.type === 'test') + return runTest(c) + + else if (c.type === 'suite') + return runSuite(c) } async function runSuites(suites: Suite[]) { @@ -301,7 +393,6 @@ export async function runFiles(files: File[], config: ResolvedConfig) { } } } - await runSuite(file) } } diff --git a/packages/vitest/src/runtime/suite.ts b/packages/vitest/src/runtime/suite.ts index 6d10bf352a49..b41b855ea1dd 100644 --- a/packages/vitest/src/runtime/suite.ts +++ b/packages/vitest/src/runtime/suite.ts @@ -1,6 +1,6 @@ import util from 'util' -import type { File, RunMode, Suite, SuiteAPI, SuiteCollector, SuiteFactory, SuiteHooks, Task, Test, TestAPI, TestFunction, TestOptions } from '../types' -import { getWorkerState, isObject, noop } from '../utils' +import type { BenchFunction, BenchOptions, Benchmark, BenchmarkAPI, File, RunMode, Suite, SuiteAPI, SuiteCollector, SuiteFactory, SuiteHooks, Task, Test, TestAPI, TestFunction, TestOptions } from '../types' +import { getWorkerState, isObject, isRunningInBenchmark, isRunningInTest, noop } from '../utils' import { createChainable } from './chain' import { collectTask, collectorContext, createTestContext, runWithSuite, withTimeout } from './context' import { getHooks, setFn, setHooks } from './map' @@ -9,11 +9,16 @@ import { getHooks, setFn, setHooks } from './map' export const suite = createSuite() export const test = createTest( function (name: string, fn?: TestFunction, options?: number | TestOptions) { - // @ts-expect-error untyped internal prop getCurrentSuite().test.fn.call(this, name, fn, options) }, ) +export const bench = createBenchmark( + function (name, fn, options) { + getCurrentSuite().benchmark.fn.call(this, name, fn, options) + }, +) + function formatTitle(template: string, items: any[], idx: number) { if (template.includes('%#')) { // '%#' match index of the test case @@ -64,7 +69,7 @@ export function createSuiteHooks() { } function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, mode: RunMode, concurrent?: boolean, shuffle?: boolean) { - const tasks: (Test | Suite | SuiteCollector)[] = [] + const tasks: (Benchmark | Test | Suite | SuiteCollector)[] = [] const factoryQueue: (Test | Suite | SuiteCollector)[] = [] let suite: Suite @@ -72,6 +77,9 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m initSuite() const test = createTest(function (name: string, fn = noop, options?: number | TestOptions) { + if (!isRunningInTest()) + throw new Error('`test()` and `it()` is only available in test mode.') + const mode = this.only ? 'only' : this.skip ? 'skip' : this.todo ? 'todo' : 'run' if (typeof options === 'number') @@ -107,12 +115,32 @@ function createSuiteCollector(name: string, factory: SuiteFactory = () => { }, m tasks.push(test) }) + const benchmark = createBenchmark(function (name: string, fn = noop, options: BenchOptions) { + const mode = this.skip ? 'skip' : 'run' + + if (!isRunningInBenchmark()) + throw new Error('`bench()` is only available in benchmark mode. Run with `vitest bench` instead.') + + const benchmark: Benchmark = { + type: 'benchmark', + id: '', + name, + mode, + options, + suite: undefined!, + } + + setFn(benchmark, fn) + tasks.push(benchmark) + }) + const collector: SuiteCollector = { type: 'collector', name, mode, test, tasks, + benchmark, collect, clear, on: addHook, @@ -221,3 +249,22 @@ function createTest(fn: ( testFn, ) as TestAPI } + +function createBenchmark(fn: ( + ( + this: Record<'skip', boolean | undefined>, + name: string, + fn: BenchFunction, + options: BenchOptions + ) => void +)) { + const benchmark = createChainable( + ['skip'], + fn, + ) as BenchmarkAPI + + benchmark.skipIf = (condition: any) => (condition ? benchmark.skip : benchmark) as BenchmarkAPI + benchmark.runIf = (condition: any) => (condition ? benchmark : benchmark.skip) as BenchmarkAPI + + return benchmark as BenchmarkAPI +} diff --git a/packages/vitest/src/types/benchmark.ts b/packages/vitest/src/types/benchmark.ts new file mode 100644 index 000000000000..9d8a1af98e88 --- /dev/null +++ b/packages/vitest/src/types/benchmark.ts @@ -0,0 +1,62 @@ +import type { Bench as BenchFactory, Options as BenchOptions, Task as BenchTask, TaskResult as BenchTaskResult, TaskResult as TinybenchResult } from 'tinybench' +import type { BenchmarkBuiltinReporters } from '../node/reporters' +import type { ChainableFunction } from '../runtime/chain' +import type { Arrayable, Reporter, Suite, TaskBase, TaskResult } from '.' + +export interface BenchmarkUserOptions { + /** + * Include globs for benchmark test files + * + * @default ['**\/*.{bench,benchmark}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'] + */ + include?: string[] + + /** + * Exclude globs for benchmark test files + * @default ['node_modules', 'dist', '.idea', '.git', '.cache'] + */ + exclude?: string[] + + /** + * Include globs for in-source test files + * + * @default [] + */ + includeSource?: string[] + + /** + * Custom reporter for output. Can contain one or more built-in report names, reporter instances, + * and/or paths to custom reporters + */ + reporters?: Arrayable +} + +export interface Benchmark extends TaskBase { + type: 'benchmark' + suite: Suite + result?: TaskResult + fails?: boolean + options: BenchOptions +} + +export interface BenchmarkResult extends TinybenchResult { + name: string + rank: number +} + +export type BenchFunction = (this: BenchFactory) => Promise | void +export type BenchmarkAPI = ChainableFunction< +'skip', +[name: string, fn: BenchFunction, options?: BenchOptions], +void +> & { + skipIf(condition: any): BenchmarkAPI + runIf(condition: any): BenchmarkAPI +} + +export { + BenchTaskResult, + BenchOptions, + BenchFactory, + BenchTask, +} diff --git a/packages/vitest/src/types/config.ts b/packages/vitest/src/types/config.ts index b588aa514bcb..41506769500a 100644 --- a/packages/vitest/src/types/config.ts +++ b/packages/vitest/src/types/config.ts @@ -8,6 +8,7 @@ import type { JSDOMOptions } from './jsdom-options' import type { Reporter } from './reporter' import type { SnapshotStateOptions } from './snapshot' import type { Arrayable } from './general' +import type { BenchmarkUserOptions } from './benchmark' export type BuiltinEnvironment = 'node' | 'jsdom' | 'happy-dom' | 'edge-runtime' // Record is used, so user can get intellisense for builtin environments, but still allow custom environments @@ -26,7 +27,16 @@ export interface EnvironmentOptions { [x: string]: unknown } +export type VitestRunMode = 'test' | 'benchmark' + export interface InlineConfig { + /** + * Benchmark options. + * + * @default {} + */ + benchmark?: BenchmarkUserOptions + /** * Include globs for test files * @@ -479,7 +489,9 @@ export interface UserConfig extends InlineConfig { shard?: string } -export interface ResolvedConfig extends Omit, 'config' | 'filters' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'shard' | 'cache' | 'sequence'> { +export interface ResolvedConfig extends Omit, 'config' | 'filters' | 'coverage' | 'testNamePattern' | 'related' | 'api' | 'reporters' | 'resolveSnapshotPath' | 'benchmark' | 'shard' | 'cache' | 'sequence'> { + mode: VitestRunMode + base?: string config?: string @@ -495,6 +507,9 @@ export interface ResolvedConfig extends Omit, 'config' | 'f defines: Record api?: ApiConfig + + benchmark?: Required + shard?: { index: number count: number diff --git a/packages/vitest/src/types/index.ts b/packages/vitest/src/types/index.ts index 6635ce313c97..5eaeef90cd0d 100644 --- a/packages/vitest/src/types/index.ts +++ b/packages/vitest/src/types/index.ts @@ -8,7 +8,7 @@ export * from './snapshot' export * from './worker' export * from './general' export * from './coverage' - +export * from './benchmark' export type { EnhancedSpy, MockedFunction, diff --git a/packages/vitest/src/types/tasks.ts b/packages/vitest/src/types/tasks.ts index 61cecf9c95bb..3d5701872cba 100644 --- a/packages/vitest/src/types/tasks.ts +++ b/packages/vitest/src/types/tasks.ts @@ -1,6 +1,7 @@ import type { ChainableFunction } from '../runtime/chain' +import type { BenchFactory } from './benchmark' import type { Awaitable, ErrorWithDiff } from './general' -import type { UserConsoleLog } from '.' +import type { Benchmark, BenchmarkAPI, BenchmarkResult, UserConsoleLog } from '.' export type RunMode = 'run' | 'skip' | 'only' | 'todo' export type TaskState = RunMode | 'pass' | 'fail' @@ -26,6 +27,7 @@ export interface TaskResult { error?: ErrorWithDiff htmlError?: string hooks?: Partial> + benchmark?: BenchmarkResult retryCount?: number } @@ -35,6 +37,7 @@ export interface Suite extends TaskBase { type: 'suite' tasks: Task[] filepath?: string + benchmark?: BenchFactory } export interface File extends Suite { @@ -51,7 +54,7 @@ export interface Test extends TaskBase { context: TestContext & ExtraContext } -export type Task = Test | Suite | File +export type Task = Test | Suite | File | Benchmark export type DoneCallback = (error?: any) => void export type TestFunction = (context: TestContext & ExtraContext) => Awaitable | void @@ -186,7 +189,8 @@ export interface SuiteCollector { readonly mode: RunMode type: 'collector' test: TestAPI - tasks: (Suite | Test | SuiteCollector)[] + benchmark: BenchmarkAPI + tasks: (Suite | Test | Benchmark | SuiteCollector)[] collect: (file?: File) => Promise clear: () => void on: >(name: T, ...fn: SuiteHooks[T]) => void diff --git a/packages/vitest/src/utils/config-helpers.ts b/packages/vitest/src/utils/config-helpers.ts index 4af0199c4304..a1574f1198f9 100644 --- a/packages/vitest/src/utils/config-helpers.ts +++ b/packages/vitest/src/utils/config-helpers.ts @@ -1,7 +1,7 @@ import type { Vitest } from '../node/core' -import type { BuiltinReporters } from '../node/reporters' +import type { BenchmarkBuiltinReporters, BuiltinReporters } from '../node/reporters' -export const getOutputFile = ({ config }: Vitest, reporter: BuiltinReporters) => { +export const getOutputFile = ({ config }: Vitest, reporter: BuiltinReporters | BenchmarkBuiltinReporters) => { if (!config.outputFile) return diff --git a/packages/vitest/src/utils/index.ts b/packages/vitest/src/utils/index.ts index 58b2f7651ea9..ff754e8aafdc 100644 --- a/packages/vitest/src/utils/index.ts +++ b/packages/vitest/src/utils/index.ts @@ -6,6 +6,7 @@ import { relative as relativeNode } from 'pathe' import type { ModuleCacheMap } from 'vite-node' import type { Suite, Task } from '../types' import { EXIT_CODE_RESTART } from '../constants' +import { getWorkerState } from '../utils' import { getNames } from './tasks' export * from './tasks' @@ -17,6 +18,9 @@ export const isNode = typeof process < 'u' && typeof process.stdout < 'u' && !pr // export const isNode = typeof process !== 'undefined' && typeof process.platform !== 'undefined' export const isBrowser = typeof window !== 'undefined' export const isWindows = isNode && process.platform === 'win32' +export const getRunMode = () => getWorkerState().config.mode +export const isRunningInTest = () => getRunMode() === 'test' +export const isRunningInBenchmark = () => getRunMode() === 'benchmark' export const relativePath = isBrowser ? relativeBrowser : relativeNode @@ -150,3 +154,22 @@ class AggregateErrorPonyfill extends Error { } } export { AggregateErrorPonyfill as AggregateError } + +type DeferPromise = Promise & { + resolve: (value: T | PromiseLike) => void + reject: (reason?: any) => void +} + +export function createDefer(): DeferPromise { + let resolve: ((value: T | PromiseLike) => void) | null = null + let reject: ((reason?: any) => void) | null = null + + const p = new Promise((_resolve, _reject) => { + resolve = _resolve + reject = _reject + }) as DeferPromise + + p.resolve = resolve! + p.reject = reject! + return p +} diff --git a/packages/vitest/src/utils/tasks.ts b/packages/vitest/src/utils/tasks.ts index 43f12e137944..ee56ef0aac31 100644 --- a/packages/vitest/src/utils/tasks.ts +++ b/packages/vitest/src/utils/tasks.ts @@ -1,12 +1,16 @@ -import type { Arrayable, Suite, Task, Test } from '../types' +import type { Arrayable, Benchmark, Suite, Task, Test } from '../types' import { toArray } from './base' -export function getTests(suite: Arrayable): Test[] { - return toArray(suite).flatMap(s => s.type === 'test' ? [s] : s.tasks.flatMap(c => c.type === 'test' ? [c] : getTests(c))) +function isAtomTest(s: Task): s is Test | Benchmark { + return (s.type === 'test' || s.type === 'benchmark') +} + +export function getTests(suite: Arrayable): (Test | Benchmark)[] { + return toArray(suite).flatMap(s => isAtomTest(s) ? [s] : s.tasks.flatMap(c => isAtomTest(c) ? [c] : getTests(c))) } export function getTasks(tasks: Arrayable = []): Task[] { - return toArray(tasks).flatMap(s => s.type === 'test' ? [s] : [s, ...getTasks(s.tasks)]) + return toArray(tasks).flatMap(s => isAtomTest(s) ? [s] : [s, ...getTasks(s.tasks)]) } export function getSuites(suite: Arrayable): Suite[] { @@ -14,7 +18,11 @@ export function getSuites(suite: Arrayable): Suite[] { } export function hasTests(suite: Arrayable): boolean { - return toArray(suite).some(s => s.tasks.some(c => c.type === 'test' || hasTests(c as Suite))) + return toArray(suite).some(s => s.tasks.some(c => isAtomTest(c) || hasTests(c))) +} + +export function hasBenchmark(suite: Arrayable): boolean { + return toArray(suite).some(s => s?.tasks?.some(c => c.type === 'benchmark' || hasBenchmark(c as Suite))) } export function hasFailed(suite: Arrayable): boolean { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 69050cbb1709..9dbb0fb174cd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -61,7 +61,7 @@ importers: '@rollup/plugin-node-resolve': 13.3.0_rollup@2.77.3 '@types/fs-extra': 9.0.13 '@types/lodash': 4.14.184 - '@types/node': 18.7.14 + '@types/node': 18.7.13 '@types/ws': 8.5.3 '@vitest/browser': link:packages/browser '@vitest/coverage-c8': link:packages/coverage-c8 @@ -88,7 +88,7 @@ importers: rollup-plugin-esbuild: 4.10.1_l5nt6dc6big2rec47wcz6gi7du rollup-plugin-license: 2.8.1_rollup@2.77.3 simple-git-hooks: 2.8.0 - ts-node: 10.9.1_tphhiizkxv2hzwkunblc3hbmra + ts-node: 10.9.1_hwinnrf7y5nyyzygpj45jmvjia tsup: 6.2.3_s5ojjbx2isjkawqptqpitvy25q typescript: 4.8.2 vite: 3.0.9 @@ -126,7 +126,7 @@ importers: unocss: 0.45.13_vite@3.0.9 unplugin-vue-components: 0.22.4_vite@3.0.9+vue@3.2.38 vite: 3.0.9 - vite-plugin-pwa: 0.12.3_f7se6o6eqkwcix4u3svh6mkvda + vite-plugin-pwa: 0.12.3_vite@3.0.9 vitepress: 1.0.0-alpha.13 workbox-window: 6.5.4 @@ -550,7 +550,7 @@ importers: vue: 3.2.38 devDependencies: '@vitejs/plugin-vue': 3.0.3_vite@3.0.9+vue@3.2.38 - '@vue/test-utils': 2.0.2_vue@3.2.38 + '@vue/test-utils': 2.0.0_vue@3.2.38 jsdom: 20.0.0 vite: 3.0.9 vitest: link:../../packages/vitest @@ -767,6 +767,7 @@ importers: cli-truncate: ^3.1.0 debug: ^4.3.4 diff: ^5.1.0 + event-target-polyfill: ^0.0.3 execa: ^6.1.0 fast-glob: ^3.2.11 find-up: ^6.3.0 @@ -789,6 +790,7 @@ importers: source-map-js: ^1.0.2 strip-ansi: ^7.0.1 strip-literal: ^0.4.0 + tinybench: ^2.1.3 tinypool: ^0.2.4 tinyspy: ^1.0.2 typescript: ^4.8.2 @@ -798,11 +800,12 @@ importers: dependencies: '@types/chai': 4.3.3 '@types/chai-subset': 1.3.3 - '@types/node': 18.7.14 + '@types/node': 18.7.13 chai: 4.3.6 debug: 4.3.4 local-pkg: 0.4.2 strip-literal: 0.4.0 + tinybench: 2.1.3 tinypool: 0.2.4 tinyspy: 1.0.2 vite: 3.0.9 @@ -822,6 +825,7 @@ importers: chai-subset: 1.6.0 cli-truncate: 3.1.0 diff: 5.1.0 + event-target-polyfill: 0.0.3 execa: 6.1.0 fast-glob: 3.2.11 find-up: 6.3.0 @@ -874,6 +878,9 @@ importers: devDependencies: vitest: link:../../packages/vitest + test/benchmark: + specifiers: {} + test/browser: specifiers: '@vitest/browser': workspace:* @@ -3811,7 +3818,7 @@ packages: engines: {node: '>=14'} hasBin: true dependencies: - '@types/node': 18.7.14 + '@types/node': 18.7.13 playwright-core: 1.25.1 dev: true @@ -5576,7 +5583,7 @@ packages: /@types/cheerio/0.22.31: resolution: {integrity: sha512-Kt7Cdjjdi2XWSfrZ53v4Of0wG3ZcmaegFXjMmz9tfNrZSkzzo36G0AL1YqSdcIA78Etjt6E609pt5h1xnQkPUw==} dependencies: - '@types/node': 18.7.14 + '@types/node': 18.7.13 dev: true /@types/codemirror/5.60.5: @@ -5654,14 +5661,14 @@ packages: /@types/fs-extra/9.0.13: resolution: {integrity: sha512-nEnwB++1u5lVDM2UI4c1+5R+FYaKfaAzS4OococimjVm3nQw3TuzH5UNsocrcTBbhnerblyHj4A49qXbIiZdpA==} dependencies: - '@types/node': 18.7.14 + '@types/node': 18.7.13 dev: true /@types/glob/7.2.0: resolution: {integrity: sha512-ZUxbzKl0IfJILTS6t7ip5fQQM/J3TJYubDm3nMbgubNNYS62eXeUpoLUC8/7fJNiFYHTrGPQn7hspDUzIHX3UA==} dependencies: '@types/minimatch': 5.1.1 - '@types/node': 18.7.14 + '@types/node': 18.7.13 dev: true /@types/glob/8.0.0: @@ -5795,8 +5802,12 @@ packages: resolution: {integrity: sha512-aFcUkv7EddxxOa/9f74DINReQ/celqH8DiB3fRYgVDM2Xm5QJL8sl80QKuAnGvwAsMn+H3IFA6WCrQh1CY7m1A==} dev: true + /@types/node/18.7.13: + resolution: {integrity: sha512-46yIhxSe5xEaJZXWdIBP7GU4HDTG8/eo0qd9atdiL+lFpA03y8KS+lkTN834TWJj5767GbWv4n/P6efyTFt1Dw==} + /@types/node/18.7.14: resolution: {integrity: sha512-6bbDaETVi8oyIARulOE9qF1/Qdi/23z6emrUh0fNJRUmjznqrixD4MpGDdgOFk5Xb0m2H6Xu42JGdvAxaJR/wA==} + dev: true /@types/node/8.10.66: resolution: {integrity: sha512-tktOkFUA4kXx2hhhrB8bIFb5TbwzS4uOhKEmwiD+NoiL0qtP2OQ9mFldbgD4dV1djrlBYP6eBuQZiWjuHUpqFw==} @@ -5893,7 +5904,7 @@ packages: /@types/set-cookie-parser/2.4.2: resolution: {integrity: sha512-fBZgytwhYAUkj/jC/FAV4RQ5EerRup1YQsXQCh8rZfiHkc4UahC192oH0smGwsXol3cL3A5oETuAHeQHmhXM4w==} dependencies: - '@types/node': 18.7.14 + '@types/node': 18.7.13 dev: true /@types/sinonjs__fake-timers/8.1.1: @@ -5982,7 +5993,7 @@ packages: /@types/ws/8.5.3: resolution: {integrity: sha512-6YOoWjruKj1uLf3INHH7D3qTXwFfEsg1kf3c0uDdSBJwfa/llkwIjrAGV7j7mVgGNbzTQ3HiHKKDXl6bJPD97w==} dependencies: - '@types/node': 18.7.14 + '@types/node': 18.7.13 dev: true /@types/yargs-parser/21.0.0: @@ -6005,7 +6016,7 @@ packages: resolution: {integrity: sha512-Cn6WYCm0tXv8p6k+A8PvbDG763EDpBoTzHdA+Q/MF6H3sapGjCm9NzoaJncJS9tUKSuCoDs9XHxYYsQDgxR6kw==} requiresBuild: true dependencies: - '@types/node': 18.7.14 + '@types/node': 18.7.13 dev: true optional: true @@ -6512,6 +6523,14 @@ packages: vue-template-compiler: 2.7.10 dev: true + /@vue/test-utils/2.0.0_vue@3.2.38: + resolution: {integrity: sha512-zL5kygNq7hONrO1CzaUGprEAklAX+pH8J1MPMCU3Rd2xtSYkZ+PmKU3oEDRg8VAGdL5lNJHzDgrud5amFPtirw==} + peerDependencies: + vue: ^3.0.1 + dependencies: + vue: 3.2.38 + dev: true + /@vue/test-utils/2.0.2_vue@3.2.38: resolution: {integrity: sha512-E2P4oXSaWDqTZNbmKZFVLrNN/siVN78YkEqs7pHryWerrlZR9bBFLWdJwRoguX45Ru6HxIflzKl4vQvwRMwm5g==} peerDependencies: @@ -6543,7 +6562,7 @@ packages: '@types/web-bluetooth': 0.0.15 '@vueuse/metadata': 9.1.1 '@vueuse/shared': 9.1.1_vue@3.2.38 - vue-demi: 0.13.11_vue@3.2.38 + vue-demi: 0.12.5_vue@3.2.38 transitivePeerDependencies: - '@vue/composition-api' - vue @@ -8956,7 +8975,7 @@ packages: dayjs: 1.11.5 debug: 4.3.4_supports-color@8.1.1 enquirer: 2.3.6 - eventemitter2: 6.4.7 + eventemitter2: 6.4.5 execa: 4.1.0 executable: 4.1.1 extract-zip: 2.0.1_supports-color@8.1.1 @@ -10520,8 +10539,12 @@ packages: engines: {node: '>= 0.6'} dev: true - /eventemitter2/6.4.7: - resolution: {integrity: sha512-tYUSVOGeQPKt/eC1ABfhHy5Xd96N3oIijJvN3O9+TsC28T5V9yX9oEfEK5faP0EFSNVOG97qtAS68GBrQB2hDg==} + /event-target-polyfill/0.0.3: + resolution: {integrity: sha512-ZMc6UuvmbinrCk4RzGyVmRyIsAyxMRlp4CqSrcQRO8Dy0A9ldbiRy5kdtBj4OtP7EClGdqGfIqo9JmOClMsGLQ==} + dev: true + + /eventemitter2/6.4.5: + resolution: {integrity: sha512-bXE7Dyc1i6oQElDG0jMRZJrRAn9QR2xyyFGmBdZleNmyQX0FqGYmhZIrIrpPfm/w//LTo4tVQGOGQcGCb5q9uw==} dev: true /eventemitter3/4.0.7: @@ -14925,7 +14948,7 @@ packages: optional: true dependencies: lilconfig: 2.0.6 - ts-node: 10.9.1_tphhiizkxv2hzwkunblc3hbmra + ts-node: 10.9.1_hwinnrf7y5nyyzygpj45jmvjia yaml: 1.10.2 dev: true @@ -17404,6 +17427,11 @@ packages: setimmediate: 1.0.5 dev: true + /tinybench/2.1.3: + resolution: {integrity: sha512-HXOcRGMD35vulAdRRXCKceLcws6aKfIZAfNTraX24XyK1SmcxJjWW1rkgmET+irNvmXBrNeKrcJh16XcvF9q4Q==} + engines: {node: '>=16.0.0'} + dev: false + /tinypool/0.2.4: resolution: {integrity: sha512-Vs3rhkUH6Qq1t5bqtb816oT+HeJTXfwt2cbPH17sWHIYKTotQIFPk3tf2fgqRrVyMDVOc1EnPgzIxfIulXVzwQ==} engines: {node: '>=14.0.0'} @@ -17558,7 +17586,7 @@ packages: tslib: 2.4.0 dev: false - /ts-node/10.9.1_tphhiizkxv2hzwkunblc3hbmra: + /ts-node/10.9.1_hwinnrf7y5nyyzygpj45jmvjia: resolution: {integrity: sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==} hasBin: true peerDependencies: @@ -17577,7 +17605,7 @@ packages: '@tsconfig/node12': 1.0.11 '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.3 - '@types/node': 18.7.14 + '@types/node': 18.7.13 acorn: 8.8.0 acorn-walk: 8.2.0 arg: 4.1.3 @@ -18397,11 +18425,10 @@ packages: - supports-color dev: true - /vite-plugin-pwa/0.12.3_f7se6o6eqkwcix4u3svh6mkvda: + /vite-plugin-pwa/0.12.3_vite@3.0.9: resolution: {integrity: sha512-gmYdIVXpmBuNjzbJFPZFzxWYrX4lHqwMAlOtjmXBbxApiHjx9QPXKQPJjSpeTeosLKvVbNcKSAAhfxMda0QVNQ==} peerDependencies: vite: ^2.0.0 || ^3.0.0-0 - workbox-window: ^6.4.0 dependencies: debug: 4.3.4 fast-glob: 3.2.11 @@ -18506,6 +18533,20 @@ packages: resolution: {integrity: sha512-gu73tuZfJgu+mvCSy4UZwd2JXykjK9zAZsfmDeut5dx/1a7FeTk0XwJsSuqQn+cuMCGVbIBfl+s53X4T19DnzQ==} dev: true + /vue-demi/0.12.5_vue@3.2.38: + resolution: {integrity: sha512-BREuTgTYlUr0zw0EZn3hnhC3I6gPWv+Kwh4MCih6QcAeaTlaIX0DwOVN0wHej7hSvDPecz4jygy/idsgKfW58Q==} + engines: {node: '>=12'} + hasBin: true + requiresBuild: true + peerDependencies: + '@vue/composition-api': ^1.0.0-rc.1 + vue: ^3.0.0-0 || ^2.6.0 + peerDependenciesMeta: + '@vue/composition-api': + optional: true + dependencies: + vue: 3.2.38 + /vue-demi/0.13.11: resolution: {integrity: sha512-IR8HoEEGM65YY3ZJYAjMlKygDQn25D5ajNFNoKh9RSDMQtlzCxtfQjdQgv9jjK+m3377SsJXY8ysq8kLCZL25A==} engines: {node: '>=12'} diff --git a/test/benchmark/package.json b/test/benchmark/package.json new file mode 100644 index 000000000000..9e0def28981a --- /dev/null +++ b/test/benchmark/package.json @@ -0,0 +1,9 @@ +{ + "name": "@vitest/benchmark", + "private": true, + "scripts": { + "test": "vitest bench", + "testu": "vitest -u", + "coverage": "vitest run --coverage" + } +} diff --git a/test/benchmark/test/base.bench.ts b/test/benchmark/test/base.bench.ts new file mode 100644 index 000000000000..421cde41c6f3 --- /dev/null +++ b/test/benchmark/test/base.bench.ts @@ -0,0 +1,52 @@ +import { bench, describe } from 'vitest' + +describe('sort', () => { + bench('normal', () => { + const x = [1, 5, 4, 2, 3] + x.sort((a, b) => { + return a - b + }) + }) + + bench('reverse', () => { + const x = [1, 5, 4, 2, 3] + x.reverse().sort((a, b) => { + return a - b + }) + }) + + // TODO: move to failed tests + // should not be collect + // it('test', () => { + // expect(1 + 1).toBe(3) + // }) +}) + +function timeout(time: number) { + return new Promise((resolve) => { + setTimeout(resolve, time) + }) +} + +describe('timeout', () => { + bench('timeout100', async () => { + await timeout(100) + }) + + bench('timeout75', async () => { + await timeout(75) + }) + + bench('timeout50', async () => { + await timeout(50) + }) + + bench('timeout25', async () => { + await timeout(25) + }) + + // TODO: move to failed tests + // test('reduce', () => { + // expect(1 - 1).toBe(2) + // }) +}) diff --git a/test/benchmark/vitest.config.ts b/test/benchmark/vitest.config.ts new file mode 100644 index 000000000000..550650171f9c --- /dev/null +++ b/test/benchmark/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from 'vitest/config' + +export default defineConfig({ + test: { + benchmark: { + reporters: 'json', + }, + }, +}) diff --git a/test/css/testing.mjs b/test/css/testing.mjs index cce302cac228..e3e2202cc300 100644 --- a/test/css/testing.mjs +++ b/test/css/testing.mjs @@ -10,7 +10,7 @@ const configs = [ async function runTests() { for (const [name, config] of configs) { - const success = await startVitest([name], { + const success = await startVitest('test', [name], { run: true, css: config, update: false,