From fbe01771a1fda6506a1c63987c7e939f40e3f1f6 Mon Sep 17 00:00:00 2001 From: Simen Bekkhus Date: Fri, 6 Jan 2023 15:37:43 +0100 Subject: [PATCH 1/6] feat: use `os.availableParallelism` if available --- CHANGELOG.md | 3 ++- packages/jest-config/src/getMaxWorkers.ts | 12 +++++++++--- packages/jest-repl/src/cli/runtime-cli.ts | 13 +++++++++++-- packages/jest-worker/src/index.ts | 15 +++++++++++++-- scripts/buildTs.mjs | 7 ++++++- scripts/lintTs.mjs | 8 +++++++- 6 files changed, 48 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1984784c897e..b6ada800599f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,9 @@ ### Features -- `[@jest/globals, jest-mock]` Add `jest.replaceProperty()` that replaces property value ([#13496](https://github.com/facebook/jest/pull/13496)) - `[expect, @jest/expect-utils]` Support custom equality testers ([#13654](https://github.com/facebook/jest/pull/13654)) +- `[jest-config, jest-worker]` Use `os.availableParallelism` if available to calculate number of workers to spawn +- `[@jest/globals, jest-mock]` Add `jest.replaceProperty()` that replaces property value ([#13496](https://github.com/facebook/jest/pull/13496)) - `[jest-haste-map]` ignore Sapling vcs directories (`.sl/`) ([#13674](https://github.com/facebook/jest/pull/13674)) - `[jest-resolve]` Support subpath imports ([#13705](https://github.com/facebook/jest/pull/13705), [#13723](https://github.com/facebook/jest/pull/13723)) - `[jest-runtime]` Add `jest.isolateModulesAsync` for scoped module initialization of asynchronous functions ([#13680](https://github.com/facebook/jest/pull/13680)) diff --git a/packages/jest-config/src/getMaxWorkers.ts b/packages/jest-config/src/getMaxWorkers.ts index 8ccb1ce54a61..9a2c5496666f 100644 --- a/packages/jest-config/src/getMaxWorkers.ts +++ b/packages/jest-config/src/getMaxWorkers.ts @@ -5,7 +5,11 @@ * LICENSE file in the root directory of this source tree. */ -import {cpus} from 'os'; +import { + // @ts-expect-error - added in Node 19.4.0 + availableParallelism, + cpus, +} from 'os'; import type {Config} from '@jest/types'; export default function getMaxWorkers( @@ -22,8 +26,10 @@ export default function getMaxWorkers( return parseWorkers(defaultOptions.maxWorkers); } else { // In watch mode, Jest should be unobtrusive and not use all available CPUs. - const cpusInfo = cpus(); - const numCpus = cpusInfo?.length ?? 1; + const numCpus: number = + typeof availableParallelism === 'function' + ? availableParallelism() + : cpus().length; const isWatchModeEnabled = argv.watch || argv.watchAll; return Math.max( isWatchModeEnabled ? Math.floor(numCpus / 2) : numCpus - 1, diff --git a/packages/jest-repl/src/cli/runtime-cli.ts b/packages/jest-repl/src/cli/runtime-cli.ts index ad772abf47a7..d0f8d9d5d353 100644 --- a/packages/jest-repl/src/cli/runtime-cli.ts +++ b/packages/jest-repl/src/cli/runtime-cli.ts @@ -5,7 +5,11 @@ * LICENSE file in the root directory of this source tree. */ -import {cpus} from 'os'; +import { + // @ts-expect-error - added in Node 19.4.0 + availableParallelism, + cpus, +} from 'os'; import * as path from 'path'; import * as util from 'util'; import chalk = require('chalk'); @@ -70,8 +74,13 @@ export async function run( }; try { + const numCpus: number = + typeof availableParallelism === 'function' + ? availableParallelism() + : cpus().length; + const hasteMap = await Runtime.createContext(projectConfig, { - maxWorkers: Math.max(cpus().length - 1, 1), + maxWorkers: Math.max(numCpus - 1, 1), watchman: globalConfig.watchman, }); diff --git a/packages/jest-worker/src/index.ts b/packages/jest-worker/src/index.ts index 25fb79e55ca2..406bebf044b1 100644 --- a/packages/jest-worker/src/index.ts +++ b/packages/jest-worker/src/index.ts @@ -5,7 +5,11 @@ * LICENSE file in the root directory of this source tree. */ -import {cpus} from 'os'; +import { + // @ts-expect-error - added in Node 19.4.0 + availableParallelism, + cpus, +} from 'os'; import {isAbsolute} from 'path'; import Farm from './Farm'; import WorkerPool from './WorkerPool'; @@ -54,6 +58,12 @@ function getExposedMethods( return exposedMethods; } +function getNumberOfCpus(): number { + return availableParallelism === 'function' + ? availableParallelism() + : cpus().length; +} + /** * The Jest farm (publicly called "Worker") is a class that allows you to queue * methods across multiple child processes, in order to parallelize work. This @@ -98,7 +108,8 @@ export class Worker { forkOptions: this._options.forkOptions ?? {}, idleMemoryLimit: this._options.idleMemoryLimit, maxRetries: this._options.maxRetries ?? 3, - numWorkers: this._options.numWorkers ?? Math.max(cpus().length - 1, 1), + numWorkers: + this._options.numWorkers ?? Math.max(getNumberOfCpus() - 1, 1), resourceLimits: this._options.resourceLimits ?? {}, setupArgs: this._options.setupArgs ?? [], }; diff --git a/scripts/buildTs.mjs b/scripts/buildTs.mjs index 8674d1fe09e0..4deca57131b2 100644 --- a/scripts/buildTs.mjs +++ b/scripts/buildTs.mjs @@ -110,7 +110,12 @@ try { console.log(chalk.inverse(' Validating TypeScript definition files ')); // we want to limit the number of processes we spawn -const cpus = Math.max(1, os.cpus().length - 1); +const cpus = Math.max( + 1, + (os.availableParallelism === 'function' + ? os.availableParallelism() + : os.cpus().length) - 1, +); const typesReferenceDirective = '/// `; diff --git a/scripts/lintTs.mjs b/scripts/lintTs.mjs index 60ef1e7a8f23..8f01cfaf1a1c 100644 --- a/scripts/lintTs.mjs +++ b/scripts/lintTs.mjs @@ -14,7 +14,13 @@ import pLimit from 'p-limit'; import {getPackagesWithTsConfig} from './buildUtils.mjs'; // we want to limit the number of processes we spawn -const cpus = Math.max(1, os.cpus().length - 1); +const cpus = Math.max( + 1, + (os.availableParallelism === 'function' + ? os.availableParallelism() + : os.cpus().length) - 1, +); + const mutex = pLimit(cpus); const fix = process.argv.slice(2).some(arg => arg === '--fix'); From f14600a1c64fff3e90119a61def4010662f71f4e Mon Sep 17 00:00:00 2001 From: Simen Bekkhus Date: Fri, 6 Jan 2023 15:40:05 +0100 Subject: [PATCH 2/6] link in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6ada800599f..c1fb4bc7ae42 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ### Features - `[expect, @jest/expect-utils]` Support custom equality testers ([#13654](https://github.com/facebook/jest/pull/13654)) -- `[jest-config, jest-worker]` Use `os.availableParallelism` if available to calculate number of workers to spawn +- `[jest-config, jest-worker]` Use `os.availableParallelism` if available to calculate number of workers to spawn ([#13738](https://github.com/facebook/jest/pull/13738)) - `[@jest/globals, jest-mock]` Add `jest.replaceProperty()` that replaces property value ([#13496](https://github.com/facebook/jest/pull/13496)) - `[jest-haste-map]` ignore Sapling vcs directories (`.sl/`) ([#13674](https://github.com/facebook/jest/pull/13674)) - `[jest-resolve]` Support subpath imports ([#13705](https://github.com/facebook/jest/pull/13705), [#13723](https://github.com/facebook/jest/pull/13723)) From 9389f14779ef536df0d85a143a977ba283d6542f Mon Sep 17 00:00:00 2001 From: Simen Bekkhus Date: Fri, 6 Jan 2023 15:45:30 +0100 Subject: [PATCH 3/6] typeof --- scripts/buildTs.mjs | 2 +- scripts/lintTs.mjs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/buildTs.mjs b/scripts/buildTs.mjs index 4deca57131b2..2d3dd3c97c07 100644 --- a/scripts/buildTs.mjs +++ b/scripts/buildTs.mjs @@ -112,7 +112,7 @@ console.log(chalk.inverse(' Validating TypeScript definition files ')); // we want to limit the number of processes we spawn const cpus = Math.max( 1, - (os.availableParallelism === 'function' + (typeof os.availableParallelism === 'function' ? os.availableParallelism() : os.cpus().length) - 1, ); diff --git a/scripts/lintTs.mjs b/scripts/lintTs.mjs index 8f01cfaf1a1c..32b4a2a88540 100644 --- a/scripts/lintTs.mjs +++ b/scripts/lintTs.mjs @@ -16,7 +16,7 @@ import {getPackagesWithTsConfig} from './buildUtils.mjs'; // we want to limit the number of processes we spawn const cpus = Math.max( 1, - (os.availableParallelism === 'function' + (typeof os.availableParallelism === 'function' ? os.availableParallelism() : os.cpus().length) - 1, ); From 0205e500de1643125a50b2f1383d663a5732448d Mon Sep 17 00:00:00 2001 From: Simen Bekkhus Date: Fri, 6 Jan 2023 15:51:08 +0100 Subject: [PATCH 4/6] another typeof --- packages/jest-worker/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jest-worker/src/index.ts b/packages/jest-worker/src/index.ts index 406bebf044b1..f85eb63ed88f 100644 --- a/packages/jest-worker/src/index.ts +++ b/packages/jest-worker/src/index.ts @@ -59,7 +59,7 @@ function getExposedMethods( } function getNumberOfCpus(): number { - return availableParallelism === 'function' + return typeof availableParallelism === 'function' ? availableParallelism() : cpus().length; } From 9ff61bc6e3f7219327f7b6cbe4dd97a6d9736418 Mon Sep 17 00:00:00 2001 From: Simen Bekkhus Date: Fri, 6 Jan 2023 16:17:10 +0100 Subject: [PATCH 5/6] another another typeof --- packages/jest-config/src/getMaxWorkers.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/packages/jest-config/src/getMaxWorkers.ts b/packages/jest-config/src/getMaxWorkers.ts index 9a2c5496666f..e40d47e7008d 100644 --- a/packages/jest-config/src/getMaxWorkers.ts +++ b/packages/jest-config/src/getMaxWorkers.ts @@ -12,6 +12,12 @@ import { } from 'os'; import type {Config} from '@jest/types'; +function getNumCpus(): number { + return typeof availableParallelism === 'function' + ? availableParallelism() + : cpus()?.length ?? 1; +} + export default function getMaxWorkers( argv: Partial< Pick @@ -26,10 +32,7 @@ export default function getMaxWorkers( return parseWorkers(defaultOptions.maxWorkers); } else { // In watch mode, Jest should be unobtrusive and not use all available CPUs. - const numCpus: number = - typeof availableParallelism === 'function' - ? availableParallelism() - : cpus().length; + const numCpus = getNumCpus(); const isWatchModeEnabled = argv.watch || argv.watchAll; return Math.max( isWatchModeEnabled ? Math.floor(numCpus / 2) : numCpus - 1, @@ -47,7 +50,7 @@ const parseWorkers = (maxWorkers: string | number): number => { parsed > 0 && parsed <= 100 ) { - const numCpus = cpus().length; + const numCpus = getNumCpus(); const workers = Math.floor((parsed / 100) * numCpus); return Math.max(workers, 1); } From 0d9732f154b1aaaa1a54f711ab1084cd11a4dd07 Mon Sep 17 00:00:00 2001 From: Simen Bekkhus Date: Fri, 6 Jan 2023 16:44:54 +0100 Subject: [PATCH 6/6] add mock for node 19 --- packages/jest-config/src/__mocks__/os.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/jest-config/src/__mocks__/os.js b/packages/jest-config/src/__mocks__/os.js index a17b60dc17e7..d14380735db7 100644 --- a/packages/jest-config/src/__mocks__/os.js +++ b/packages/jest-config/src/__mocks__/os.js @@ -17,4 +17,8 @@ function __setCpus(newCpus) { os.__setCpus = __setCpus; os.cpus = jest.fn(() => cpus); +if (typeof os.availableParallelism === 'function') { + os.availableParallelism = jest.fn(() => cpus?.length ?? 0); +} + module.exports = os;