From 03ba9e391b3b4ae7f251d4b681418d0a0c0ddea2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Fri, 4 Jun 2021 14:59:22 -0400 Subject: [PATCH 1/7] chore: add glob and jest-worker --- package.json | 2 ++ yarn.lock | 18 ++++++++++-------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 25f4f09640e3..21d69a87d176 100644 --- a/package.json +++ b/package.json @@ -51,6 +51,7 @@ "eslint-plugin-prettier": "^3.4.0", "fancy-log": "^1.3.3", "flow-bin": "^0.123.0", + "glob": "^7.1.7", "gulp": "^4.0.2", "gulp-babel": "^8.0.0", "gulp-filter": "^5.1.0", @@ -59,6 +60,7 @@ "gulp-revert-path": "^2.0.0", "husky": "^3.0.0", "jest": "^27.0.0", + "jest-worker": "^27.0.2", "lint-staged": "^9.2.0", "lodash": "^4.17.21", "mergeiterator": "^1.2.5", diff --git a/yarn.lock b/yarn.lock index 28b0fabd1038..74f808a9dc72 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5763,6 +5763,7 @@ __metadata: eslint-plugin-prettier: ^3.4.0 fancy-log: ^1.3.3 flow-bin: ^0.123.0 + glob: ^7.1.7 gulp: ^4.0.2 gulp-babel: ^8.0.0 gulp-filter: ^5.1.0 @@ -5771,6 +5772,7 @@ __metadata: gulp-revert-path: ^2.0.0 husky: ^3.0.0 jest: ^27.0.0 + jest-worker: ^27.0.2 lint-staged: ^9.2.0 lodash: ^4.17.21 mergeiterator: ^1.2.5 @@ -8835,9 +8837,9 @@ fsevents@^1.2.7: languageName: node linkType: hard -"glob@npm:^7.0.0, glob@npm:^7.0.3, glob@npm:^7.1.0, glob@npm:^7.1.1, glob@npm:^7.1.2, glob@npm:^7.1.3, glob@npm:^7.1.4, glob@npm:^7.1.6": - version: 7.1.6 - resolution: "glob@npm:7.1.6" +"glob@npm:^7.0.0, glob@npm:^7.0.3, glob@npm:^7.1.0, glob@npm:^7.1.1, glob@npm:^7.1.2, glob@npm:^7.1.3, glob@npm:^7.1.4, glob@npm:^7.1.6, glob@npm:^7.1.7": + version: 7.1.7 + resolution: "glob@npm:7.1.7" dependencies: fs.realpath: ^1.0.0 inflight: ^1.0.4 @@ -8845,7 +8847,7 @@ fsevents@^1.2.7: minimatch: ^3.0.4 once: ^1.3.0 path-is-absolute: ^1.0.0 - checksum: 789977b52432865bd63846da5c75a6efc2c56abdc0cb5ffcdb8e91eeb67a58fa5594c1195d18b2b4aff99675b0739ed6bd61024b26562e0cca18c8f993efdc82 + checksum: 352f74f08247db5420161a2f68f2bd84b53228b5fcfc9dcc37cd54d3f19ec0232495d84aeff1286d0727059e9fdc1031400e00b971bdc59e30f8f82b199c9d02 languageName: node linkType: hard @@ -10530,14 +10532,14 @@ fsevents@^1.2.7: languageName: node linkType: hard -"jest-worker@npm:^27.0.1": - version: 27.0.1 - resolution: "jest-worker@npm:27.0.1" +"jest-worker@npm:^27.0.1, jest-worker@npm:^27.0.2": + version: 27.0.2 + resolution: "jest-worker@npm:27.0.2" dependencies: "@types/node": "*" merge-stream: ^2.0.0 supports-color: ^8.0.0 - checksum: c99fcfd6bf2ef0c9fef30e08fb00d7fe951be84d2cff37a3d8983d683857b150a7ac60829ed30893c69277a1d74b57fc2b8aae83ded208fd574985fd9b04162f + checksum: bfbfd3d0af94a5505e841719bba4f57823305a3333b3dcecad333eea517c18ee3ba528e8fab017444ad93666ca15b73f1969b71fe0ba9f9abec8843a74b081e7 languageName: node linkType: hard From 9adecfb5c8a42f11fad0e3f4d48fef3210ec846f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Fri, 4 Jun 2021 14:59:40 -0400 Subject: [PATCH 2/7] add buildBabel2 routine --- Gulpfile.mjs | 50 +++++++++++++++++++++++++++++++++++++++++++++--- babel-worker.cjs | 39 +++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 babel-worker.cjs diff --git a/Gulpfile.mjs b/Gulpfile.mjs index 2abf94c7f87d..e72d1e1b5033 100644 --- a/Gulpfile.mjs +++ b/Gulpfile.mjs @@ -21,6 +21,8 @@ import rollupReplace from "@rollup/plugin-replace"; import { terser as rollupTerser } from "rollup-plugin-terser"; import _rollupDts from "rollup-plugin-dts"; const { default: rollupDts } = _rollupDts; +import { Worker as JestWorker } from "jest-worker"; +import glob from "glob"; import rollupBabelSource from "./scripts/rollup-plugin-babel-source.js"; import formatCode from "./scripts/utils/formatCode.js"; @@ -251,6 +253,44 @@ function buildBabel(exclude) { .pipe(gulp.dest(base)); } +function createWorker() { + const worker = new JestWorker(require.resolve("./babel-worker.cjs"), { + numWorkers: require("os").cpus().length / 2 - 1, + exposedMethods: ["transform"], + }); + worker.getStdout().on("data", chunk => process.stdout.write(chunk)); + worker.getStderr().on("data", chunk => process.stderr.write(chunk)); + return worker; +} + +async function buildBabel2(ignore = [], useWorker) { + const worker = useWorker ? createWorker() : require("./babel-worker.js"); + const files = await new Promise((resolve, reject) => { + glob( + defaultSourcesGlob, + { + ignore: ignore.map(p => `./${p.src}/**`), + }, + (err, files) => { + if (err) reject(err); + resolve(files); + } + ); + }); + + const promises = []; + for (const file of files) { + // @example ./packages/babel-parser/src/index.js + const dest = "./" + mapSrcToLib(file.slice(2)); + promises.push(worker.transform(file, dest)); + } + return Promise.all(promises).finally(() => { + if (useWorker) { + worker.end(); + } + }); +} + /** * Resolve a nested dependency starting from the given file */ @@ -517,7 +557,7 @@ gulp.task( gulp.series("copy-dts", () => buildRollupDts(dtsBundles)) ); -gulp.task("build-babel", () => buildBabel(/* exclude */ libBundles)); +gulp.task("build-babel", () => buildBabel2(/* exclude */ libBundles, true)); gulp.task( "build", @@ -536,7 +576,11 @@ gulp.task( gulp.task("default", gulp.series("build")); -gulp.task("build-no-bundle", () => buildBabel()); +gulp.task("build-no-bundle1", () => buildBabel(undefined, true)); +// First complete build on workers for complilation speed +gulp.task("build-no-bundle", () => buildBabel2(undefined, true)); +// Incremental builds is main thread for faster communication +gulp.task("build-no-bundle-watch", () => buildBabel2(undefined, false)); gulp.task( "build-dev", @@ -556,7 +600,7 @@ gulp.task( gulp.task( "watch", gulp.series("build-dev", function watch() { - gulp.watch(defaultSourcesGlob, gulp.task("build-no-bundle")); + gulp.watch(defaultSourcesGlob, gulp.task("build-no-bundle-watch")); gulp.watch( babelStandalonePluginConfigGlob, gulp.task("generate-standalone") diff --git a/babel-worker.cjs b/babel-worker.cjs new file mode 100644 index 000000000000..abd77a8818ab --- /dev/null +++ b/babel-worker.cjs @@ -0,0 +1,39 @@ +const { transformSync } = require("@babel/core"); +const { mkdirSync, statSync, readFileSync, writeFileSync } = require("fs"); +const { dirname } = require("path"); +const chalk = require("chalk"); +const fancyLog = require("fancy-log"); + +function needCompile(src, dest) { + let destStat; + const srcStat = statSync(src); + try { + destStat = statSync(dest); + } catch (err) { + if (err.code === "ENOENT") { + return true; + } else { + throw err; + } + } + return srcStat.mtimeMs > destStat.mtimeMs; +} + +exports.transform = function (src, dest) { + mkdirSync(dirname(dest), { recursive: true }); + if (!needCompile(src, dest)) { + return; + } + fancyLog(`Compiling '${chalk.cyan(src)}'...`); + const content = readFileSync(src, { encoding: "utf8" }); + const { code } = transformSync(content, { + filename: src, + caller: { + // We have wrapped packages/babel-core/src/config/files/configuration.js with feature detection + supportsDynamicImport: true, + name: "babel-worker", + }, + }); + + writeFileSync(dest, code, "utf8"); +}; From 925303d9b5fc6d0383addb65dd39dca5646e701c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Fri, 4 Jun 2021 15:16:24 -0400 Subject: [PATCH 3/7] cleanup --- Gulpfile.mjs | 62 ++++++++++------------------------------------ package.json | 3 --- yarn.lock | 69 ++-------------------------------------------------- 3 files changed, 15 insertions(+), 119 deletions(-) diff --git a/Gulpfile.mjs b/Gulpfile.mjs index e72d1e1b5033..7a5a7e8d5957 100644 --- a/Gulpfile.mjs +++ b/Gulpfile.mjs @@ -5,11 +5,8 @@ import { fileURLToPath } from "url"; import plumber from "gulp-plumber"; import through from "through2"; import chalk from "chalk"; -import newer from "gulp-newer"; -import babel from "gulp-babel"; import fancyLog from "fancy-log"; import filter from "gulp-filter"; -import revertPath from "gulp-revert-path"; import gulp from "gulp"; import { rollup } from "rollup"; import { babel as rollupBabel } from "@rollup/plugin-babel"; @@ -77,13 +74,6 @@ function getIndexFromPackage(name) { } } -function compilationLogger() { - return through.obj(function (file, enc, callback) { - fancyLog(`Compiling '${chalk.cyan(file.relative)}'...`); - callback(null, file); - }); -} - function errorsLogger() { return plumber({ errorHandler(err) { @@ -224,38 +214,13 @@ function getFiles(glob, { include, exclude }) { return stream; } -function buildBabel(exclude) { - const base = monorepoRoot; - - return getFiles(defaultSourcesGlob, { - exclude: exclude && exclude.map(p => p.src), - }) - .pipe(errorsLogger()) - .pipe(newer({ dest: base, map: mapSrcToLib })) - .pipe(compilationLogger()) - .pipe( - babel({ - caller: { - // We have wrapped packages/babel-core/src/config/files/configuration.js with feature detection - supportsDynamicImport: true, - }, - }) - ) - .pipe( - // gulp-babel always converts the extension to .js, but we want to keep the original one - revertPath() - ) - .pipe( - // Passing 'file.relative' because newer() above uses a relative - // path and this keeps it consistent. - rename(file => path.resolve(file.base, mapSrcToLib(file.relative))) - ) - .pipe(gulp.dest(base)); -} - -function createWorker() { +function createWorker(useWorker) { + const numWorkers = require("os").cpus().length / 2 - 1; + if (numWorkers === 0 || !useWorker) { + return require("./babel-worker.cjs"); + } const worker = new JestWorker(require.resolve("./babel-worker.cjs"), { - numWorkers: require("os").cpus().length / 2 - 1, + numWorkers, exposedMethods: ["transform"], }); worker.getStdout().on("data", chunk => process.stdout.write(chunk)); @@ -263,8 +228,8 @@ function createWorker() { return worker; } -async function buildBabel2(ignore = [], useWorker) { - const worker = useWorker ? createWorker() : require("./babel-worker.js"); +async function buildBabel(useWorker, ignore = []) { + const worker = createWorker(useWorker); const files = await new Promise((resolve, reject) => { glob( defaultSourcesGlob, @@ -557,7 +522,7 @@ gulp.task( gulp.series("copy-dts", () => buildRollupDts(dtsBundles)) ); -gulp.task("build-babel", () => buildBabel2(/* exclude */ libBundles, true)); +gulp.task("build-babel", () => buildBabel(true, /* exclude */ libBundles)); gulp.task( "build", @@ -576,11 +541,10 @@ gulp.task( gulp.task("default", gulp.series("build")); -gulp.task("build-no-bundle1", () => buildBabel(undefined, true)); -// First complete build on workers for complilation speed -gulp.task("build-no-bundle", () => buildBabel2(undefined, true)); -// Incremental builds is main thread for faster communication -gulp.task("build-no-bundle-watch", () => buildBabel2(undefined, false)); +// First build on worker processes for complilation speed +gulp.task("build-no-bundle", () => buildBabel(true)); +// Incremental builds take place in main process +gulp.task("build-no-bundle-watch", () => buildBabel(false)); gulp.task( "build-dev", diff --git a/package.json b/package.json index 21d69a87d176..3c5af654726e 100644 --- a/package.json +++ b/package.json @@ -53,11 +53,8 @@ "flow-bin": "^0.123.0", "glob": "^7.1.7", "gulp": "^4.0.2", - "gulp-babel": "^8.0.0", "gulp-filter": "^5.1.0", - "gulp-newer": "^1.0.0", "gulp-plumber": "^1.2.1", - "gulp-revert-path": "^2.0.0", "husky": "^3.0.0", "jest": "^27.0.0", "jest-worker": "^27.0.2", diff --git a/yarn.lock b/yarn.lock index 74f808a9dc72..251a29e88e23 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5765,11 +5765,8 @@ __metadata: flow-bin: ^0.123.0 glob: ^7.1.7 gulp: ^4.0.2 - gulp-babel: ^8.0.0 gulp-filter: ^5.1.0 - gulp-newer: ^1.0.0 gulp-plumber: ^1.2.1 - gulp-revert-path: ^2.0.0 husky: ^3.0.0 jest: ^27.0.0 jest-worker: ^27.0.2 @@ -8837,7 +8834,7 @@ fsevents@^1.2.7: languageName: node linkType: hard -"glob@npm:^7.0.0, glob@npm:^7.0.3, glob@npm:^7.1.0, glob@npm:^7.1.1, glob@npm:^7.1.2, glob@npm:^7.1.3, glob@npm:^7.1.4, glob@npm:^7.1.6, glob@npm:^7.1.7": +"glob@npm:^7.0.0, glob@npm:^7.1.0, glob@npm:^7.1.1, glob@npm:^7.1.2, glob@npm:^7.1.3, glob@npm:^7.1.4, glob@npm:^7.1.6, glob@npm:^7.1.7": version: 7.1.7 resolution: "glob@npm:7.1.7" dependencies: @@ -8965,20 +8962,6 @@ fsevents@^1.2.7: languageName: node linkType: hard -"gulp-babel@npm:^8.0.0": - version: 8.0.0 - resolution: "gulp-babel@npm:8.0.0" - dependencies: - plugin-error: ^1.0.1 - replace-ext: ^1.0.0 - through2: ^2.0.0 - vinyl-sourcemaps-apply: ^0.2.0 - peerDependencies: - "@babel/core": ^7.0.0 - checksum: d4589095f5de3f9b823e119186966550a713d4aa5868e4efd094c84054e0864f336d5140fb9418d0b17d27c18224fc982be70e8ae53ad60c9c64915b080e817f - languageName: node - linkType: hard - "gulp-cli@npm:^2.2.0": version: 2.3.0 resolution: "gulp-cli@npm:2.3.0" @@ -9018,17 +9001,6 @@ fsevents@^1.2.7: languageName: node linkType: hard -"gulp-newer@npm:^1.0.0": - version: 1.4.0 - resolution: "gulp-newer@npm:1.4.0" - dependencies: - glob: ^7.0.3 - kew: ^0.7.0 - plugin-error: ^0.1.2 - checksum: 4bdcc14ef83a8cf27e4e8eadde76c607f1af0096e446c77ad9ea72518a326f31aa623c79c3b3f26138bc91012f9c3f58a9167d907fdcfbed9b0a19e721d432d2 - languageName: node - linkType: hard - "gulp-plumber@npm:^1.2.1": version: 1.2.1 resolution: "gulp-plumber@npm:1.2.1" @@ -9041,15 +9013,6 @@ fsevents@^1.2.7: languageName: node linkType: hard -"gulp-revert-path@npm:^2.0.0": - version: 2.0.0 - resolution: "gulp-revert-path@npm:2.0.0" - dependencies: - through2: ^2.0.0 - checksum: c2b01db2942c084f8cb11cecff22663a3ab894cdf43daed601bdaa9c713797e11739415b46f18342171fb0c65c1fe57d18dd40f93da403002e99f453b5661d77 - languageName: node - linkType: hard - "gulp@npm:^4.0.2": version: 4.0.2 resolution: "gulp@npm:4.0.2" @@ -10803,13 +10766,6 @@ fsevents@^1.2.7: languageName: node linkType: hard -"kew@npm:^0.7.0": - version: 0.7.0 - resolution: "kew@npm:0.7.0" - checksum: be928d4248c934f38edda580144533bc93751f6b81d34e01cb1ff4196e63faf8957f5d54e7ef5c6529cbfdcc8f63a1acd53511e20955744b27c7eae965445775 - languageName: node - linkType: hard - "kind-of@npm:^1.1.0": version: 1.1.0 resolution: "kind-of@npm:1.1.0" @@ -12619,18 +12575,6 @@ fsevents@^1.2.7: languageName: node linkType: hard -"plugin-error@npm:^1.0.1": - version: 1.0.1 - resolution: "plugin-error@npm:1.0.1" - dependencies: - ansi-colors: ^1.0.1 - arr-diff: ^4.0.0 - arr-union: ^3.1.0 - extend-shallow: ^3.0.2 - checksum: d2e48e6b1884eb02dccb295213607a6a3da60156f5dc1b77a342577a4eb80195fb1194026dc3dd571591e678644947e56e944b21cf70568c49f18a2e375fd1df - languageName: node - linkType: hard - "posix-character-classes@npm:^0.1.0": version: 0.1.1 resolution: "posix-character-classes@npm:0.1.1" @@ -13876,7 +13820,7 @@ fsevents@^1.2.7: languageName: node linkType: hard -"source-map@npm:^0.5.0, source-map@npm:^0.5.1, source-map@npm:^0.5.3, source-map@npm:^0.5.6, source-map@npm:~0.5.1, source-map@npm:~0.5.3": +"source-map@npm:^0.5.0, source-map@npm:^0.5.3, source-map@npm:^0.5.6, source-map@npm:~0.5.1, source-map@npm:~0.5.3": version: 0.5.7 resolution: "source-map@npm:0.5.7" checksum: 737face96577a2184a42f141607fcc2c9db5620cb8517ae8ab3924476defa138fc26b0bab31e98cbd6f19211ecbf78400b59f801ff7a0f87aa9faa79f7433e10 @@ -15226,15 +15170,6 @@ typescript@~4.2.3: languageName: node linkType: hard -"vinyl-sourcemaps-apply@npm:^0.2.0": - version: 0.2.1 - resolution: "vinyl-sourcemaps-apply@npm:0.2.1" - dependencies: - source-map: ^0.5.1 - checksum: c1d826acf474831a58609e0d19f0f7e907cd6f2a347a67c2879aa44e149491f0b47449bfe906e9da54f026806196b3ac92f8ae4dfe3d3a44353baab1885a03b4 - languageName: node - linkType: hard - "vinyl@npm:^2.0.0": version: 2.2.1 resolution: "vinyl@npm:2.2.1" From 509593666137a53efd3bc927a2c1e60877140f30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Fri, 4 Jun 2021 15:34:34 -0400 Subject: [PATCH 4/7] exit gracefully when CI does not have available workers --- Gulpfile.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gulpfile.mjs b/Gulpfile.mjs index 7a5a7e8d5957..b98804677bed 100644 --- a/Gulpfile.mjs +++ b/Gulpfile.mjs @@ -250,7 +250,7 @@ async function buildBabel(useWorker, ignore = []) { promises.push(worker.transform(file, dest)); } return Promise.all(promises).finally(() => { - if (useWorker) { + if (worker.end !== undefined) { worker.end(); } }); From 9ed788038d8e364ce68eac2a874daefbc2e257cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Fri, 4 Jun 2021 16:09:00 -0400 Subject: [PATCH 5/7] perf: less IO when dest is not entity --- babel-worker.cjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/babel-worker.cjs b/babel-worker.cjs index abd77a8818ab..6a27d27d1d9c 100644 --- a/babel-worker.cjs +++ b/babel-worker.cjs @@ -6,7 +6,6 @@ const fancyLog = require("fancy-log"); function needCompile(src, dest) { let destStat; - const srcStat = statSync(src); try { destStat = statSync(dest); } catch (err) { @@ -16,6 +15,7 @@ function needCompile(src, dest) { throw err; } } + const srcStat = statSync(src); return srcStat.mtimeMs > destStat.mtimeMs; } From 71fd2e939e49036b17a3887012329c68744e99ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Wed, 9 Jun 2021 16:26:03 -0400 Subject: [PATCH 6/7] perf: pipe worker stdout/stderr to parent process --- Gulpfile.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Gulpfile.mjs b/Gulpfile.mjs index b98804677bed..3afefc667232 100644 --- a/Gulpfile.mjs +++ b/Gulpfile.mjs @@ -223,8 +223,8 @@ function createWorker(useWorker) { numWorkers, exposedMethods: ["transform"], }); - worker.getStdout().on("data", chunk => process.stdout.write(chunk)); - worker.getStderr().on("data", chunk => process.stderr.write(chunk)); + worker.getStdout().pipe(process.stdout); + worker.getStderr().pipe(process.stderr); return worker; } From 9203e0f3b61f0f66a90b739fac389b55bb687db7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Fri, 11 Jun 2021 11:05:00 -0400 Subject: [PATCH 7/7] Update Gulpfile.mjs Co-authored-by: Brian Ng --- Gulpfile.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gulpfile.mjs b/Gulpfile.mjs index 3afefc667232..42b0d2159b59 100644 --- a/Gulpfile.mjs +++ b/Gulpfile.mjs @@ -541,7 +541,7 @@ gulp.task( gulp.task("default", gulp.series("build")); -// First build on worker processes for complilation speed +// First build on worker processes for compilation speed gulp.task("build-no-bundle", () => buildBabel(true)); // Incremental builds take place in main process gulp.task("build-no-bundle-watch", () => buildBabel(false));