Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Faster babel build #13423

Merged
merged 7 commits into from Jun 11, 2021
Merged

Faster babel build #13423

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
03ba9e3
chore: add glob and jest-worker
JLHwung Jun 4, 2021
9adecfb
add buildBabel2 routine
JLHwung Jun 4, 2021
925303d
cleanup
JLHwung Jun 4, 2021
5095936
exit gracefully when CI does not have available workers
JLHwung Jun 4, 2021
9ed7880
perf: less IO when dest is not entity
JLHwung Jun 4, 2021
71fd2e9
perf: pipe worker stdout/stderr to parent process
JLHwung Jun 9, 2021
9203e0f
Update Gulpfile.mjs
JLHwung Jun 11, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
86 changes: 47 additions & 39 deletions Gulpfile.mjs
View file
Open in desktop
Expand Up @@ -5,11 +5,8 @@ import { fileURLToPath } from "url";
import plumber from "gulp-plumber";
import through from "through2";
import chalk from "chalk";
import newer from "gulp-newer";
import babel from "gulp-babel";
import fancyLog from "fancy-log";
import filter from "gulp-filter";
import revertPath from "gulp-revert-path";
import gulp from "gulp";
import { rollup } from "rollup";
import { babel as rollupBabel } from "@rollup/plugin-babel";
Expand All @@ -21,6 +18,8 @@ import rollupReplace from "@rollup/plugin-replace";
import { terser as rollupTerser } from "rollup-plugin-terser";
import _rollupDts from "rollup-plugin-dts";
const { default: rollupDts } = _rollupDts;
import { Worker as JestWorker } from "jest-worker";
import glob from "glob";

import rollupBabelSource from "./scripts/rollup-plugin-babel-source.js";
import formatCode from "./scripts/utils/formatCode.js";
Expand Down Expand Up @@ -75,13 +74,6 @@ function getIndexFromPackage(name) {
}
}

function compilationLogger() {
return through.obj(function (file, enc, callback) {
fancyLog(`Compiling '${chalk.cyan(file.relative)}'...`);
callback(null, file);
});
}

function errorsLogger() {
return plumber({
errorHandler(err) {
Expand Down Expand Up @@ -222,33 +214,46 @@ function getFiles(glob, { include, exclude }) {
return stream;
}

function buildBabel(exclude) {
const base = monorepoRoot;
function createWorker(useWorker) {
const numWorkers = require("os").cpus().length / 2 - 1;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I imagine we can/might want to make this configurable at some point? (Like the --maxWorkers flag in jest)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah. Currently the default value is assuming the CPU has hyper-threading so it havles and minus the main thread. I tested different values locally, based on our scenario, increasing the numWorkers more than that does not yield faster performance.

if (numWorkers === 0 || !useWorker) {
return require("./babel-worker.cjs");
}
const worker = new JestWorker(require.resolve("./babel-worker.cjs"), {
numWorkers,
exposedMethods: ["transform"],
});
worker.getStdout().pipe(process.stdout);
worker.getStderr().pipe(process.stderr);
return worker;
}

return getFiles(defaultSourcesGlob, {
exclude: exclude && exclude.map(p => p.src),
})
.pipe(errorsLogger())
.pipe(newer({ dest: base, map: mapSrcToLib }))
.pipe(compilationLogger())
.pipe(
babel({
caller: {
// We have wrapped packages/babel-core/src/config/files/configuration.js with feature detection
supportsDynamicImport: true,
},
})
)
.pipe(
// gulp-babel always converts the extension to .js, but we want to keep the original one
revertPath()
)
.pipe(
// Passing 'file.relative' because newer() above uses a relative
// path and this keeps it consistent.
rename(file => path.resolve(file.base, mapSrcToLib(file.relative)))
)
.pipe(gulp.dest(base));
async function buildBabel(useWorker, ignore = []) {
const worker = createWorker(useWorker);
const files = await new Promise((resolve, reject) => {
glob(
defaultSourcesGlob,
{
ignore: ignore.map(p => `./${p.src}/**`),
},
(err, files) => {
if (err) reject(err);
resolve(files);
}
);
});

const promises = [];
for (const file of files) {
// @example ./packages/babel-parser/src/index.js
const dest = "./" + mapSrcToLib(file.slice(2));
promises.push(worker.transform(file, dest));
}
return Promise.all(promises).finally(() => {
if (worker.end !== undefined) {
worker.end();
}
});
}

/**
Expand Down Expand Up @@ -517,7 +522,7 @@ gulp.task(
gulp.series("copy-dts", () => buildRollupDts(dtsBundles))
);

gulp.task("build-babel", () => buildBabel(/* exclude */ libBundles));
gulp.task("build-babel", () => buildBabel(true, /* exclude */ libBundles));
Copy link
Member

@hzoo hzoo Jun 9, 2021 •
edited

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

guess rename this to ignore?

Suggested change
gulp.task("build-babel", () => buildBabel(true, /* exclude */ libBundles));
gulp.task("build-babel", () => buildBabel(true, /* ignore */ libBundles));


gulp.task(
"build",
Expand All @@ -536,7 +541,10 @@ gulp.task(

gulp.task("default", gulp.series("build"));

gulp.task("build-no-bundle", () => buildBabel());
// First build on worker processes for complilation speed
JLHwung marked this conversation as resolved.
Show resolved Hide resolved
gulp.task("build-no-bundle", () => buildBabel(true));
// Incremental builds take place in main process
gulp.task("build-no-bundle-watch", () => buildBabel(false));

gulp.task(
"build-dev",
Expand All @@ -556,7 +564,7 @@ gulp.task(
gulp.task(
"watch",
gulp.series("build-dev", function watch() {
gulp.watch(defaultSourcesGlob, gulp.task("build-no-bundle"));
gulp.watch(defaultSourcesGlob, gulp.task("build-no-bundle-watch"));
gulp.watch(
babelStandalonePluginConfigGlob,
gulp.task("generate-standalone")
Expand Down
39 changes: 39 additions & 0 deletions babel-worker.cjs
View file
Open in desktop
@@ -0,0 +1,39 @@
const { transformSync } = require("@babel/core");
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We deviates from the conventional Gulp stream approach because:

  1. Passing the source code to sub process is more expensive than passing the path
  2. Passing the transformed code to main process in order to written via gulp.dest is expensive and ultimately counters the performance gain from multiple process

So we end up creating our own gulp-babel + gulp-newer in this file.

const { mkdirSync, statSync, readFileSync, writeFileSync } = require("fs");
const { dirname } = require("path");
const chalk = require("chalk");
const fancyLog = require("fancy-log");

function needCompile(src, dest) {
let destStat;
try {
destStat = statSync(dest);
} catch (err) {
if (err.code === "ENOENT") {
return true;
} else {
throw err;
}
}
const srcStat = statSync(src);
return srcStat.mtimeMs > destStat.mtimeMs;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

were you able to test watching/incremental, wondering how effective this is? wondering whether we can reuse chokidar here too or is that slow?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can combine it with an incremental watch pipeline, so only changed files are compiled. I can land it in a separate PR.

The stat comparison here is essentially a replacement for gulp-newer so when we run gulp build-no-bundle when we already have lib/**, we don't re-compile the compiled sources.

}

exports.transform = function (src, dest) {
mkdirSync(dirname(dest), { recursive: true });
if (!needCompile(src, dest)) {
return;
}
fancyLog(`Compiling '${chalk.cyan(src)}'...`);
const content = readFileSync(src, { encoding: "utf8" });
const { code } = transformSync(content, {
filename: src,
caller: {
// We have wrapped packages/babel-core/src/config/files/configuration.js with feature detection
supportsDynamicImport: true,
name: "babel-worker",
},
});

writeFileSync(dest, code, "utf8");
};
5 changes: 2 additions & 3 deletions package.json
View file
Open in desktop
Expand Up @@ -51,14 +51,13 @@
"eslint-plugin-prettier": "^3.4.0",
"fancy-log": "^1.3.3",
"flow-bin": "^0.123.0",
"glob": "^7.1.7",
"gulp": "^4.0.2",
"gulp-babel": "^8.0.0",
"gulp-filter": "^5.1.0",
"gulp-newer": "^1.0.0",
"gulp-plumber": "^1.2.1",
"gulp-revert-path": "^2.0.0",
"husky": "^3.0.0",
"jest": "^27.0.0",
"jest-worker": "^27.0.2",
"lint-staged": "^9.2.0",
"lodash": "^4.17.21",
"mergeiterator": "^1.2.5",
Expand Down
85 changes: 11 additions & 74 deletions yarn.lock
View file
Open in desktop
Expand Up @@ -5763,14 +5763,13 @@ __metadata:
eslint-plugin-prettier: ^3.4.0
fancy-log: ^1.3.3
flow-bin: ^0.123.0
glob: ^7.1.7
gulp: ^4.0.2
gulp-babel: ^8.0.0
gulp-filter: ^5.1.0
gulp-newer: ^1.0.0
gulp-plumber: ^1.2.1
gulp-revert-path: ^2.0.0
husky: ^3.0.0
jest: ^27.0.0
jest-worker: ^27.0.2
lint-staged: ^9.2.0
lodash: ^4.17.21
mergeiterator: ^1.2.5
Expand Down Expand Up @@ -8835,17 +8834,17 @@ fsevents@^1.2.7:
languageName: node
linkType: hard

"glob@npm:^7.0.0, glob@npm:^7.0.3, glob@npm:^7.1.0, glob@npm:^7.1.1, glob@npm:^7.1.2, glob@npm:^7.1.3, glob@npm:^7.1.4, glob@npm:^7.1.6":
version: 7.1.6
resolution: "glob@npm:7.1.6"
"glob@npm:^7.0.0, glob@npm:^7.1.0, glob@npm:^7.1.1, glob@npm:^7.1.2, glob@npm:^7.1.3, glob@npm:^7.1.4, glob@npm:^7.1.6, glob@npm:^7.1.7":
version: 7.1.7
resolution: "glob@npm:7.1.7"
dependencies:
fs.realpath: ^1.0.0
inflight: ^1.0.4
inherits: 2
minimatch: ^3.0.4
once: ^1.3.0
path-is-absolute: ^1.0.0
checksum: 789977b52432865bd63846da5c75a6efc2c56abdc0cb5ffcdb8e91eeb67a58fa5594c1195d18b2b4aff99675b0739ed6bd61024b26562e0cca18c8f993efdc82
checksum: 352f74f08247db5420161a2f68f2bd84b53228b5fcfc9dcc37cd54d3f19ec0232495d84aeff1286d0727059e9fdc1031400e00b971bdc59e30f8f82b199c9d02
languageName: node
linkType: hard

Expand Down Expand Up @@ -8963,20 +8962,6 @@ fsevents@^1.2.7:
languageName: node
linkType: hard

"gulp-babel@npm:^8.0.0":
version: 8.0.0
resolution: "gulp-babel@npm:8.0.0"
dependencies:
plugin-error: ^1.0.1
replace-ext: ^1.0.0
through2: ^2.0.0
vinyl-sourcemaps-apply: ^0.2.0
peerDependencies:
"@babel/core": ^7.0.0
checksum: d4589095f5de3f9b823e119186966550a713d4aa5868e4efd094c84054e0864f336d5140fb9418d0b17d27c18224fc982be70e8ae53ad60c9c64915b080e817f
languageName: node
linkType: hard

"gulp-cli@npm:^2.2.0":
version: 2.3.0
resolution: "gulp-cli@npm:2.3.0"
Expand Down Expand Up @@ -9016,17 +9001,6 @@ fsevents@^1.2.7:
languageName: node
linkType: hard

"gulp-newer@npm:^1.0.0":
version: 1.4.0
resolution: "gulp-newer@npm:1.4.0"
dependencies:
glob: ^7.0.3
kew: ^0.7.0
plugin-error: ^0.1.2
checksum: 4bdcc14ef83a8cf27e4e8eadde76c607f1af0096e446c77ad9ea72518a326f31aa623c79c3b3f26138bc91012f9c3f58a9167d907fdcfbed9b0a19e721d432d2
languageName: node
linkType: hard

"gulp-plumber@npm:^1.2.1":
version: 1.2.1
resolution: "gulp-plumber@npm:1.2.1"
Expand All @@ -9039,15 +9013,6 @@ fsevents@^1.2.7:
languageName: node
linkType: hard

"gulp-revert-path@npm:^2.0.0":
version: 2.0.0
resolution: "gulp-revert-path@npm:2.0.0"
dependencies:
through2: ^2.0.0
checksum: c2b01db2942c084f8cb11cecff22663a3ab894cdf43daed601bdaa9c713797e11739415b46f18342171fb0c65c1fe57d18dd40f93da403002e99f453b5661d77
languageName: node
linkType: hard

"gulp@npm:^4.0.2":
version: 4.0.2
resolution: "gulp@npm:4.0.2"
Expand Down Expand Up @@ -10530,14 +10495,14 @@ fsevents@^1.2.7:
languageName: node
linkType: hard

"jest-worker@npm:^27.0.1":
version: 27.0.1
resolution: "jest-worker@npm:27.0.1"
"jest-worker@npm:^27.0.1, jest-worker@npm:^27.0.2":
version: 27.0.2
resolution: "jest-worker@npm:27.0.2"
dependencies:
"@types/node": "*"
merge-stream: ^2.0.0
supports-color: ^8.0.0
checksum: c99fcfd6bf2ef0c9fef30e08fb00d7fe951be84d2cff37a3d8983d683857b150a7ac60829ed30893c69277a1d74b57fc2b8aae83ded208fd574985fd9b04162f
checksum: bfbfd3d0af94a5505e841719bba4f57823305a3333b3dcecad333eea517c18ee3ba528e8fab017444ad93666ca15b73f1969b71fe0ba9f9abec8843a74b081e7
languageName: node
linkType: hard

Expand Down Expand Up @@ -10801,13 +10766,6 @@ fsevents@^1.2.7:
languageName: node
linkType: hard

"kew@npm:^0.7.0":
version: 0.7.0
resolution: "kew@npm:0.7.0"
checksum: be928d4248c934f38edda580144533bc93751f6b81d34e01cb1ff4196e63faf8957f5d54e7ef5c6529cbfdcc8f63a1acd53511e20955744b27c7eae965445775
languageName: node
linkType: hard

"kind-of@npm:^1.1.0":
version: 1.1.0
resolution: "kind-of@npm:1.1.0"
Expand Down Expand Up @@ -12617,18 +12575,6 @@ fsevents@^1.2.7:
languageName: node
linkType: hard

"plugin-error@npm:^1.0.1":
version: 1.0.1
resolution: "plugin-error@npm:1.0.1"
dependencies:
ansi-colors: ^1.0.1
arr-diff: ^4.0.0
arr-union: ^3.1.0
extend-shallow: ^3.0.2
checksum: d2e48e6b1884eb02dccb295213607a6a3da60156f5dc1b77a342577a4eb80195fb1194026dc3dd571591e678644947e56e944b21cf70568c49f18a2e375fd1df
languageName: node
linkType: hard

"posix-character-classes@npm:^0.1.0":
version: 0.1.1
resolution: "posix-character-classes@npm:0.1.1"
Expand Down Expand Up @@ -13874,7 +13820,7 @@ fsevents@^1.2.7:
languageName: node
linkType: hard

"source-map@npm:^0.5.0, source-map@npm:^0.5.1, source-map@npm:^0.5.3, source-map@npm:^0.5.6, source-map@npm:~0.5.1, source-map@npm:~0.5.3":
"source-map@npm:^0.5.0, source-map@npm:^0.5.3, source-map@npm:^0.5.6, source-map@npm:~0.5.1, source-map@npm:~0.5.3":
version: 0.5.7
resolution: "source-map@npm:0.5.7"
checksum: 737face96577a2184a42f141607fcc2c9db5620cb8517ae8ab3924476defa138fc26b0bab31e98cbd6f19211ecbf78400b59f801ff7a0f87aa9faa79f7433e10
Expand Down Expand Up @@ -15224,15 +15170,6 @@ typescript@~4.2.3:
languageName: node
linkType: hard

"vinyl-sourcemaps-apply@npm:^0.2.0":
version: 0.2.1
resolution: "vinyl-sourcemaps-apply@npm:0.2.1"
dependencies:
source-map: ^0.5.1
checksum: c1d826acf474831a58609e0d19f0f7e907cd6f2a347a67c2879aa44e149491f0b47449bfe906e9da54f026806196b3ac92f8ae4dfe3d3a44353baab1885a03b4
languageName: node
linkType: hard

"vinyl@npm:^2.0.0":
version: 2.2.1
resolution: "vinyl@npm:2.2.1"
Expand Down