Skip to content

Commit

Permalink
simplify building & fix issues
Browse files Browse the repository at this point in the history
  • Loading branch information
@joshgoebel
joshgoebel committed Jun 2, 2021
1 parent 1743f9d commit c7a96b4
Show file tree
Hide file tree
Showing 5 changed files with 80 additions and 141 deletions.
106 changes: 32 additions & 74 deletions tools/build_browser.js
View file
Expand Up @@ -73,14 +73,12 @@ async function buildBrowser(options) {

detailedGrammarSizes(languages);

const size = await buildBrowserHighlightJS(languages, { minify: options.minify });
const size = await buildCore("highlight", languages, { minify: options.minify, format: "cjs" });

log("-----");
log("Core :", size.core, "bytes");
if (options.minify) { log("Core (min) :", size.core_min, "bytes"); }
log("Languages (raw) :",
languages.map((el) => el.data.length).reduce((acc, curr) => acc + curr, 0), "bytes");
log("highlight.js :", size.full, "bytes");
log("highlight.js :", size.fullSize, "bytes");
if (options.minify) {
log("highlight.min.js :", size.minified, "bytes");
log("highlight.min.js.gz :", zlib.gzipSync(size.minifiedSrc).length, "bytes");
Expand Down Expand Up @@ -191,105 +189,65 @@ const builtInLanguagesPlugin = (languages) => ({
load(id) {
const escape = (s) => "grmr_" + s.replace("-", "_");
if (id === "builtInLanguages") {
return languages.map((lang) =>
return languages.map((lang) =>
`export { default as ${escape(lang.name)} } from ${JSON.stringify(lang.path)};`
).join("\n");
}
return null;
},
})

async function buildBrowserHighlightJS(languages, { minify }) {
log("Building highlight.js.");
}
});

async function buildCore(name, languages, options) {
const header = buildHeader();

const outFile = `${process.env.BUILD_DIR}/highlight.js`;
const minifiedFile = outFile.replace(/js$/, "min.js");
const plugins = [...config.rollup.browser_core.input.plugins, builtInLanguagesPlugin(languages)];

const input = { ...config.rollup.browser_core.input, input: `src/stub.js`, plugins };
const output = { ...config.rollup.browser_core.output, file: outFile };
let librarySrc = await rollupCode(input, output);


// we don't use this, we just use it to get a size approximation for the build stats
const coreSrc = await rollupCode({ ...config.rollup.browser_core.input, input: `src/highlight.js`, plugins }, output);

// strip off the original top comment
librarySrc = librarySrc.replace(/\/\*.*?\*\//s, "");

const fullSrc = `${header}\n${librarySrc}`;

const tasks = [];
tasks.push(fs.writeFile(outFile, fullSrc, { encoding: "utf8" }));
const shas = {
"highlight.js": bundling.sha384(fullSrc)
let relativePath = "";
const input = {
...(options.format === "es" ? config.rollup.node.input : config.rollup.browser_iife.input),
input: `src/stub.js`
};
input.plugins = [
...input.plugins,
builtInLanguagesPlugin(languages)
];
const output = {
...config.rollup.node.output,
file: `${process.env.BUILD_DIR}/${name}.js`
};

let minifiedSrc, minified, core_min;

if (minify) {
const tersed = await Terser.minify(librarySrc, config.terser);

const coreTersed = await Terser.minify(coreSrc, config.terser);
core_min = header.length + 1 + coreTersed.code.length;

minifiedSrc = `${header}\n${tersed.code}`;

// get approximate core minified size
minified = minifiedSrc.length;

tasks.push(fs.writeFile(minifiedFile, minifiedSrc, { encoding: "utf8" }));
shas["highlight.min.js"] = bundling.sha384(minifiedSrc);
// optimize for no languages by not including the language loading stub
if (languages.length === 0) {
input.input = "src/highlight.js";
}

await Promise.all(tasks);
return {
core: coreSrc.length,
core_min,
fullSrc,
full: fullSrc.length,
minifiedSrc,
minified,
shas,
};
}
if (options.format === "es") {
output.format = "es";
output.file = `${process.env.BUILD_DIR}/es/${name}.js`;
relativePath = "es/";
}

async function buildBrowserESMHighlightJS(name, languages, options) {
log("Building highlight.mjs.");
const header = buildHeader();
const input = { ...config.rollup.node.input, input: `src/stub.js`, plugins: [
...config.rollup.node.input.plugins,
builtInLanguagesPlugin(languages),
] };
const output = {
...config.rollup.node.output,
format: "es",
file: `${process.env.BUILD_DIR}/es/${name}.js`,
};
log(`Building ${relativePath}${name}.js.`);

const index = await rollupCode(input, output);
const sizeInfo = {}
const writePromises = []
const sizeInfo = { shas: [] };
const writePromises = [];
if (options.minify) {
const { code } = await Terser.minify(index, {...config.terser, module: true})
const src = `${header}\n${code}`;
writePromises.push(fs.writeFile(output.file.replace(/js$/, "min.js"), src));
sizeInfo.minified = src.length;
sizeInfo.minifiedSrc = src;
sizeInfo.shas[`${relativePath}${name}.min.js`] = bundling.sha384(src)
}
{
const src = `${header}\n${index}`;
writePromises.push(fs.writeFile(output.file, src));
sizeInfo.fullSize = src.length;
sizeInfo.fullSrc = src;
sizeInfo.shas[`${relativePath}${name}.js`] = bundling.sha384(src)
}
await Promise.all(writePromises);
return sizeInfo;
}

// CDN build uses the exact same highlight.js distributable
module.exports.buildBrowserHighlightJS = buildBrowserHighlightJS;
module.exports.buildBrowserESMHighlightJS = buildBrowserESMHighlightJS;
module.exports.buildCore = buildCore;
module.exports.build = buildBrowser;
105 changes: 43 additions & 62 deletions tools/build_cdn.js
View file
@@ -1,53 +1,27 @@
const fs = require("fs").promises;
const fss = require("fs");
const glob = require("glob");
const Terser = require("terser");
const zlib = require('zlib');
const { getLanguages } = require("./lib/language");
const { filter } = require("./lib/dependencies");
const config = require("./build_config");
const { install, installCleanCSS, mkdir } = require("./lib/makestuff");
const { getLanguages } = require("./lib/language.js");
const { filter } = require("./lib/dependencies.js");
const config = require("./build_config.js");
const { install, installCleanCSS, mkdir } = require("./lib/makestuff.js");
const log = (...args) => console.log(...args);
const { buildBrowserHighlightJS, buildBrowserESMHighlightJS } = require("./build_browser");
const { buildPackageJSON, writePackageJSON } = require("./build_node");
const { rollupCode } = require("./lib/bundling.js");
const { buildCore } = require("./build_browser.js");
const { buildPackageJSON, writePackageJSON } = require("./build_node.js");
const path = require("path");
const bundling = require('./lib/bundling.js');

async function installPackageJSON(options) {
const json = buildPackageJSON(options, {
".": {
import: options.minify ? "./es/index.min.js" : "./es/index.js",
browser: options.minify ? "./highlight.min.js" : "./highlight.js",
},
"./lib/languages/*": {
import: "./es/languages/*.js",
browser: "./languages/*.js"
},
get "./lib/common"(){ return this["."]; },
"./lib/core": { import: "./es/core.js" },
"./styles/*": "./styles/*",
"./package.json": "./package.json",
});
const json = buildPackageJSON(options);
json.name = "@highlightjs/cdn-assets";
json.description = json.description.concat(" (pre-compiled CDN assets)");
// CDN assets do not need an export map, they are just a bunch of files.
// The NPM package mostly only exists to populate CDNs and provide raw files.
delete json.exports;
await writePackageJSON(json);
}

async function buildESMCore(options) {
const input = { ...config.rollup.node.input, input: `src/highlight.js` };
const output = {
...config.rollup.node.output,
format: "es",
file: `${process.env.BUILD_DIR}/es/core.js`,
};
const core = await rollupCode(input, output);

const miniCore = options.minify ? await Terser.minify(core, {...config.terser, module: true}) : { code: core };
await fs.writeFile(output.file, miniCore.code);
return miniCore.code.length;
}

let shas = {};

async function buildCDN(options) {
Expand All @@ -59,13 +33,9 @@ async function buildCDN(options) {

// all the languages are built for the CDN and placed into `/languages`
const languages = await getLanguages();

let esmCoreSize, esmIndexSize;
if (options.esm) {
mkdir("es");
await fs.writeFile(`${process.env.BUILD_DIR}/es/package.json`, `{ "type": "module" }`);
esmCoreSize = await buildESMCore(options);
}

let esmCoreSize = {};
let esmCommonSize = {};

await installLanguages(languages, options);

Expand All @@ -79,34 +49,43 @@ async function buildCDN(options) {
embedLanguages = [];
}

const size = await buildBrowserHighlightJS(embedLanguages, { minify: options.minify });
if (options.esm) esmIndexSize = await buildBrowserESMHighlightJS("index", embedLanguages, { minify: options.minify });
shas = Object.assign({}, size.shas, shas);
const size = await buildCore("highlight", embedLanguages, { minify: options.minify, format: "cjs" });
if (options.esm) {
mkdir("es");
await fs.writeFile(`${process.env.BUILD_DIR}/es/package.json`, `{ "type": "module" }`);
esmCoreSize = await buildCore("core", [], {minify: options.minify, format: "es"});
esmCommonSize = await buildCore("highlight", embedLanguages, { minify: options.minify, format: "es" });
}
shas = {
...size.shas, ...esmCommonSize.shas, ...esmCoreSize.shas, ...shas
};

await buildSRIDigests(shas);

log("-----");
log("Embedded Lang :",
log("Embedded Lang :",
embedLanguages.map((el) => el.minified.length).reduce((acc, curr) => acc + curr, 0), "bytes");
log("All Lang :",
log("All Lang :",
languages.map((el) => el.minified.length).reduce((acc, curr) => acc + curr, 0), "bytes");
log("highlight.js :",
size.full, "bytes");
log("highlight.js :",
size.fullSize, "bytes");

if (options.minify) {
log("highlight.min.js :", size.minified, "bytes");
log("highlight.min.js.gz :", zlib.gzipSync(size.minifiedSrc).length, "bytes");
log("highlight.min.js :", size.minified, "bytes");
log("highlight.min.js.gz :", zlib.gzipSync(size.minifiedSrc).length, "bytes");
} else {
log("highlight.js.gz :", zlib.gzipSync(size.fullSrc).length, "bytes");
log("highlight.js.gz :", zlib.gzipSync(size.fullSrc).length, "bytes");
}
if(options.esm) {
log("es/core.js :", esmCoreSize, "bytes");
log("es/index.js :", esmIndexSize.fullSize, "bytes");
if (options.esm) {
log("es/core.js :", esmCoreSize.fullSize, "bytes");
log("es/highlight.js :", esmCommonSize.fullSize, "bytes");
if (options.minify) {
log("es/index.min.js :", esmIndexSize.minified, "bytes");
log("es/index.min.js.gz :", zlib.gzipSync(esmIndexSize.minifiedSrc).length, "bytes");
log("es/core.min.js :", esmCoreSize.minified, "bytes");
log("es/core.min.js.gz :", zlib.gzipSync(esmCoreSize.minifiedSrc).length, "bytes");
log("es/highlight.min.js :", esmCommonSize.minified, "bytes");
log("es/highlight.min.js.gz :", zlib.gzipSync(esmCommonSize.minifiedSrc).length, "bytes");
} else {
log("es/index.js.gz :", zlib.gzipSync(esmIndexSize.fullSrc).length, "bytes");
log("es/highlight.js.gz :", zlib.gzipSync(esmCommonSize.fullSrc).length, "bytes");
}
}
log("-----");
Expand All @@ -117,7 +96,7 @@ async function buildSRIDigests(shas) {
const temp = await fs.readFile("./tools/templates/DIGESTS.md");
const DIGEST_MD = temp.toString();

const version = require("../package").version;
const version = require("../package.json").version;
const digestList = Object.entries(shas).map(([k, v]) => `${v} ${k}`).join("\n");

const out = DIGEST_MD
Expand All @@ -131,7 +110,7 @@ async function buildSRIDigests(shas) {
async function installLanguages(languages, options) {
log("Building language files.");
mkdir("languages");
if(options.esm) mkdir("es/languages");
if (options.esm) mkdir("es/languages");

await Promise.all(
languages.map(async(language) => {
Expand Down Expand Up @@ -181,8 +160,10 @@ async function buildCDNLanguage(language, options) {
await language.compile({ terser: config.terser });
shas[name] = bundling.sha384(language.minified);
await fs.writeFile(`${process.env.BUILD_DIR}/${name}`, language.minified);
if (options.esm)
if (options.esm) {
shas[`es/${name}`] = bundling.sha384(language.minifiedESM);
await fs.writeFile(`${process.env.BUILD_DIR}/es/${name}`, language.minifiedESM);
}
}

module.exports.build = buildCDN;
Expand Down
2 changes: 1 addition & 1 deletion tools/build_config.js
View file
Expand Up @@ -27,7 +27,7 @@ module.exports = {
]
}
},
browser_core: {
browser_iife: {
input: {
plugins: [
jsonPlugin(),
Expand Down
6 changes: 3 additions & 3 deletions tools/build_node.js
View file
Expand Up @@ -106,10 +106,10 @@ const generatePackageExports = () => ({
"./styles/*": "./styles/*",
"./types/*": "./types/*",
});
function buildPackageJSON(options, exports = generatePackageExports()) {
const packageJson = require("../package");
function buildPackageJSON(options) {
const packageJson = require("../package.json");

if (options.esm) packageJson.exports = exports;
if (options.esm) packageJson.exports = generatePackageExports();

return packageJson;
}
Expand Down
2 changes: 1 addition & 1 deletion tools/lib/bundling.js
View file
Expand Up @@ -22,7 +22,7 @@ function sha384(contents) {
const hash = crypto.createHash('sha384');
const data = hash.update(contents, 'utf-8');
const gen_hash = data.digest('base64');
return `sha384-${gen_hash}`
return `sha384-${gen_hash}`;
}

module.exports = { rollupWrite, rollupCode, sha384 };

0 comments on commit c7a96b4

Please sign in to comment.