Skip to content

Commit

Permalink
(feat) include ESM in CDN build
Browse files Browse the repository at this point in the history 
Fixes: highlightjs#3199
  • Loading branch information
@aduh95 @joshgoebel
aduh95 authored and joshgoebel committed Jul 2, 2021
1 parent 50eca53 commit 2ca66c3
Show file tree
Hide file tree
Showing 4 changed files with 102 additions and 43 deletions.
89 changes: 74 additions & 15 deletions tools/build_cdn.js
View file
@@ -1,42 +1,98 @@
const fs = require("fs").promises;
const fss = require("fs");
const glob = require("glob");
const Terser = require("terser");
const zlib = require('zlib');
const { getLanguages } = require("./lib/language");
const { filter } = require("./lib/dependencies");
const config = require("./build_config");
const { install, installCleanCSS, mkdir } = require("./lib/makestuff");
const log = (...args) => console.log(...args);
const { buildBrowserHighlightJS } = require("./build_browser");
const { buildPackageJSON } = require("./build_node");
const { buildPackageJSON, writePackageJSON } = require("./build_node");
const { rollupCode } = require("./lib/bundling.js");
const path = require("path");
const bundling = require('./lib/bundling.js');

async function installPackageJSON(options) {
await buildPackageJSON(options);
const json = require(`${process.env.BUILD_DIR}/package`);
const json = buildPackageJSON(options, {
".": {
import: "./es/index.js",
browser: "./highlight.min.js",
},
"./lib/languages/*": {
import: "./es/languages/*.js",
browser: "./languages/*.js"
},
"./lib/common": { import: "./es/index.js" },
"./lib/core": { import: "./es/core.js" },
"./styles/*": "./styles/*",
"./package.json": "./package.json",
});
json.name = "@highlightjs/cdn-assets";
json.description = json.description.concat(" (pre-compiled CDN assets)");
// this is not a replacement for `highlightjs` package
delete json.exports;
delete json.type;
delete json.main;
delete json.types;
fs.writeFile(`${process.env.BUILD_DIR}/package.json`, JSON.stringify(json, null, ' '));
await writePackageJSON(json);
}

const safeImportName = (s) => {
s = s.replace(/-/g, "_");
if (/^\d/.test(s)) s = `L_${s}`;
return s;
};

async function buildESMIndex(name, languages) {
const header = `import hljs from './core.js';`;
const footer = "export default hljs;";

const registration = languages.map((lang) => {
const importName = safeImportName(lang.name);
return `import ${importName} from './languages/${lang.name}.js';\n` +
`hljs.registerLanguage('${lang.name}', ${importName});`;
});

const index = `${header}\n\n${registration.join("\n")}\n\n${footer}`;
await fs.writeFile(`${process.env.BUILD_DIR}/es/${name}.js`, index);
}

async function buildESMCore(options) {
const input = { ...config.rollup.node.input, input: `src/highlight.js` };
const output = {
...config.rollup.node.output,
format: "es",
file: `${process.env.BUILD_DIR}/es/core.js`,
};
const core = await rollupCode(input, output);

const miniCore = options.minify ? await Terser.minify(core, config.terser) : { code: core };
await fs.writeFile(output.file, miniCore.code || core);
return miniCore.code.length;
}

let shas = {};

async function buildCDN(options) {
install("./LICENSE", "LICENSE");
install("./README.CDN.md", "README.md");
installPackageJSON(options);
await installPackageJSON(options);

installStyles();

// all the languages are built for the CDN and placed into `/languages`
const languages = await getLanguages();
await installLanguages(languages);

let esmCoreSize;
if (options.esm) {
mkdir("es");
await fs.writeFile(`${process.env.BUILD_DIR}/es/package.json`, `{ "type": "module" }`);
esmCoreSize = await buildESMCore(options);
}

await installLanguages(languages, options);

// filter languages for inclusion in the highlight.js bundle
let embedLanguages = filter(languages, options.languages);
Expand All @@ -49,6 +105,7 @@ async function buildCDN(options) {
}

const size = await buildBrowserHighlightJS(embedLanguages, { minify: options.minify });
if (options.esm) await buildESMIndex("index", embedLanguages, { minify: options.minify });
shas = Object.assign({}, size.shas, shas);

await buildSRIDigests(shas);
Expand All @@ -60,6 +117,8 @@ async function buildCDN(options) {
languages.map((el) => el.minified.length).reduce((acc, curr) => acc + curr, 0), "bytes");
log("highlight.js :",
size.full, "bytes");
if(options.esm)
log("es/core.js :", esmCoreSize, "bytes");

if (options.minify) {
log("highlight.min.js :", size.minified, "bytes");
Expand All @@ -86,23 +145,22 @@ async function buildSRIDigests(shas) {
fs.writeFile(`${process.env.BUILD_DIR}/DIGESTS.md`, out);
}

async function installLanguages(languages) {
async function installLanguages(languages, options) {
log("Building language files.");
mkdir("languages");
if(options.esm) mkdir("es/languages");

await Promise.all(
languages.map(async(language) => {
await buildCDNLanguage(language);
await buildCDNLanguage(language, options);
process.stdout.write(".");
})
);
log("");

await Promise.all(
languages.filter((l) => l.third_party)
.map(async(language) => {
await buildDistributable(language);
})
.map(buildDistributable)
);

log("");
Expand Down Expand Up @@ -131,16 +189,17 @@ async function buildDistributable(language) {
const distDir = path.join(language.moduleDir, "dist");
log(`Building ${distDir}/${filename}.`);
await fs.mkdir(distDir, { recursive: true });
fs.writeFile(path.join(language.moduleDir, "dist", filename), language.minified);
await fs.writeFile(path.join(language.moduleDir, "dist", filename), language.minified);
}

async function buildCDNLanguage(language) {
async function buildCDNLanguage(language, options) {
const name = `languages/${language.name}.min.js`;
const filename = `${process.env.BUILD_DIR}/${name}`;

await language.compile({ terser: config.terser });
shas[name] = bundling.sha384(language.minified);
fs.writeFile(filename, language.minified);
await fs.writeFile(`${process.env.BUILD_DIR}/${name}`, language.minified);
if (options.esm)
await fs.writeFile(`${process.env.BUILD_DIR}/es/${name}`, language.minifiedESM);
}

module.exports.build = buildCDN;
Expand Down
8 changes: 2 additions & 6 deletions tools/build_config.js
View file
Expand Up @@ -44,14 +44,10 @@ module.exports = {
},
browser: {
input: {
plugins: [
cjsPlugin(),
jsonPlugin()
]
plugins: []
},
output: {
format: "iife",
outro: "return module.exports.definer || module.exports;",
format: "es",
interop: false
}
}
Expand Down
30 changes: 17 additions & 13 deletions tools/build_node.js
View file
Expand Up @@ -95,22 +95,25 @@ function dual(file) {
};
}

async function buildPackageJSON(options) {
const generatePackageExports = () => ({
".": dual("./lib/index.js"),
"./package.json": "./package.json",
"./lib/common": dual("./lib/common.js"),
"./lib/core": dual("./lib/core.js"),
"./lib/languages/*": dual("./lib/languages/*.js"),
"./scss/*": "./scss/*",
"./styles/*": "./styles/*",
"./types/*": "./types/*",
});
function buildPackageJSON(options, exports = generatePackageExports()) {
const packageJson = require("../package");

const exports = {
".": dual("./lib/index.js"),
"./package.json": "./package.json",
"./lib/common": dual("./lib/common.js"),
"./lib/core": dual("./lib/core.js"),
"./lib/languages/*": dual("./lib/languages/*.js"),
"./scss/*": "./scss/*",
"./styles/*": "./styles/*",
"./types/*": "./types/*",
};
if (options.esm) packageJson.exports = exports;

await fs.writeFile(`${process.env.BUILD_DIR}/package.json`, JSON.stringify(packageJson, null, 2));
return packageJson;
}
function writePackageJSON(packageJson) {
return fs.writeFile(`${process.env.BUILD_DIR}/package.json`, JSON.stringify(packageJson, null, 2));
}

async function buildLanguages(languages, options) {
Expand Down Expand Up @@ -170,7 +173,7 @@ async function buildNode(options) {
const common = languages.filter(l => l.categories.includes("common"));

log("Writing package.json.");
await buildPackageJSON(options);
await writePackageJSON(buildPackageJSON(options));

if (options.esm) {
await fs.writeFile(`${process.env.BUILD_DIR}/es/package.json`, `{ "type": "module" }`);
Expand All @@ -188,3 +191,4 @@ async function buildNode(options) {

module.exports.build = buildNode;
module.exports.buildPackageJSON = buildPackageJSON;
module.exports.writePackageJSON = writePackageJSON;
18 changes: 9 additions & 9 deletions tools/lib/language.js
View file
Expand Up @@ -80,19 +80,19 @@ class Language {

async function compileLanguage (language, options) {
const EXPORT_REGEX = /export default (.*);/;
const IIFE_HEADER_REGEX = /^(var dummyName = )?\(function \(\)/;

// TODO: cant we use the source we already have?
const input = { ...build_config.rollup.browser.input, input: language.path };
const output = { ...build_config.rollup.browser.output, name: `dummyName`, file: "out.js" };
var data = await rollupCode(input, output)

data = data.replace(IIFE_HEADER_REGEX, `hljs.registerLanguage('${language.name}', function ()`)

var original = data;
language.module = data;
data = await Terser.minify(data, options.terser);
language.minified = data.code || original;

const esm = await rollupCode(input, output);
const iife = `hljs.registerLanguage('${language.name}', function (){"use strict";` + esm.replace(EXPORT_REGEX, 'return $1') + '})';

language.module = iife;
const miniESM = await Terser.minify(esm, options.terser);
const miniIIFE = await Terser.minify(iife, options.terser);
language.minified = miniIIFE.code || iife;
language.minifiedESM = miniESM.code || esm;
}

async function getLanguages() {
Expand Down

0 comments on commit 2ca66c3

Please sign in to comment.