diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 1c265e0808..55c5aa64d6 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -33,6 +33,11 @@ jobs: run: | npm run test-browser node test/builds/browser_build_as_commonjs.js + # CDN build should be easily importable + - if: contains(matrix.build-how, 'cdn') + name: Test that we can import CDN esm build + run: | + node test/builds/cdn_build_as_esm.mjs - if: contains(matrix.build-how, 'node') name: Test Node.js build diff --git a/test/builds/cdn_build_as_esm.mjs b/test/builds/cdn_build_as_esm.mjs new file mode 100644 index 0000000000..b217e7d5ce --- /dev/null +++ b/test/builds/cdn_build_as_esm.mjs @@ -0,0 +1,24 @@ +import hljs from "../../build/es/highlight.js"; + +const API = [ + "getLanguage", + "registerLanguage", + "highlight", + "highlightAuto", + "highlightAll", + "highlightElement" +]; + +const assert = (f,msg) => { + if (!f()) { + console.error(msg); + process.exit(1); + } +}; +const keys = Object.keys(hljs); + +API.forEach(n => { + assert(_ => keys.includes(n), `API should include ${n}`); +}); + +console.log("Pass: browser build works with Node.js just fine.") diff --git a/tools/build_browser.js b/tools/build_browser.js index 02cb891bb3..48c0f465b6 100644 --- a/tools/build_browser.js +++ b/tools/build_browser.js @@ -14,9 +14,14 @@ const log = (...args) => console.log(...args); const { rollupCode } = require("./lib/bundling.js"); const bundling = require('./lib/bundling.js'); const Table = require('cli-table'); -const { result } = require('lodash'); -function buildHeader(args) { +const getDefaultHeader = () => ({ + ...require('../package.json'), + git_sha : child_process + .execSync("git rev-parse --short=10 HEAD") + .toString().trim(), +}); +function buildHeader(args = getDefaultHeader()) { return "/*!\n" + ` Highlight.js v${args.version} (git: ${args.git_sha})\n` + ` (c) ${config.copyrightYears} ${args.author.name} and other contributors\n` + @@ -68,14 +73,12 @@ async function buildBrowser(options) { detailedGrammarSizes(languages); - const size = await buildBrowserHighlightJS(languages, { minify: options.minify }); + const size = await buildCore("highlight", languages, { minify: options.minify, format: "cjs" }); log("-----"); - log("Core :", size.core, "bytes"); - if (options.minify) { log("Core (min) :", size.core_min, "bytes"); } log("Languages (raw) :", languages.map((el) => el.data.length).reduce((acc, curr) => acc + curr, 0), "bytes"); - log("highlight.js :", size.full, "bytes"); + log("highlight.js :", size.fullSize, "bytes"); if (options.minify) { log("highlight.min.js :", size.minified, "bytes"); log("highlight.min.js.gz :", zlib.gzipSync(size.minifiedSrc).length, "bytes"); @@ -175,93 +178,76 @@ function installDemoStyles() { }); } -async function buildBrowserHighlightJS(languages, { minify }) { - log("Building highlight.js."); - - const git_sha = child_process - .execSync("git rev-parse HEAD") - .toString().trim() - .slice(0, 10); - const versionDetails = { ...require("../package"), git_sha }; - const header = buildHeader(versionDetails); - - const outFile = `${process.env.BUILD_DIR}/highlight.js`; - const minifiedFile = outFile.replace(/js$/, "min.js"); - - const built_in_langs = { - name: "dynamicLanguages", - resolveId: (source) => { - if (source == "builtInLanguages") { return "builtInLanguages"} - return null; - }, - load: (id) => { - if (id == "builtInLanguages") { - const escape = (s) => "grmr_" + s.replace("-", "_"); - let src = ""; - src += languages.map((x) => `import ${escape(x.name)} from '${x.path}'`).join("\n"); - src += `\nexport {${languages.map((x) => escape(x.name)).join(",")}}`; - return src; - } - return null; +const builtInLanguagesPlugin = (languages) => ({ + name: "hljs-index", + resolveId(source) { + if (source === "builtInLanguages") { + return source; // this signals that rollup should not ask other plugins or check the file system to find this id } + return null; // other ids should be handled as usually + }, + load(id) { + const escape = (s) => "grmr_" + s.replace("-", "_"); + if (id === "builtInLanguages") { + return languages.map((lang) => + `export { default as ${escape(lang.name)} } from ${JSON.stringify(lang.path)};` + ).join("\n"); + } + return null; } - - const plugins = [...config.rollup.browser_core.input.plugins, built_in_langs]; - - const input = { ...config.rollup.browser_core.input, input: `src/stub.js`, plugins }; - const output = { ...config.rollup.browser_core.output, file: outFile }; - let librarySrc = await rollupCode(input, output); - - - // we don't use this, we just use it to get a size approximation for the build stats - const coreSrc = await rollupCode({ ...config.rollup.browser_core.input, input: `src/highlight.js`, plugins }, output); - const coreSize = coreSrc.length; - - // strip off the original top comment - librarySrc = librarySrc.replace(/\/\*.*?\*\//s, ""); - - const fullSrc = [ - header, librarySrc, - // ...languages.map((lang) => lang.module) - ].join("\n"); - - const tasks = []; - tasks.push(fs.writeFile(outFile, fullSrc, { encoding: "utf8" })); - const shas = { - "highlight.js": bundling.sha384(fullSrc) +}); + +async function buildCore(name, languages, options) { + const header = buildHeader(); + let relativePath = ""; + const input = { + ...config.rollup.core.input, + input: `src/stub.js` + }; + input.plugins = [ + ...input.plugins, + builtInLanguagesPlugin(languages) + ]; + const output = { + ...(options.format === "es" ? config.rollup.node.output : config.rollup.browser_iife.output), + file: `${process.env.BUILD_DIR}/${name}.js` }; - let core_min = []; - let minifiedSrc = ""; - - if (minify) { - const tersed = await Terser.minify(librarySrc, config.terser); - const tersedCore = await Terser.minify(coreSrc, config.terser); + // optimize for no languages by not including the language loading stub + if (languages.length === 0) { + input.input = "src/highlight.js"; + } - minifiedSrc = [ - header, tersed.code, - // ...languages.map((lang) => lang.minified) - ].join("\n"); + if (options.format === "es") { + output.format = "es"; + output.file = `${process.env.BUILD_DIR}/es/${name}.js`; + relativePath = "es/"; + } - // get approximate core minified size - core_min = [header, tersedCore.code].join().length; + log(`Building ${relativePath}${name}.js.`); - tasks.push(fs.writeFile(minifiedFile, minifiedSrc, { encoding: "utf8" })); - shas["highlight.min.js"] = bundling.sha384(minifiedSrc); + const index = await rollupCode(input, output); + const sizeInfo = { shas: [] }; + const writePromises = []; + if (options.minify) { + const { code } = await Terser.minify(index, {...config.terser, module: (options.format === "es") }); + const src = `${header}\n${code}`; + writePromises.push(fs.writeFile(output.file.replace(/js$/, "min.js"), src)); + sizeInfo.minified = src.length; + sizeInfo.minifiedSrc = src; + sizeInfo.shas[`${relativePath}${name}.min.js`] = bundling.sha384(src) } - - await Promise.all(tasks); - return { - core: coreSize, - core_min: core_min, - minified: Buffer.byteLength(minifiedSrc, 'utf8'), - minifiedSrc, - fullSrc, - shas, - full: Buffer.byteLength(fullSrc, 'utf8') - }; + { + const src = `${header}\n${index}`; + writePromises.push(fs.writeFile(output.file, src)); + sizeInfo.fullSize = src.length; + sizeInfo.fullSrc = src; + sizeInfo.shas[`${relativePath}${name}.js`] = bundling.sha384(src) + } + await Promise.all(writePromises); + return sizeInfo; } // CDN build uses the exact same highlight.js distributable -module.exports.buildBrowserHighlightJS = buildBrowserHighlightJS; +module.exports.buildCore = buildCore; module.exports.build = buildBrowser; diff --git a/tools/build_cdn.js b/tools/build_cdn.js index 86c9f1214c..9cc3463ea2 100644 --- a/tools/build_cdn.js +++ b/tools/build_cdn.js @@ -2,27 +2,28 @@ const fs = require("fs").promises; const fss = require("fs"); const glob = require("glob"); const zlib = require('zlib'); -const { getLanguages } = require("./lib/language"); -const { filter } = require("./lib/dependencies"); -const config = require("./build_config"); -const { install, installCleanCSS, mkdir } = require("./lib/makestuff"); +const { getLanguages } = require("./lib/language.js"); +const { filter } = require("./lib/dependencies.js"); +const config = require("./build_config.js"); +const { install, installCleanCSS, mkdir } = require("./lib/makestuff.js"); const log = (...args) => console.log(...args); -const { buildBrowserHighlightJS } = require("./build_browser"); -const { buildPackageJSON } = require("./build_node"); +const { buildCore } = require("./build_browser.js"); +const { buildPackageJSON, writePackageJSON } = require("./build_node.js"); const path = require("path"); const bundling = require('./lib/bundling.js'); async function installPackageJSON(options) { - await buildPackageJSON(options); - const json = require(`${process.env.BUILD_DIR}/package`); + const json = buildPackageJSON(options); json.name = "@highlightjs/cdn-assets"; json.description = json.description.concat(" (pre-compiled CDN assets)"); // this is not a replacement for `highlightjs` package + // CDN assets do not need an export map, they are just a bunch of files. + // The NPM package mostly only exists to populate CDNs and provide raw files. delete json.exports; delete json.type; delete json.main; delete json.types; - fs.writeFile(`${process.env.BUILD_DIR}/package.json`, JSON.stringify(json, null, ' ')); + await writePackageJSON(json); } let shas = {}; @@ -30,13 +31,17 @@ let shas = {}; async function buildCDN(options) { install("./LICENSE", "LICENSE"); install("./README.CDN.md", "README.md"); - installPackageJSON(options); + await installPackageJSON(options); installStyles(); // all the languages are built for the CDN and placed into `/languages` const languages = await getLanguages(); - await installLanguages(languages); + + let esmCoreSize = {}; + let esmCommonSize = {}; + + await installLanguages(languages, options); // filter languages for inclusion in the highlight.js bundle let embedLanguages = filter(languages, options.languages); @@ -48,24 +53,44 @@ async function buildCDN(options) { embedLanguages = []; } - const size = await buildBrowserHighlightJS(embedLanguages, { minify: options.minify }); - shas = Object.assign({}, size.shas, shas); + const size = await buildCore("highlight", embedLanguages, { minify: options.minify, format: "cjs" }); + if (options.esm) { + mkdir("es"); + await fs.writeFile(`${process.env.BUILD_DIR}/es/package.json`, `{ "type": "module" }`); + esmCoreSize = await buildCore("core", [], {minify: options.minify, format: "es"}); + esmCommonSize = await buildCore("highlight", embedLanguages, { minify: options.minify, format: "es" }); + } + shas = { + ...size.shas, ...esmCommonSize.shas, ...esmCoreSize.shas, ...shas + }; await buildSRIDigests(shas); log("-----"); - log("Embedded Lang :", + log("Embedded Lang :", embedLanguages.map((el) => el.minified.length).reduce((acc, curr) => acc + curr, 0), "bytes"); - log("All Lang :", + log("All Lang :", languages.map((el) => el.minified.length).reduce((acc, curr) => acc + curr, 0), "bytes"); - log("highlight.js :", - size.full, "bytes"); + log("highlight.js :", + size.fullSize, "bytes"); if (options.minify) { - log("highlight.min.js :", size.minified, "bytes"); - log("highlight.min.js.gz :", zlib.gzipSync(size.minifiedSrc).length, "bytes"); + log("highlight.min.js :", size.minified, "bytes"); + log("highlight.min.js.gz :", zlib.gzipSync(size.minifiedSrc).length, "bytes"); } else { - log("highlight.js.gz :", zlib.gzipSync(size.fullSrc).length, "bytes"); + log("highlight.js.gz :", zlib.gzipSync(size.fullSrc).length, "bytes"); + } + if (options.esm) { + log("es/core.js :", esmCoreSize.fullSize, "bytes"); + log("es/highlight.js :", esmCommonSize.fullSize, "bytes"); + if (options.minify) { + log("es/core.min.js :", esmCoreSize.minified, "bytes"); + log("es/core.min.js.gz :", zlib.gzipSync(esmCoreSize.minifiedSrc).length, "bytes"); + log("es/highlight.min.js :", esmCommonSize.minified, "bytes"); + log("es/highlight.min.js.gz :", zlib.gzipSync(esmCommonSize.minifiedSrc).length, "bytes"); + } else { + log("es/highlight.js.gz :", zlib.gzipSync(esmCommonSize.fullSrc).length, "bytes"); + } } log("-----"); } @@ -75,7 +100,7 @@ async function buildSRIDigests(shas) { const temp = await fs.readFile("./tools/templates/DIGESTS.md"); const DIGEST_MD = temp.toString(); - const version = require("../package").version; + const version = require("../package.json").version; const digestList = Object.entries(shas).map(([k, v]) => `${v} ${k}`).join("\n"); const out = DIGEST_MD @@ -86,13 +111,14 @@ async function buildSRIDigests(shas) { fs.writeFile(`${process.env.BUILD_DIR}/DIGESTS.md`, out); } -async function installLanguages(languages) { +async function installLanguages(languages, options) { log("Building language files."); mkdir("languages"); + if (options.esm) mkdir("es/languages"); await Promise.all( languages.map(async(language) => { - await buildCDNLanguage(language); + await buildCDNLanguage(language, options); process.stdout.write("."); }) ); @@ -100,9 +126,7 @@ async function installLanguages(languages) { await Promise.all( languages.filter((l) => l.third_party) - .map(async(language) => { - await buildDistributable(language); - }) + .map(async(lang) => await buildDistributable(lang, options)) ); log(""); @@ -125,22 +149,28 @@ function installStyles() { }); } -async function buildDistributable(language) { +async function buildDistributable(language, options) { const filename = `${language.name}.min.js`; const distDir = path.join(language.moduleDir, "dist"); log(`Building ${distDir}/${filename}.`); await fs.mkdir(distDir, { recursive: true }); - fs.writeFile(path.join(language.moduleDir, "dist", filename), language.minified); + await fs.writeFile(path.join(language.moduleDir, "dist", filename), language.minified); + if (options.esm) { + await fs.writeFile(path.join(language.moduleDir, "dist", filename.replace(".min.js",".es.min.js")), language.minifiedESM); + } } -async function buildCDNLanguage(language) { +async function buildCDNLanguage(language, options) { const name = `languages/${language.name}.min.js`; - const filename = `${process.env.BUILD_DIR}/${name}`; await language.compile({ terser: config.terser }); shas[name] = bundling.sha384(language.minified); - fs.writeFile(filename, language.minified); + await fs.writeFile(`${process.env.BUILD_DIR}/${name}`, language.minified); + if (options.esm) { + shas[`es/${name}`] = bundling.sha384(language.minifiedESM); + await fs.writeFile(`${process.env.BUILD_DIR}/es/${name}`, language.minifiedESM); + } } module.exports.build = buildCDN; diff --git a/tools/build_config.js b/tools/build_config.js index b522e54a49..6d6557f4ab 100644 --- a/tools/build_config.js +++ b/tools/build_config.js @@ -9,13 +9,13 @@ module.exports = { level: 2 }, rollup: { - node: { - output: { format: "cjs", strict: false, exports: "auto" }, + core: { input: { plugins: [ cjsPlugin(), jsonPlugin(), nodeResolve(), + // TODO: remove with version 12 { transform: (x) => { if (/var module/.exec(x)) { @@ -27,7 +27,10 @@ module.exports = { ] } }, - browser_core: { + node: { + output: { format: "cjs", strict: false, exports: "auto" } + }, + browser_iife: { input: { plugins: [ jsonPlugin(), @@ -41,19 +44,6 @@ module.exports = { footer: "if (typeof exports === 'object' && typeof module !== 'undefined') { module.exports = hljs; }", interop: false } - }, - browser: { - input: { - plugins: [ - cjsPlugin(), - jsonPlugin() - ] - }, - output: { - format: "iife", - outro: "return module.exports.definer || module.exports;", - interop: false - } } }, terser: { diff --git a/tools/build_node.js b/tools/build_node.js index 94463a2131..235061a4fe 100644 --- a/tools/build_node.js +++ b/tools/build_node.js @@ -45,7 +45,7 @@ async function buildNodeLanguage(language, options) { const ES_STUB = `${EMIT} import lang from './%%%%.js'; export default lang;`; - const input = { ...config.rollup.node.input, input: language.path }; + const input = { ...config.rollup.core.input, input: language.path }; const output = { ...config.rollup.node.output, file: `${process.env.BUILD_DIR}/lib/languages/${language.name}.js` }; await rollupWrite(input, output); await fs.writeFile(`${process.env.BUILD_DIR}/lib/languages/${language.name}.js.js`, @@ -63,7 +63,7 @@ async function buildNodeLanguage(language, options) { const EXCLUDE = ["join"]; async function buildESMUtils() { - const input = { ...config.rollup.node.input, input: `src/lib/regex.js` }; + const input = { ...config.rollup.core.input, input: `src/lib/regex.js` }; input.plugins = [...input.plugins, { transform: (code) => { EXCLUDE.forEach((fn) => { @@ -80,7 +80,7 @@ async function buildESMUtils() { } async function buildNodeHighlightJS(options) { - const input = { ...config.rollup.node.input, input: `src/highlight.js` }; + const input = { ...config.rollup.core.input, input: `src/highlight.js` }; const output = { ...config.rollup.node.output, file: `${process.env.BUILD_DIR}/lib/core.js` }; await rollupWrite(input, output); if (options.esm) { @@ -95,22 +95,25 @@ function dual(file) { }; } -async function buildPackageJSON(options) { - const packageJson = require("../package"); - - const exports = { - ".": dual("./lib/index.js"), - "./package.json": "./package.json", - "./lib/common": dual("./lib/common.js"), - "./lib/core": dual("./lib/core.js"), - "./lib/languages/*": dual("./lib/languages/*.js"), - "./scss/*": "./scss/*", - "./styles/*": "./styles/*", - "./types/*": "./types/*", - }; - if (options.esm) packageJson.exports = exports; - - await fs.writeFile(`${process.env.BUILD_DIR}/package.json`, JSON.stringify(packageJson, null, 2)); +const generatePackageExports = () => ({ + ".": dual("./lib/index.js"), + "./package.json": "./package.json", + "./lib/common": dual("./lib/common.js"), + "./lib/core": dual("./lib/core.js"), + "./lib/languages/*": dual("./lib/languages/*.js"), + "./scss/*": "./scss/*", + "./styles/*": "./styles/*", + "./types/*": "./types/*", +}); +function buildPackageJSON(options) { + const packageJson = require("../package.json"); + + if (options.esm) packageJson.exports = generatePackageExports(); + + return packageJson; +} +function writePackageJSON(packageJson) { + return fs.writeFile(`${process.env.BUILD_DIR}/package.json`, JSON.stringify(packageJson, null, 2)); } async function buildLanguages(languages, options) { @@ -170,7 +173,7 @@ async function buildNode(options) { const common = languages.filter(l => l.categories.includes("common")); log("Writing package.json."); - await buildPackageJSON(options); + await writePackageJSON(buildPackageJSON(options)); if (options.esm) { await fs.writeFile(`${process.env.BUILD_DIR}/es/package.json`, `{ "type": "module" }`); @@ -188,3 +191,4 @@ async function buildNode(options) { module.exports.build = buildNode; module.exports.buildPackageJSON = buildPackageJSON; +module.exports.writePackageJSON = writePackageJSON; diff --git a/tools/lib/bundling.js b/tools/lib/bundling.js index c3c63df80f..1ca11a71c6 100644 --- a/tools/lib/bundling.js +++ b/tools/lib/bundling.js @@ -22,7 +22,7 @@ function sha384(contents) { const hash = crypto.createHash('sha384'); const data = hash.update(contents, 'utf-8'); const gen_hash = data.digest('base64'); - return `sha384-${gen_hash}` + return `sha384-${gen_hash}`; } module.exports = { rollupWrite, rollupCode, sha384 }; diff --git a/tools/lib/language.js b/tools/lib/language.js index 5873d79ccb..0e25f973c0 100644 --- a/tools/lib/language.js +++ b/tools/lib/language.js @@ -79,20 +79,22 @@ class Language { async function compileLanguage (language, options) { - const EXPORT_REGEX = /export default (.*);/; - const IIFE_HEADER_REGEX = /^(var dummyName = )?\(function \(\)/; + const IIFE_HEADER_REGEX = /^(var hljsGrammar = )?\(function \(\)/; // TODO: cant we use the source we already have? - const input = { ...build_config.rollup.browser.input, input: language.path }; - const output = { ...build_config.rollup.browser.output, name: `dummyName`, file: "out.js" }; - var data = await rollupCode(input, output) - - data = data.replace(IIFE_HEADER_REGEX, `hljs.registerLanguage('${language.name}', function ()`) - - var original = data; - language.module = data; - data = await Terser.minify(data, options.terser); - language.minified = data.code || original; + const input = { ...build_config.rollup.browser_iife.input, input: language.path }; + const output = { ...build_config.rollup.browser_iife.output, name: `hljsGrammar`, file: "out.js" }; + output.footer = null; + + const data = await rollupCode(input, output); + const iife = data.replace(IIFE_HEADER_REGEX, `hljs.registerLanguage('${language.name}', function ()`); + const esm = `${data};\nexport default hljsGrammar;`; + + language.module = iife; + const miniESM = await Terser.minify(esm, options.terser); + const miniIIFE = await Terser.minify(iife, options.terser); + language.minified = miniIIFE.code; + language.minifiedESM = miniESM.code; } async function getLanguages() {