From 2ca66c3b1ec808218f931726c9ed81536bbf77ff Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Sun, 30 May 2021 14:55:54 +0200 Subject: [PATCH 01/25] (feat) include ESM in CDN build Fixes: https://github.com/highlightjs/highlight.js/issues/3199 --- tools/build_cdn.js | 89 +++++++++++++++++++++++++++++++++++-------- tools/build_config.js | 8 +--- tools/build_node.js | 30 ++++++++------- tools/lib/language.js | 18 ++++----- 4 files changed, 102 insertions(+), 43 deletions(-) diff --git a/tools/build_cdn.js b/tools/build_cdn.js index 86c9f1214c..b15870d8bc 100644 --- a/tools/build_cdn.js +++ b/tools/build_cdn.js @@ -1,6 +1,7 @@ const fs = require("fs").promises; const fss = require("fs"); const glob = require("glob"); +const Terser = require("terser"); const zlib = require('zlib'); const { getLanguages } = require("./lib/language"); const { filter } = require("./lib/dependencies"); @@ -8,13 +9,26 @@ const config = require("./build_config"); const { install, installCleanCSS, mkdir } = require("./lib/makestuff"); const log = (...args) => console.log(...args); const { buildBrowserHighlightJS } = require("./build_browser"); -const { buildPackageJSON } = require("./build_node"); +const { buildPackageJSON, writePackageJSON } = require("./build_node"); +const { rollupCode } = require("./lib/bundling.js"); const path = require("path"); const bundling = require('./lib/bundling.js'); async function installPackageJSON(options) { - await buildPackageJSON(options); - const json = require(`${process.env.BUILD_DIR}/package`); + const json = buildPackageJSON(options, { + ".": { + import: "./es/index.js", + browser: "./highlight.min.js", + }, + "./lib/languages/*": { + import: "./es/languages/*.js", + browser: "./languages/*.js" + }, + "./lib/common": { import: "./es/index.js" }, + "./lib/core": { import: "./es/core.js" }, + "./styles/*": "./styles/*", + "./package.json": "./package.json", + }); json.name = "@highlightjs/cdn-assets"; json.description = json.description.concat(" (pre-compiled CDN assets)"); // this is not a replacement for `highlightjs` package @@ -22,7 +36,41 @@ async function installPackageJSON(options) { delete json.type; delete json.main; delete json.types; - fs.writeFile(`${process.env.BUILD_DIR}/package.json`, JSON.stringify(json, null, ' ')); + await writePackageJSON(json); +} + +const safeImportName = (s) => { + s = s.replace(/-/g, "_"); + if (/^\d/.test(s)) s = `L_${s}`; + return s; +}; + +async function buildESMIndex(name, languages) { + const header = `import hljs from './core.js';`; + const footer = "export default hljs;"; + + const registration = languages.map((lang) => { + const importName = safeImportName(lang.name); + return `import ${importName} from './languages/${lang.name}.js';\n` + + `hljs.registerLanguage('${lang.name}', ${importName});`; + }); + + const index = `${header}\n\n${registration.join("\n")}\n\n${footer}`; + await fs.writeFile(`${process.env.BUILD_DIR}/es/${name}.js`, index); +} + +async function buildESMCore(options) { + const input = { ...config.rollup.node.input, input: `src/highlight.js` }; + const output = { + ...config.rollup.node.output, + format: "es", + file: `${process.env.BUILD_DIR}/es/core.js`, + }; + const core = await rollupCode(input, output); + + const miniCore = options.minify ? await Terser.minify(core, config.terser) : { code: core }; + await fs.writeFile(output.file, miniCore.code || core); + return miniCore.code.length; } let shas = {}; @@ -30,13 +78,21 @@ let shas = {}; async function buildCDN(options) { install("./LICENSE", "LICENSE"); install("./README.CDN.md", "README.md"); - installPackageJSON(options); + await installPackageJSON(options); installStyles(); // all the languages are built for the CDN and placed into `/languages` const languages = await getLanguages(); - await installLanguages(languages); + + let esmCoreSize; + if (options.esm) { + mkdir("es"); + await fs.writeFile(`${process.env.BUILD_DIR}/es/package.json`, `{ "type": "module" }`); + esmCoreSize = await buildESMCore(options); + } + + await installLanguages(languages, options); // filter languages for inclusion in the highlight.js bundle let embedLanguages = filter(languages, options.languages); @@ -49,6 +105,7 @@ async function buildCDN(options) { } const size = await buildBrowserHighlightJS(embedLanguages, { minify: options.minify }); + if (options.esm) await buildESMIndex("index", embedLanguages, { minify: options.minify }); shas = Object.assign({}, size.shas, shas); await buildSRIDigests(shas); @@ -60,6 +117,8 @@ async function buildCDN(options) { languages.map((el) => el.minified.length).reduce((acc, curr) => acc + curr, 0), "bytes"); log("highlight.js :", size.full, "bytes"); + if(options.esm) + log("es/core.js :", esmCoreSize, "bytes"); if (options.minify) { log("highlight.min.js :", size.minified, "bytes"); @@ -86,13 +145,14 @@ async function buildSRIDigests(shas) { fs.writeFile(`${process.env.BUILD_DIR}/DIGESTS.md`, out); } -async function installLanguages(languages) { +async function installLanguages(languages, options) { log("Building language files."); mkdir("languages"); + if(options.esm) mkdir("es/languages"); await Promise.all( languages.map(async(language) => { - await buildCDNLanguage(language); + await buildCDNLanguage(language, options); process.stdout.write("."); }) ); @@ -100,9 +160,7 @@ async function installLanguages(languages) { await Promise.all( languages.filter((l) => l.third_party) - .map(async(language) => { - await buildDistributable(language); - }) + .map(buildDistributable) ); log(""); @@ -131,16 +189,17 @@ async function buildDistributable(language) { const distDir = path.join(language.moduleDir, "dist"); log(`Building ${distDir}/${filename}.`); await fs.mkdir(distDir, { recursive: true }); - fs.writeFile(path.join(language.moduleDir, "dist", filename), language.minified); + await fs.writeFile(path.join(language.moduleDir, "dist", filename), language.minified); } -async function buildCDNLanguage(language) { +async function buildCDNLanguage(language, options) { const name = `languages/${language.name}.min.js`; - const filename = `${process.env.BUILD_DIR}/${name}`; await language.compile({ terser: config.terser }); shas[name] = bundling.sha384(language.minified); - fs.writeFile(filename, language.minified); + await fs.writeFile(`${process.env.BUILD_DIR}/${name}`, language.minified); + if (options.esm) + await fs.writeFile(`${process.env.BUILD_DIR}/es/${name}`, language.minifiedESM); } module.exports.build = buildCDN; diff --git a/tools/build_config.js b/tools/build_config.js index b522e54a49..3cedd90663 100644 --- a/tools/build_config.js +++ b/tools/build_config.js @@ -44,14 +44,10 @@ module.exports = { }, browser: { input: { - plugins: [ - cjsPlugin(), - jsonPlugin() - ] + plugins: [] }, output: { - format: "iife", - outro: "return module.exports.definer || module.exports;", + format: "es", interop: false } } diff --git a/tools/build_node.js b/tools/build_node.js index 94463a2131..c4a928d367 100644 --- a/tools/build_node.js +++ b/tools/build_node.js @@ -95,22 +95,25 @@ function dual(file) { }; } -async function buildPackageJSON(options) { +const generatePackageExports = () => ({ + ".": dual("./lib/index.js"), + "./package.json": "./package.json", + "./lib/common": dual("./lib/common.js"), + "./lib/core": dual("./lib/core.js"), + "./lib/languages/*": dual("./lib/languages/*.js"), + "./scss/*": "./scss/*", + "./styles/*": "./styles/*", + "./types/*": "./types/*", +}); +function buildPackageJSON(options, exports = generatePackageExports()) { const packageJson = require("../package"); - const exports = { - ".": dual("./lib/index.js"), - "./package.json": "./package.json", - "./lib/common": dual("./lib/common.js"), - "./lib/core": dual("./lib/core.js"), - "./lib/languages/*": dual("./lib/languages/*.js"), - "./scss/*": "./scss/*", - "./styles/*": "./styles/*", - "./types/*": "./types/*", - }; if (options.esm) packageJson.exports = exports; - await fs.writeFile(`${process.env.BUILD_DIR}/package.json`, JSON.stringify(packageJson, null, 2)); + return packageJson; +} +function writePackageJSON(packageJson) { + return fs.writeFile(`${process.env.BUILD_DIR}/package.json`, JSON.stringify(packageJson, null, 2)); } async function buildLanguages(languages, options) { @@ -170,7 +173,7 @@ async function buildNode(options) { const common = languages.filter(l => l.categories.includes("common")); log("Writing package.json."); - await buildPackageJSON(options); + await writePackageJSON(buildPackageJSON(options)); if (options.esm) { await fs.writeFile(`${process.env.BUILD_DIR}/es/package.json`, `{ "type": "module" }`); @@ -188,3 +191,4 @@ async function buildNode(options) { module.exports.build = buildNode; module.exports.buildPackageJSON = buildPackageJSON; +module.exports.writePackageJSON = writePackageJSON; diff --git a/tools/lib/language.js b/tools/lib/language.js index 5873d79ccb..d5124a80f4 100644 --- a/tools/lib/language.js +++ b/tools/lib/language.js @@ -80,19 +80,19 @@ class Language { async function compileLanguage (language, options) { const EXPORT_REGEX = /export default (.*);/; - const IIFE_HEADER_REGEX = /^(var dummyName = )?\(function \(\)/; // TODO: cant we use the source we already have? const input = { ...build_config.rollup.browser.input, input: language.path }; const output = { ...build_config.rollup.browser.output, name: `dummyName`, file: "out.js" }; - var data = await rollupCode(input, output) - - data = data.replace(IIFE_HEADER_REGEX, `hljs.registerLanguage('${language.name}', function ()`) - - var original = data; - language.module = data; - data = await Terser.minify(data, options.terser); - language.minified = data.code || original; + + const esm = await rollupCode(input, output); + const iife = `hljs.registerLanguage('${language.name}', function (){"use strict";` + esm.replace(EXPORT_REGEX, 'return $1') + '})'; + + language.module = iife; + const miniESM = await Terser.minify(esm, options.terser); + const miniIIFE = await Terser.minify(iife, options.terser); + language.minified = miniIIFE.code || iife; + language.minifiedESM = miniESM.code || esm; } async function getLanguages() { From 38d6531a93f71ceaf687fb9ab2b6d18f7358b8f9 Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Tue, 1 Jun 2021 16:53:01 +0200 Subject: [PATCH 02/25] Add common ESM build --- tools/build_browser.js | 81 ++++++++++++++++++++++++++++++++++-------- tools/build_cdn.js | 53 ++++++++++++--------------- 2 files changed, 88 insertions(+), 46 deletions(-) diff --git a/tools/build_browser.js b/tools/build_browser.js index 02cb891bb3..d3e6136d8f 100644 --- a/tools/build_browser.js +++ b/tools/build_browser.js @@ -14,7 +14,8 @@ const log = (...args) => console.log(...args); const { rollupCode } = require("./lib/bundling.js"); const bundling = require('./lib/bundling.js'); const Table = require('cli-table'); -const { result } = require('lodash'); + +const textEncoder = new TextEncoder(); function buildHeader(args) { return "/*!\n" + @@ -214,16 +215,13 @@ async function buildBrowserHighlightJS(languages, { minify }) { // we don't use this, we just use it to get a size approximation for the build stats - const coreSrc = await rollupCode({ ...config.rollup.browser_core.input, input: `src/highlight.js`, plugins }, output); + const coreSrc = textEncoder.encode(await rollupCode({ ...config.rollup.browser_core.input, input: `src/highlight.js`, plugins }, output)); const coreSize = coreSrc.length; // strip off the original top comment librarySrc = librarySrc.replace(/\/\*.*?\*\//s, ""); - const fullSrc = [ - header, librarySrc, - // ...languages.map((lang) => lang.module) - ].join("\n"); + const fullSrc = textEncoder.encode(`${header}\n${librarySrc}`); const tasks = []; tasks.push(fs.writeFile(outFile, fullSrc, { encoding: "utf8" })); @@ -232,19 +230,15 @@ async function buildBrowserHighlightJS(languages, { minify }) { }; let core_min = []; - let minifiedSrc = ""; + let minifiedSrc; if (minify) { const tersed = await Terser.minify(librarySrc, config.terser); - const tersedCore = await Terser.minify(coreSrc, config.terser); - minifiedSrc = [ - header, tersed.code, - // ...languages.map((lang) => lang.minified) - ].join("\n"); + minifiedSrc = textEncoder.encode(`${header}\n${tersed.code}`); // get approximate core minified size - core_min = [header, tersedCore.code].join().length; + core_min = minifiedSrc.length; tasks.push(fs.writeFile(minifiedFile, minifiedSrc, { encoding: "utf8" })); shas["highlight.min.js"] = bundling.sha384(minifiedSrc); @@ -254,14 +248,71 @@ async function buildBrowserHighlightJS(languages, { minify }) { return { core: coreSize, core_min: core_min, - minified: Buffer.byteLength(minifiedSrc, 'utf8'), + minified: minifiedSrc.length, minifiedSrc, fullSrc, shas, - full: Buffer.byteLength(fullSrc, 'utf8') + full: fullSrc.length, }; } +const safeImportName = (s) => { + s = s.replace(/-/g, "_"); + if (/^\d/.test(s)) s = `L_${s}`; + return s; +}; + +async function buildBrowserESMHighlightJS(name, languages, options) { + const input = { ...config.rollup.node.input, input: `src/stub.js`, plugins: [ + ...config.rollup.node.input.plugins, + { + name: "hljs-index", + resolveId(source) { + if (source === "builtInLanguages") { + return source; // this signals that rollup should not ask other plugins or check the file system to find this id + } + return null; // other ids should be handled as usually + }, + load(id) { + if (id === "builtInLanguages") { + return languages.map((lang) => { + const importName = safeImportName(lang.name); + return `import ${importName} from './languages/${lang.name}.js';\n` + + `hljs.registerLanguage('${lang.name}', ${importName});`; + }).join("\n"); + } + return null; + }, + }, + ] }; + const output = { + ...config.rollup.node.output, + format: "es", + file: `${process.env.BUILD_DIR}/es/${name}.js`, + }; + + + const index = await rollupCode(input, output); + const sizeInfo = {} + const writePromises = [] + if (options.minify) { + const { code } = await Terser.minify(index, {...config.terser, module: true}) + const buf = textEncoder.encode(code); + writePromises.push(fs.writeFile(output.file.replace(/js$/, "min.js"), buf)); + sizeInfo.minified = buf.length; + sizeInfo.minifiedSrc = buf; + } + { + const buf = textEncoder.encode(index); + writePromises.push(fs.writeFile(output.file, buf)); + sizeInfo.fullSize = buf.length; + sizeInfo.fullSrc = buf; + } + await Promise.all(writePromises); + return sizeInfo; +} + // CDN build uses the exact same highlight.js distributable module.exports.buildBrowserHighlightJS = buildBrowserHighlightJS; +module.exports.buildBrowserESMHighlightJS = buildBrowserESMHighlightJS; module.exports.build = buildBrowser; diff --git a/tools/build_cdn.js b/tools/build_cdn.js index b15870d8bc..8cb6d48991 100644 --- a/tools/build_cdn.js +++ b/tools/build_cdn.js @@ -8,23 +8,25 @@ const { filter } = require("./lib/dependencies"); const config = require("./build_config"); const { install, installCleanCSS, mkdir } = require("./lib/makestuff"); const log = (...args) => console.log(...args); -const { buildBrowserHighlightJS } = require("./build_browser"); +const { buildBrowserHighlightJS, buildBrowserESMHighlightJS } = require("./build_browser"); const { buildPackageJSON, writePackageJSON } = require("./build_node"); const { rollupCode } = require("./lib/bundling.js"); const path = require("path"); const bundling = require('./lib/bundling.js'); +const textEncoder = new TextEncoder(); + async function installPackageJSON(options) { const json = buildPackageJSON(options, { ".": { - import: "./es/index.js", - browser: "./highlight.min.js", + import: options.minify ? "./es/index.min.js" : "./es/index.js", + browser: options.minify ? "./highlight.min.js" : "./highlight.js", }, "./lib/languages/*": { import: "./es/languages/*.js", browser: "./languages/*.js" }, - "./lib/common": { import: "./es/index.js" }, + get "./lib/common"(){ return this["."]; }, "./lib/core": { import: "./es/core.js" }, "./styles/*": "./styles/*", "./package.json": "./package.json", @@ -39,26 +41,6 @@ async function installPackageJSON(options) { await writePackageJSON(json); } -const safeImportName = (s) => { - s = s.replace(/-/g, "_"); - if (/^\d/.test(s)) s = `L_${s}`; - return s; -}; - -async function buildESMIndex(name, languages) { - const header = `import hljs from './core.js';`; - const footer = "export default hljs;"; - - const registration = languages.map((lang) => { - const importName = safeImportName(lang.name); - return `import ${importName} from './languages/${lang.name}.js';\n` + - `hljs.registerLanguage('${lang.name}', ${importName});`; - }); - - const index = `${header}\n\n${registration.join("\n")}\n\n${footer}`; - await fs.writeFile(`${process.env.BUILD_DIR}/es/${name}.js`, index); -} - async function buildESMCore(options) { const input = { ...config.rollup.node.input, input: `src/highlight.js` }; const output = { @@ -68,9 +50,10 @@ async function buildESMCore(options) { }; const core = await rollupCode(input, output); - const miniCore = options.minify ? await Terser.minify(core, config.terser) : { code: core }; - await fs.writeFile(output.file, miniCore.code || core); - return miniCore.code.length; + const miniCore = options.minify ? await Terser.minify(core, {...config.terser, module: true}) : { code: core }; + const code = textEncoder.encode(miniCore.code || core); + await fs.writeFile(output.file, code); + return code.length; } let shas = {}; @@ -85,7 +68,7 @@ async function buildCDN(options) { // all the languages are built for the CDN and placed into `/languages` const languages = await getLanguages(); - let esmCoreSize; + let esmCoreSize, esmIndexSize; if (options.esm) { mkdir("es"); await fs.writeFile(`${process.env.BUILD_DIR}/es/package.json`, `{ "type": "module" }`); @@ -105,7 +88,7 @@ async function buildCDN(options) { } const size = await buildBrowserHighlightJS(embedLanguages, { minify: options.minify }); - if (options.esm) await buildESMIndex("index", embedLanguages, { minify: options.minify }); + if (options.esm) esmIndexSize = await buildBrowserESMHighlightJS("index", embedLanguages, { minify: options.minify }); shas = Object.assign({}, size.shas, shas); await buildSRIDigests(shas); @@ -117,8 +100,6 @@ async function buildCDN(options) { languages.map((el) => el.minified.length).reduce((acc, curr) => acc + curr, 0), "bytes"); log("highlight.js :", size.full, "bytes"); - if(options.esm) - log("es/core.js :", esmCoreSize, "bytes"); if (options.minify) { log("highlight.min.js :", size.minified, "bytes"); @@ -126,6 +107,16 @@ async function buildCDN(options) { } else { log("highlight.js.gz :", zlib.gzipSync(size.fullSrc).length, "bytes"); } + if(options.esm) { + log("es/index.js :", esmIndexSize.fullSize, "bytes"); + log("es/core.js :", esmCoreSize, "bytes"); + if (options.minify) { + log("es/index.min.js :", esmIndexSize.minified, "bytes"); + log("es/index.min.js.gz :", zlib.gzipSync(esmIndexSize.minifiedSrc).length, "bytes"); + } else { + log("es/index.js.gz :", zlib.gzipSync(esmIndexSize.fullSrc).length, "bytes"); + } + } log("-----"); } From aef0cab980fa339f2442a4b725517e68ed8cd1b5 Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Tue, 1 Jun 2021 17:03:59 +0200 Subject: [PATCH 03/25] Add header to ESM common dist files --- tools/build_browser.js | 22 ++++++++++++---------- tools/build_cdn.js | 2 +- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/tools/build_browser.js b/tools/build_browser.js index d3e6136d8f..10b8cc0909 100644 --- a/tools/build_browser.js +++ b/tools/build_browser.js @@ -17,7 +17,13 @@ const Table = require('cli-table'); const textEncoder = new TextEncoder(); -function buildHeader(args) { +const getDefaultHeader = () => ({ + ...require('../package.json'), + git_sha : child_process + .execSync("git rev-parse --short=10 HEAD") + .toString().trim(), +}); +function buildHeader(args = getDefaultHeader()) { return "/*!\n" + ` Highlight.js v${args.version} (git: ${args.git_sha})\n` + ` (c) ${config.copyrightYears} ${args.author.name} and other contributors\n` + @@ -179,12 +185,7 @@ function installDemoStyles() { async function buildBrowserHighlightJS(languages, { minify }) { log("Building highlight.js."); - const git_sha = child_process - .execSync("git rev-parse HEAD") - .toString().trim() - .slice(0, 10); - const versionDetails = { ...require("../package"), git_sha }; - const header = buildHeader(versionDetails); + const header = buildHeader(); const outFile = `${process.env.BUILD_DIR}/highlight.js`; const minifiedFile = outFile.replace(/js$/, "min.js"); @@ -263,6 +264,8 @@ const safeImportName = (s) => { }; async function buildBrowserESMHighlightJS(name, languages, options) { + log("Building highlight.mjs."); + const header = buildHeader(); const input = { ...config.rollup.node.input, input: `src/stub.js`, plugins: [ ...config.rollup.node.input.plugins, { @@ -291,19 +294,18 @@ async function buildBrowserESMHighlightJS(name, languages, options) { file: `${process.env.BUILD_DIR}/es/${name}.js`, }; - const index = await rollupCode(input, output); const sizeInfo = {} const writePromises = [] if (options.minify) { const { code } = await Terser.minify(index, {...config.terser, module: true}) - const buf = textEncoder.encode(code); + const buf = textEncoder.encode(`${header}\n${code}`); writePromises.push(fs.writeFile(output.file.replace(/js$/, "min.js"), buf)); sizeInfo.minified = buf.length; sizeInfo.minifiedSrc = buf; } { - const buf = textEncoder.encode(index); + const buf = textEncoder.encode(`${header}\n${index}`); writePromises.push(fs.writeFile(output.file, buf)); sizeInfo.fullSize = buf.length; sizeInfo.fullSrc = buf; diff --git a/tools/build_cdn.js b/tools/build_cdn.js index 8cb6d48991..2569155de4 100644 --- a/tools/build_cdn.js +++ b/tools/build_cdn.js @@ -108,8 +108,8 @@ async function buildCDN(options) { log("highlight.js.gz :", zlib.gzipSync(size.fullSrc).length, "bytes"); } if(options.esm) { - log("es/index.js :", esmIndexSize.fullSize, "bytes"); log("es/core.js :", esmCoreSize, "bytes"); + log("es/index.js :", esmIndexSize.fullSize, "bytes"); if (options.minify) { log("es/index.min.js :", esmIndexSize.minified, "bytes"); log("es/index.min.js.gz :", zlib.gzipSync(esmIndexSize.minifiedSrc).length, "bytes"); From ddbfae7f93c9c3dc1306ad33df481a45271747ec Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Wed, 2 Jun 2021 18:32:59 +0200 Subject: [PATCH 04/25] Remove `TextEncoder` uses --- tools/build_browser.js | 24 +++++++++++------------- tools/build_cdn.js | 7 ++----- 2 files changed, 13 insertions(+), 18 deletions(-) diff --git a/tools/build_browser.js b/tools/build_browser.js index 10b8cc0909..f484229467 100644 --- a/tools/build_browser.js +++ b/tools/build_browser.js @@ -15,8 +15,6 @@ const { rollupCode } = require("./lib/bundling.js"); const bundling = require('./lib/bundling.js'); const Table = require('cli-table'); -const textEncoder = new TextEncoder(); - const getDefaultHeader = () => ({ ...require('../package.json'), git_sha : child_process @@ -216,13 +214,13 @@ async function buildBrowserHighlightJS(languages, { minify }) { // we don't use this, we just use it to get a size approximation for the build stats - const coreSrc = textEncoder.encode(await rollupCode({ ...config.rollup.browser_core.input, input: `src/highlight.js`, plugins }, output)); + const coreSrc = await rollupCode({ ...config.rollup.browser_core.input, input: `src/highlight.js`, plugins }, output); const coreSize = coreSrc.length; // strip off the original top comment librarySrc = librarySrc.replace(/\/\*.*?\*\//s, ""); - const fullSrc = textEncoder.encode(`${header}\n${librarySrc}`); + const fullSrc = `${header}\n${librarySrc}`; const tasks = []; tasks.push(fs.writeFile(outFile, fullSrc, { encoding: "utf8" })); @@ -236,7 +234,7 @@ async function buildBrowserHighlightJS(languages, { minify }) { if (minify) { const tersed = await Terser.minify(librarySrc, config.terser); - minifiedSrc = textEncoder.encode(`${header}\n${tersed.code}`); + minifiedSrc = `${header}\n${tersed.code}`; // get approximate core minified size core_min = minifiedSrc.length; @@ -299,16 +297,16 @@ async function buildBrowserESMHighlightJS(name, languages, options) { const writePromises = [] if (options.minify) { const { code } = await Terser.minify(index, {...config.terser, module: true}) - const buf = textEncoder.encode(`${header}\n${code}`); - writePromises.push(fs.writeFile(output.file.replace(/js$/, "min.js"), buf)); - sizeInfo.minified = buf.length; - sizeInfo.minifiedSrc = buf; + const src = `${header}\n${code}`; + writePromises.push(fs.writeFile(output.file.replace(/js$/, "min.js"), src)); + sizeInfo.minified = src.length; + sizeInfo.minifiedSrc = src; } { - const buf = textEncoder.encode(`${header}\n${index}`); - writePromises.push(fs.writeFile(output.file, buf)); - sizeInfo.fullSize = buf.length; - sizeInfo.fullSrc = buf; + const src = `${header}\n${index}`; + writePromises.push(fs.writeFile(output.file, src)); + sizeInfo.fullSize = src.length; + sizeInfo.fullSrc = src; } await Promise.all(writePromises); return sizeInfo; diff --git a/tools/build_cdn.js b/tools/build_cdn.js index 2569155de4..6e30599232 100644 --- a/tools/build_cdn.js +++ b/tools/build_cdn.js @@ -14,8 +14,6 @@ const { rollupCode } = require("./lib/bundling.js"); const path = require("path"); const bundling = require('./lib/bundling.js'); -const textEncoder = new TextEncoder(); - async function installPackageJSON(options) { const json = buildPackageJSON(options, { ".": { @@ -51,9 +49,8 @@ async function buildESMCore(options) { const core = await rollupCode(input, output); const miniCore = options.minify ? await Terser.minify(core, {...config.terser, module: true}) : { code: core }; - const code = textEncoder.encode(miniCore.code || core); - await fs.writeFile(output.file, code); - return code.length; + await fs.writeFile(output.file, miniCore.code); + return miniCore.code.length; } let shas = {}; From 5201111e82fc181dd760bf3e6294bb52b9168ad1 Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Wed, 2 Jun 2021 18:39:19 +0200 Subject: [PATCH 05/25] Fix undefined length property --- tools/build_browser.js | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/tools/build_browser.js b/tools/build_browser.js index f484229467..26bad2b124 100644 --- a/tools/build_browser.js +++ b/tools/build_browser.js @@ -215,7 +215,6 @@ async function buildBrowserHighlightJS(languages, { minify }) { // we don't use this, we just use it to get a size approximation for the build stats const coreSrc = await rollupCode({ ...config.rollup.browser_core.input, input: `src/highlight.js`, plugins }, output); - const coreSize = coreSrc.length; // strip off the original top comment librarySrc = librarySrc.replace(/\/\*.*?\*\//s, ""); @@ -228,8 +227,7 @@ async function buildBrowserHighlightJS(languages, { minify }) { "highlight.js": bundling.sha384(fullSrc) }; - let core_min = []; - let minifiedSrc; + let minifiedSrc, minified; if (minify) { const tersed = await Terser.minify(librarySrc, config.terser); @@ -237,7 +235,7 @@ async function buildBrowserHighlightJS(languages, { minify }) { minifiedSrc = `${header}\n${tersed.code}`; // get approximate core minified size - core_min = minifiedSrc.length; + minified = minifiedSrc.length; tasks.push(fs.writeFile(minifiedFile, minifiedSrc, { encoding: "utf8" })); shas["highlight.min.js"] = bundling.sha384(minifiedSrc); @@ -245,13 +243,12 @@ async function buildBrowserHighlightJS(languages, { minify }) { await Promise.all(tasks); return { - core: coreSize, - core_min: core_min, - minified: minifiedSrc.length, - minifiedSrc, fullSrc, - shas, full: fullSrc.length, + core: coreSrc.length, + minifiedSrc, + minified, + shas, }; } From 3ebdfb6a489b4764ff58227011373f87473cc1b2 Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Wed, 2 Jun 2021 18:40:30 +0200 Subject: [PATCH 06/25] Test build CDN without ESM support --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 1c265e0808..02755c7872 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: node-version: [12.x, 14.x, 16.x] - build-how: ["node", "browser", "browser -n", "cdn :common"] + build-how: ["node", "browser", "browser -n", "browser -ne", "browser -n -ne", "cdn :common"] steps: - uses: actions/checkout@v2 From 8b7f34e83deeaa38ddde1c97072c4fda8530b3e5 Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Wed, 2 Jun 2021 18:55:09 +0200 Subject: [PATCH 07/25] Fix `builtInLanguages` rollup plugin --- tools/build_browser.js | 72 ++++++++++++++++-------------------------- 1 file changed, 27 insertions(+), 45 deletions(-) diff --git a/tools/build_browser.js b/tools/build_browser.js index 26bad2b124..56c1c2dd3b 100644 --- a/tools/build_browser.js +++ b/tools/build_browser.js @@ -180,6 +180,31 @@ function installDemoStyles() { }); } +const safeImportName = (s) => { + s = s.replace(/-/g, "_"); + if (/^\d/.test(s)) s = `L_${s}`; + return s; +}; +const builtInLanguagesPlugin = (languages) => ({ + name: "hljs-index", + resolveId(source) { + if (source === "builtInLanguages") { + return source; // this signals that rollup should not ask other plugins or check the file system to find this id + } + return null; // other ids should be handled as usually + }, + load(id) { + if (id === "builtInLanguages") { + return languages.map((lang) => { + const importName = safeImportName(lang.name); + return `import ${importName} from ${JSON.stringify(lang.path)};\n` + + `hljs.registerLanguage(${JSON.stringify(lang.name)}, ${importName});`; + }).join("\n"); + } + return null; + }, +}) + async function buildBrowserHighlightJS(languages, { minify }) { log("Building highlight.js."); @@ -187,26 +212,7 @@ async function buildBrowserHighlightJS(languages, { minify }) { const outFile = `${process.env.BUILD_DIR}/highlight.js`; const minifiedFile = outFile.replace(/js$/, "min.js"); - - const built_in_langs = { - name: "dynamicLanguages", - resolveId: (source) => { - if (source == "builtInLanguages") { return "builtInLanguages"} - return null; - }, - load: (id) => { - if (id == "builtInLanguages") { - const escape = (s) => "grmr_" + s.replace("-", "_"); - let src = ""; - src += languages.map((x) => `import ${escape(x.name)} from '${x.path}'`).join("\n"); - src += `\nexport {${languages.map((x) => escape(x.name)).join(",")}}`; - return src; - } - return null; - } - } - - const plugins = [...config.rollup.browser_core.input.plugins, built_in_langs]; + const plugins = [...config.rollup.browser_core.input.plugins, builtInLanguagesPlugin(languages)]; const input = { ...config.rollup.browser_core.input, input: `src/stub.js`, plugins }; const output = { ...config.rollup.browser_core.output, file: outFile }; @@ -252,36 +258,12 @@ async function buildBrowserHighlightJS(languages, { minify }) { }; } -const safeImportName = (s) => { - s = s.replace(/-/g, "_"); - if (/^\d/.test(s)) s = `L_${s}`; - return s; -}; - async function buildBrowserESMHighlightJS(name, languages, options) { log("Building highlight.mjs."); const header = buildHeader(); const input = { ...config.rollup.node.input, input: `src/stub.js`, plugins: [ ...config.rollup.node.input.plugins, - { - name: "hljs-index", - resolveId(source) { - if (source === "builtInLanguages") { - return source; // this signals that rollup should not ask other plugins or check the file system to find this id - } - return null; // other ids should be handled as usually - }, - load(id) { - if (id === "builtInLanguages") { - return languages.map((lang) => { - const importName = safeImportName(lang.name); - return `import ${importName} from './languages/${lang.name}.js';\n` + - `hljs.registerLanguage('${lang.name}', ${importName});`; - }).join("\n"); - } - return null; - }, - }, + builtInLanguagesPlugin(languages), ] }; const output = { ...config.rollup.node.output, From 8c246f5743642e925468ad23eeaa4e114f605ce7 Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Wed, 2 Jun 2021 19:00:53 +0200 Subject: [PATCH 08/25] fixup! Test build CDN without ESM support --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 02755c7872..f9d87bb71f 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: node-version: [12.x, 14.x, 16.x] - build-how: ["node", "browser", "browser -n", "browser -ne", "browser -n -ne", "cdn :common"] + build-how: ["node", "browser", "browser -n", "cdn -ne", "cdn -n -ne", "cdn :common"] steps: - uses: actions/checkout@v2 From 7da28ccd50bb8daa7bf3a1accf5ded574f2cd6a6 Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Wed, 2 Jun 2021 19:01:41 +0200 Subject: [PATCH 09/25] Revert removal of core minification size computing --- tools/build_browser.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tools/build_browser.js b/tools/build_browser.js index 56c1c2dd3b..6303c2d3fa 100644 --- a/tools/build_browser.js +++ b/tools/build_browser.js @@ -233,11 +233,14 @@ async function buildBrowserHighlightJS(languages, { minify }) { "highlight.js": bundling.sha384(fullSrc) }; - let minifiedSrc, minified; + let minifiedSrc, minified, core_min; if (minify) { const tersed = await Terser.minify(librarySrc, config.terser); + const coreTersed = await Terser.minify(coreSrc, config.terser); + core_min = header.length + 1 + coreTersed.code.length; + minifiedSrc = `${header}\n${tersed.code}`; // get approximate core minified size @@ -249,9 +252,10 @@ async function buildBrowserHighlightJS(languages, { minify }) { await Promise.all(tasks); return { + core: coreSrc.length, + core_min, fullSrc, full: fullSrc.length, - core: coreSrc.length, minifiedSrc, minified, shas, From 9afa0f06fb731c5f8c991dbd6fe7ee1d1ded7d4a Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Wed, 2 Jun 2021 19:11:52 +0200 Subject: [PATCH 10/25] fixup! Fix `builtInLanguages` rollup plugin --- tools/build_browser.js | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/tools/build_browser.js b/tools/build_browser.js index 6303c2d3fa..68c746d752 100644 --- a/tools/build_browser.js +++ b/tools/build_browser.js @@ -180,11 +180,6 @@ function installDemoStyles() { }); } -const safeImportName = (s) => { - s = s.replace(/-/g, "_"); - if (/^\d/.test(s)) s = `L_${s}`; - return s; -}; const builtInLanguagesPlugin = (languages) => ({ name: "hljs-index", resolveId(source) { @@ -194,12 +189,11 @@ const builtInLanguagesPlugin = (languages) => ({ return null; // other ids should be handled as usually }, load(id) { + const escape = (s) => "grmr_" + s.replace("-", "_"); if (id === "builtInLanguages") { - return languages.map((lang) => { - const importName = safeImportName(lang.name); - return `import ${importName} from ${JSON.stringify(lang.path)};\n` + - `hljs.registerLanguage(${JSON.stringify(lang.name)}, ${importName});`; - }).join("\n"); + return languages.map((lang) => + `export { default as ${escape(lang.name)} } from ${JSON.stringify(lang.path)};` + ).join("\n"); } return null; }, From 664cf3709cc2a6160d0e18308341752f74dc859f Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Wed, 2 Jun 2021 19:19:02 +0200 Subject: [PATCH 11/25] fixup! Test build CDN without ESM support --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index f9d87bb71f..10022dd5ac 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: node-version: [12.x, 14.x, 16.x] - build-how: ["node", "browser", "browser -n", "cdn -ne", "cdn -n -ne", "cdn :common"] + build-how: ["node", "browser", "browser -n", "cdn -ne :common", "cdn -n -ne :common", "cdn :common"] steps: - uses: actions/checkout@v2 From 3f5834f138aa4dd5b5df904a43b485b0b803f8e0 Mon Sep 17 00:00:00 2001 From: Josh Goebel Date: Wed, 2 Jun 2021 18:11:34 -0400 Subject: [PATCH 12/25] simplify building & fix issues --- tools/build_browser.js | 106 +++++++++++++---------------------------- tools/build_cdn.js | 104 ++++++++++++++++------------------------ tools/build_config.js | 2 +- tools/build_node.js | 6 +-- tools/lib/bundling.js | 2 +- 5 files changed, 79 insertions(+), 141 deletions(-) diff --git a/tools/build_browser.js b/tools/build_browser.js index 68c746d752..56bb36851b 100644 --- a/tools/build_browser.js +++ b/tools/build_browser.js @@ -73,14 +73,12 @@ async function buildBrowser(options) { detailedGrammarSizes(languages); - const size = await buildBrowserHighlightJS(languages, { minify: options.minify }); + const size = await buildCore("highlight", languages, { minify: options.minify, format: "cjs" }); log("-----"); - log("Core :", size.core, "bytes"); - if (options.minify) { log("Core (min) :", size.core_min, "bytes"); } log("Languages (raw) :", languages.map((el) => el.data.length).reduce((acc, curr) => acc + curr, 0), "bytes"); - log("highlight.js :", size.full, "bytes"); + log("highlight.js :", size.fullSize, "bytes"); if (options.minify) { log("highlight.min.js :", size.minified, "bytes"); log("highlight.min.js.gz :", zlib.gzipSync(size.minifiedSrc).length, "bytes"); @@ -191,105 +189,65 @@ const builtInLanguagesPlugin = (languages) => ({ load(id) { const escape = (s) => "grmr_" + s.replace("-", "_"); if (id === "builtInLanguages") { - return languages.map((lang) => + return languages.map((lang) => `export { default as ${escape(lang.name)} } from ${JSON.stringify(lang.path)};` ).join("\n"); } return null; - }, -}) - -async function buildBrowserHighlightJS(languages, { minify }) { - log("Building highlight.js."); + } +}); +async function buildCore(name, languages, options) { const header = buildHeader(); - - const outFile = `${process.env.BUILD_DIR}/highlight.js`; - const minifiedFile = outFile.replace(/js$/, "min.js"); - const plugins = [...config.rollup.browser_core.input.plugins, builtInLanguagesPlugin(languages)]; - - const input = { ...config.rollup.browser_core.input, input: `src/stub.js`, plugins }; - const output = { ...config.rollup.browser_core.output, file: outFile }; - let librarySrc = await rollupCode(input, output); - - - // we don't use this, we just use it to get a size approximation for the build stats - const coreSrc = await rollupCode({ ...config.rollup.browser_core.input, input: `src/highlight.js`, plugins }, output); - - // strip off the original top comment - librarySrc = librarySrc.replace(/\/\*.*?\*\//s, ""); - - const fullSrc = `${header}\n${librarySrc}`; - - const tasks = []; - tasks.push(fs.writeFile(outFile, fullSrc, { encoding: "utf8" })); - const shas = { - "highlight.js": bundling.sha384(fullSrc) + let relativePath = ""; + const input = { + ...(options.format === "es" ? config.rollup.node.input : config.rollup.browser_iife.input), + input: `src/stub.js` + }; + input.plugins = [ + ...input.plugins, + builtInLanguagesPlugin(languages) + ]; + const output = { + ...config.rollup.node.output, + file: `${process.env.BUILD_DIR}/${name}.js` }; - let minifiedSrc, minified, core_min; - - if (minify) { - const tersed = await Terser.minify(librarySrc, config.terser); - - const coreTersed = await Terser.minify(coreSrc, config.terser); - core_min = header.length + 1 + coreTersed.code.length; - - minifiedSrc = `${header}\n${tersed.code}`; - - // get approximate core minified size - minified = minifiedSrc.length; - - tasks.push(fs.writeFile(minifiedFile, minifiedSrc, { encoding: "utf8" })); - shas["highlight.min.js"] = bundling.sha384(minifiedSrc); + // optimize for no languages by not including the language loading stub + if (languages.length === 0) { + input.input = "src/highlight.js"; } - await Promise.all(tasks); - return { - core: coreSrc.length, - core_min, - fullSrc, - full: fullSrc.length, - minifiedSrc, - minified, - shas, - }; -} + if (options.format === "es") { + output.format = "es"; + output.file = `${process.env.BUILD_DIR}/es/${name}.js`; + relativePath = "es/"; + } -async function buildBrowserESMHighlightJS(name, languages, options) { - log("Building highlight.mjs."); - const header = buildHeader(); - const input = { ...config.rollup.node.input, input: `src/stub.js`, plugins: [ - ...config.rollup.node.input.plugins, - builtInLanguagesPlugin(languages), - ] }; - const output = { - ...config.rollup.node.output, - format: "es", - file: `${process.env.BUILD_DIR}/es/${name}.js`, - }; + log(`Building ${relativePath}${name}.js.`); const index = await rollupCode(input, output); - const sizeInfo = {} - const writePromises = [] + const sizeInfo = { shas: [] }; + const writePromises = []; if (options.minify) { const { code } = await Terser.minify(index, {...config.terser, module: true}) const src = `${header}\n${code}`; writePromises.push(fs.writeFile(output.file.replace(/js$/, "min.js"), src)); sizeInfo.minified = src.length; sizeInfo.minifiedSrc = src; + sizeInfo.shas[`${relativePath}${name}.min.js`] = bundling.sha384(src) } { const src = `${header}\n${index}`; writePromises.push(fs.writeFile(output.file, src)); sizeInfo.fullSize = src.length; sizeInfo.fullSrc = src; + sizeInfo.shas[`${relativePath}${name}.js`] = bundling.sha384(src) } await Promise.all(writePromises); return sizeInfo; } // CDN build uses the exact same highlight.js distributable -module.exports.buildBrowserHighlightJS = buildBrowserHighlightJS; -module.exports.buildBrowserESMHighlightJS = buildBrowserESMHighlightJS; +module.exports.buildCore = buildCore; module.exports.build = buildBrowser; diff --git a/tools/build_cdn.js b/tools/build_cdn.js index 6e30599232..ee6bd3dba2 100644 --- a/tools/build_cdn.js +++ b/tools/build_cdn.js @@ -1,37 +1,24 @@ const fs = require("fs").promises; const fss = require("fs"); const glob = require("glob"); -const Terser = require("terser"); const zlib = require('zlib'); -const { getLanguages } = require("./lib/language"); -const { filter } = require("./lib/dependencies"); -const config = require("./build_config"); -const { install, installCleanCSS, mkdir } = require("./lib/makestuff"); +const { getLanguages } = require("./lib/language.js"); +const { filter } = require("./lib/dependencies.js"); +const config = require("./build_config.js"); +const { install, installCleanCSS, mkdir } = require("./lib/makestuff.js"); const log = (...args) => console.log(...args); -const { buildBrowserHighlightJS, buildBrowserESMHighlightJS } = require("./build_browser"); -const { buildPackageJSON, writePackageJSON } = require("./build_node"); -const { rollupCode } = require("./lib/bundling.js"); +const { buildCore } = require("./build_browser.js"); +const { buildPackageJSON, writePackageJSON } = require("./build_node.js"); const path = require("path"); const bundling = require('./lib/bundling.js'); async function installPackageJSON(options) { - const json = buildPackageJSON(options, { - ".": { - import: options.minify ? "./es/index.min.js" : "./es/index.js", - browser: options.minify ? "./highlight.min.js" : "./highlight.js", - }, - "./lib/languages/*": { - import: "./es/languages/*.js", - browser: "./languages/*.js" - }, - get "./lib/common"(){ return this["."]; }, - "./lib/core": { import: "./es/core.js" }, - "./styles/*": "./styles/*", - "./package.json": "./package.json", - }); + const json = buildPackageJSON(options); json.name = "@highlightjs/cdn-assets"; json.description = json.description.concat(" (pre-compiled CDN assets)"); // this is not a replacement for `highlightjs` package + // CDN assets do not need an export map, they are just a bunch of files. + // The NPM package mostly only exists to populate CDNs and provide raw files. delete json.exports; delete json.type; delete json.main; @@ -39,20 +26,6 @@ async function installPackageJSON(options) { await writePackageJSON(json); } -async function buildESMCore(options) { - const input = { ...config.rollup.node.input, input: `src/highlight.js` }; - const output = { - ...config.rollup.node.output, - format: "es", - file: `${process.env.BUILD_DIR}/es/core.js`, - }; - const core = await rollupCode(input, output); - - const miniCore = options.minify ? await Terser.minify(core, {...config.terser, module: true}) : { code: core }; - await fs.writeFile(output.file, miniCore.code); - return miniCore.code.length; -} - let shas = {}; async function buildCDN(options) { @@ -64,13 +37,9 @@ async function buildCDN(options) { // all the languages are built for the CDN and placed into `/languages` const languages = await getLanguages(); - - let esmCoreSize, esmIndexSize; - if (options.esm) { - mkdir("es"); - await fs.writeFile(`${process.env.BUILD_DIR}/es/package.json`, `{ "type": "module" }`); - esmCoreSize = await buildESMCore(options); - } + + let esmCoreSize = {}; + let esmCommonSize = {}; await installLanguages(languages, options); @@ -84,34 +53,43 @@ async function buildCDN(options) { embedLanguages = []; } - const size = await buildBrowserHighlightJS(embedLanguages, { minify: options.minify }); - if (options.esm) esmIndexSize = await buildBrowserESMHighlightJS("index", embedLanguages, { minify: options.minify }); - shas = Object.assign({}, size.shas, shas); + const size = await buildCore("highlight", embedLanguages, { minify: options.minify, format: "cjs" }); + if (options.esm) { + mkdir("es"); + await fs.writeFile(`${process.env.BUILD_DIR}/es/package.json`, `{ "type": "module" }`); + esmCoreSize = await buildCore("core", [], {minify: options.minify, format: "es"}); + esmCommonSize = await buildCore("highlight", embedLanguages, { minify: options.minify, format: "es" }); + } + shas = { + ...size.shas, ...esmCommonSize.shas, ...esmCoreSize.shas, ...shas + }; await buildSRIDigests(shas); log("-----"); - log("Embedded Lang :", + log("Embedded Lang :", embedLanguages.map((el) => el.minified.length).reduce((acc, curr) => acc + curr, 0), "bytes"); - log("All Lang :", + log("All Lang :", languages.map((el) => el.minified.length).reduce((acc, curr) => acc + curr, 0), "bytes"); - log("highlight.js :", - size.full, "bytes"); + log("highlight.js :", + size.fullSize, "bytes"); if (options.minify) { - log("highlight.min.js :", size.minified, "bytes"); - log("highlight.min.js.gz :", zlib.gzipSync(size.minifiedSrc).length, "bytes"); + log("highlight.min.js :", size.minified, "bytes"); + log("highlight.min.js.gz :", zlib.gzipSync(size.minifiedSrc).length, "bytes"); } else { - log("highlight.js.gz :", zlib.gzipSync(size.fullSrc).length, "bytes"); + log("highlight.js.gz :", zlib.gzipSync(size.fullSrc).length, "bytes"); } - if(options.esm) { - log("es/core.js :", esmCoreSize, "bytes"); - log("es/index.js :", esmIndexSize.fullSize, "bytes"); + if (options.esm) { + log("es/core.js :", esmCoreSize.fullSize, "bytes"); + log("es/highlight.js :", esmCommonSize.fullSize, "bytes"); if (options.minify) { - log("es/index.min.js :", esmIndexSize.minified, "bytes"); - log("es/index.min.js.gz :", zlib.gzipSync(esmIndexSize.minifiedSrc).length, "bytes"); + log("es/core.min.js :", esmCoreSize.minified, "bytes"); + log("es/core.min.js.gz :", zlib.gzipSync(esmCoreSize.minifiedSrc).length, "bytes"); + log("es/highlight.min.js :", esmCommonSize.minified, "bytes"); + log("es/highlight.min.js.gz :", zlib.gzipSync(esmCommonSize.minifiedSrc).length, "bytes"); } else { - log("es/index.js.gz :", zlib.gzipSync(esmIndexSize.fullSrc).length, "bytes"); + log("es/highlight.js.gz :", zlib.gzipSync(esmCommonSize.fullSrc).length, "bytes"); } } log("-----"); @@ -122,7 +100,7 @@ async function buildSRIDigests(shas) { const temp = await fs.readFile("./tools/templates/DIGESTS.md"); const DIGEST_MD = temp.toString(); - const version = require("../package").version; + const version = require("../package.json").version; const digestList = Object.entries(shas).map(([k, v]) => `${v} ${k}`).join("\n"); const out = DIGEST_MD @@ -136,7 +114,7 @@ async function buildSRIDigests(shas) { async function installLanguages(languages, options) { log("Building language files."); mkdir("languages"); - if(options.esm) mkdir("es/languages"); + if (options.esm) mkdir("es/languages"); await Promise.all( languages.map(async(language) => { @@ -186,8 +164,10 @@ async function buildCDNLanguage(language, options) { await language.compile({ terser: config.terser }); shas[name] = bundling.sha384(language.minified); await fs.writeFile(`${process.env.BUILD_DIR}/${name}`, language.minified); - if (options.esm) + if (options.esm) { + shas[`es/${name}`] = bundling.sha384(language.minifiedESM); await fs.writeFile(`${process.env.BUILD_DIR}/es/${name}`, language.minifiedESM); + } } module.exports.build = buildCDN; diff --git a/tools/build_config.js b/tools/build_config.js index 3cedd90663..0e0262e7b7 100644 --- a/tools/build_config.js +++ b/tools/build_config.js @@ -27,7 +27,7 @@ module.exports = { ] } }, - browser_core: { + browser_iife: { input: { plugins: [ jsonPlugin(), diff --git a/tools/build_node.js b/tools/build_node.js index c4a928d367..a95b53a5b4 100644 --- a/tools/build_node.js +++ b/tools/build_node.js @@ -105,10 +105,10 @@ const generatePackageExports = () => ({ "./styles/*": "./styles/*", "./types/*": "./types/*", }); -function buildPackageJSON(options, exports = generatePackageExports()) { - const packageJson = require("../package"); +function buildPackageJSON(options) { + const packageJson = require("../package.json"); - if (options.esm) packageJson.exports = exports; + if (options.esm) packageJson.exports = generatePackageExports(); return packageJson; } diff --git a/tools/lib/bundling.js b/tools/lib/bundling.js index c3c63df80f..1ca11a71c6 100644 --- a/tools/lib/bundling.js +++ b/tools/lib/bundling.js @@ -22,7 +22,7 @@ function sha384(contents) { const hash = crypto.createHash('sha384'); const data = hash.update(contents, 'utf-8'); const gen_hash = data.digest('base64'); - return `sha384-${gen_hash}` + return `sha384-${gen_hash}`; } module.exports = { rollupWrite, rollupCode, sha384 }; From 671d89c7d38cc7f3b643fa783ab82e86174cbeec Mon Sep 17 00:00:00 2001 From: Josh Goebel Date: Wed, 2 Jun 2021 18:05:48 -0400 Subject: [PATCH 13/25] no benefit to testing these --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 10022dd5ac..1c265e0808 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -17,7 +17,7 @@ jobs: strategy: matrix: node-version: [12.x, 14.x, 16.x] - build-how: ["node", "browser", "browser -n", "cdn -ne :common", "cdn -n -ne :common", "cdn :common"] + build-how: ["node", "browser", "browser -n", "cdn :common"] steps: - uses: actions/checkout@v2 From c8f26f5699c2ee4183a49f7c78991bd7e9524268 Mon Sep 17 00:00:00 2001 From: Josh Goebel Date: Wed, 2 Jun 2021 18:22:11 -0400 Subject: [PATCH 14/25] wip --- tools/build_browser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build_browser.js b/tools/build_browser.js index 56bb36851b..2a653b6b06 100644 --- a/tools/build_browser.js +++ b/tools/build_browser.js @@ -209,7 +209,7 @@ async function buildCore(name, languages, options) { builtInLanguagesPlugin(languages) ]; const output = { - ...config.rollup.node.output, + ...(options.format === "es" ? config.rollup.node.output : config.rollup.browser_iife.output), file: `${process.env.BUILD_DIR}/${name}.js` }; From 404ee5aa7bccefd2137ca708954259222229cec8 Mon Sep 17 00:00:00 2001 From: Josh Goebel Date: Wed, 2 Jun 2021 18:28:36 -0400 Subject: [PATCH 15/25] module option depends on format --- tools/build_browser.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build_browser.js b/tools/build_browser.js index 2a653b6b06..cfa4e179a6 100644 --- a/tools/build_browser.js +++ b/tools/build_browser.js @@ -230,7 +230,7 @@ async function buildCore(name, languages, options) { const sizeInfo = { shas: [] }; const writePromises = []; if (options.minify) { - const { code } = await Terser.minify(index, {...config.terser, module: true}) + const { code } = await Terser.minify(index, {...config.terser, module: (options.format === "es") }); const src = `${header}\n${code}`; writePromises.push(fs.writeFile(output.file.replace(/js$/, "min.js"), src)); sizeInfo.minified = src.length; From 7e8a07d6711bd930780cf76d7ea2beba407c940d Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Thu, 3 Jun 2021 15:21:18 +0200 Subject: [PATCH 16/25] Add Deno tests and CI --- .github/workflows/deno.yml | 33 +++++++++++++++++++++++++++++++++ test/deno/index.ts | 28 ++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) create mode 100644 .github/workflows/deno.yml create mode 100644 test/deno/index.ts diff --git a/.github/workflows/deno.yml b/.github/workflows/deno.yml new file mode 100644 index 0000000000..e579d31206 --- /dev/null +++ b/.github/workflows/deno.yml @@ -0,0 +1,33 @@ +name: Deno CI + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + test-deno: + needs: build + runs-on: ubuntu-latest + + strategy: + matrix: + deno: [v1.10.3] + node-version: 16.x + build-how: ["cdn"] + + steps: + - uses: actions/checkout@v2 + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node-version }} + - name: Install Deno ${{ matrix.deno }} + run: + curl -fsSL https://deno.land/x/install/install.sh | sudo + DENO_INSTALL=/usr/local sh -s ${{ matrix.deno }} + - run: npm install + - run: node ./tools/build.js -t ${{ matrix.build-how }} + - name: Run tests + run: deno test test/deno/index.ts diff --git a/test/deno/index.ts b/test/deno/index.ts new file mode 100644 index 0000000000..5fdf0c2b78 --- /dev/null +++ b/test/deno/index.ts @@ -0,0 +1,28 @@ +import { assertStrictEquals } from "https://deno.land/std@0.97.0/testing/asserts.ts"; + +import hljs from "../../build/es/core.js"; +import js from "../../build/es/languages/javascript.min.js"; + +hljs.registerLanguage("javascript", js); + +Deno.test({ + name: "should return relevance key", + async fn() { + const out = hljs.highlight("", { language: "javascript" }); + assertStrictEquals(out.relevance, 0); + }, +}); + +Deno.test({ + name: "should highlight block", + async fn() { + const code = 'var say = "Hello";'; + const expected = + '' + + 'var say = ' + + '"Hello";'; + + const out = hljs.highlight(code, { language: "javascript" }); + assertStrictEquals(out.value, expected); + }, +}); From 69894276982167745fc522658df60726c078135f Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Thu, 3 Jun 2021 15:25:30 +0200 Subject: [PATCH 17/25] fixup! Add Deno tests and CI --- .github/workflows/deno.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deno.yml b/.github/workflows/deno.yml index e579d31206..65b7677fac 100644 --- a/.github/workflows/deno.yml +++ b/.github/workflows/deno.yml @@ -14,7 +14,7 @@ jobs: strategy: matrix: deno: [v1.10.3] - node-version: 16.x + node-version: [16.x] build-how: ["cdn"] steps: From 8a4f58f3389b6573c8dd19bf28b325e8bc4defe6 Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Thu, 3 Jun 2021 15:26:36 +0200 Subject: [PATCH 18/25] fixup! Add Deno tests and CI --- .github/workflows/deno.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/deno.yml b/.github/workflows/deno.yml index 65b7677fac..61e0be2dc4 100644 --- a/.github/workflows/deno.yml +++ b/.github/workflows/deno.yml @@ -8,7 +8,6 @@ on: jobs: test-deno: - needs: build runs-on: ubuntu-latest strategy: From 667d6e6b61f811cf0c43395b8ec3308f34fc62a1 Mon Sep 17 00:00:00 2001 From: Antoine du Hamel Date: Sun, 13 Jun 2021 16:54:31 +0200 Subject: [PATCH 19/25] Revert "fixup! Add Deno tests and CI" This reverts commit 92b28958affbe8c716acf9a754b722de7cc0bc12, 10ea6c5a2ae312f6614b837d70c81cc932ff5ff2, and eba2f2e1dcaf42635bc9bccc24551627c0b22d27. --- .github/workflows/deno.yml | 32 -------------------------------- test/deno/index.ts | 28 ---------------------------- 2 files changed, 60 deletions(-) delete mode 100644 .github/workflows/deno.yml delete mode 100644 test/deno/index.ts diff --git a/.github/workflows/deno.yml b/.github/workflows/deno.yml deleted file mode 100644 index 61e0be2dc4..0000000000 --- a/.github/workflows/deno.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Deno CI - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - test-deno: - runs-on: ubuntu-latest - - strategy: - matrix: - deno: [v1.10.3] - node-version: [16.x] - build-how: ["cdn"] - - steps: - - uses: actions/checkout@v2 - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - name: Install Deno ${{ matrix.deno }} - run: - curl -fsSL https://deno.land/x/install/install.sh | sudo - DENO_INSTALL=/usr/local sh -s ${{ matrix.deno }} - - run: npm install - - run: node ./tools/build.js -t ${{ matrix.build-how }} - - name: Run tests - run: deno test test/deno/index.ts diff --git a/test/deno/index.ts b/test/deno/index.ts deleted file mode 100644 index 5fdf0c2b78..0000000000 --- a/test/deno/index.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { assertStrictEquals } from "https://deno.land/std@0.97.0/testing/asserts.ts"; - -import hljs from "../../build/es/core.js"; -import js from "../../build/es/languages/javascript.min.js"; - -hljs.registerLanguage("javascript", js); - -Deno.test({ - name: "should return relevance key", - async fn() { - const out = hljs.highlight("", { language: "javascript" }); - assertStrictEquals(out.relevance, 0); - }, -}); - -Deno.test({ - name: "should highlight block", - async fn() { - const code = 'var say = "Hello";'; - const expected = - '' + - 'var say = ' + - '"Hello";'; - - const out = hljs.highlight(code, { language: "javascript" }); - assertStrictEquals(out.value, expected); - }, -}); From e1ebed27731369df72e798553b94746971f98641 Mon Sep 17 00:00:00 2001 From: Josh Goebel Date: Tue, 29 Jun 2021 17:44:43 -0400 Subject: [PATCH 20/25] revert to much simpler IIFE build process - also build an ESM module in `extra/grammar/dist` --- tools/build_cdn.js | 7 +++++-- tools/lib/language.js | 16 +++++++++------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/tools/build_cdn.js b/tools/build_cdn.js index ee6bd3dba2..00702a6a49 100644 --- a/tools/build_cdn.js +++ b/tools/build_cdn.js @@ -126,7 +126,7 @@ async function installLanguages(languages, options) { await Promise.all( languages.filter((l) => l.third_party) - .map(buildDistributable) + .map((lang) => buildDistributable(lang, options)) ); log(""); @@ -149,13 +149,16 @@ function installStyles() { }); } -async function buildDistributable(language) { +async function buildDistributable(language, options) { const filename = `${language.name}.min.js`; const distDir = path.join(language.moduleDir, "dist"); log(`Building ${distDir}/${filename}.`); await fs.mkdir(distDir, { recursive: true }); await fs.writeFile(path.join(language.moduleDir, "dist", filename), language.minified); + if (options.esm) { + await fs.writeFile(path.join(language.moduleDir, "dist", filename.replace(".min.js",".es.min.js")), language.minifiedESM); + } } async function buildCDNLanguage(language, options) { diff --git a/tools/lib/language.js b/tools/lib/language.js index d5124a80f4..0e25f973c0 100644 --- a/tools/lib/language.js +++ b/tools/lib/language.js @@ -79,20 +79,22 @@ class Language { async function compileLanguage (language, options) { - const EXPORT_REGEX = /export default (.*);/; + const IIFE_HEADER_REGEX = /^(var hljsGrammar = )?\(function \(\)/; // TODO: cant we use the source we already have? - const input = { ...build_config.rollup.browser.input, input: language.path }; - const output = { ...build_config.rollup.browser.output, name: `dummyName`, file: "out.js" }; + const input = { ...build_config.rollup.browser_iife.input, input: language.path }; + const output = { ...build_config.rollup.browser_iife.output, name: `hljsGrammar`, file: "out.js" }; + output.footer = null; - const esm = await rollupCode(input, output); - const iife = `hljs.registerLanguage('${language.name}', function (){"use strict";` + esm.replace(EXPORT_REGEX, 'return $1') + '})'; + const data = await rollupCode(input, output); + const iife = data.replace(IIFE_HEADER_REGEX, `hljs.registerLanguage('${language.name}', function ()`); + const esm = `${data};\nexport default hljsGrammar;`; language.module = iife; const miniESM = await Terser.minify(esm, options.terser); const miniIIFE = await Terser.minify(iife, options.terser); - language.minified = miniIIFE.code || iife; - language.minifiedESM = miniESM.code || esm; + language.minified = miniIIFE.code; + language.minifiedESM = miniESM.code; } async function getLanguages() { From 101e0b4bd04303e90a5916bc034a2431e107a0a8 Mon Sep 17 00:00:00 2001 From: Josh Goebel Date: Tue, 29 Jun 2021 18:48:07 -0400 Subject: [PATCH 21/25] add async/await back --- tools/build_cdn.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/build_cdn.js b/tools/build_cdn.js index 00702a6a49..9cc3463ea2 100644 --- a/tools/build_cdn.js +++ b/tools/build_cdn.js @@ -126,7 +126,7 @@ async function installLanguages(languages, options) { await Promise.all( languages.filter((l) => l.third_party) - .map((lang) => buildDistributable(lang, options)) + .map(async(lang) => await buildDistributable(lang, options)) ); log(""); From be3f395613484dc45a80199bfee4d43330bf61e8 Mon Sep 17 00:00:00 2001 From: Josh Goebel Date: Tue, 29 Jun 2021 19:14:43 -0400 Subject: [PATCH 22/25] simplify a bit further --- tools/build_browser.js | 2 +- tools/build_config.js | 16 +++++----------- tools/build_node.js | 6 +++--- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/tools/build_browser.js b/tools/build_browser.js index cfa4e179a6..48c0f465b6 100644 --- a/tools/build_browser.js +++ b/tools/build_browser.js @@ -201,7 +201,7 @@ async function buildCore(name, languages, options) { const header = buildHeader(); let relativePath = ""; const input = { - ...(options.format === "es" ? config.rollup.node.input : config.rollup.browser_iife.input), + ...config.rollup.core.input, input: `src/stub.js` }; input.plugins = [ diff --git a/tools/build_config.js b/tools/build_config.js index 0e0262e7b7..6d6557f4ab 100644 --- a/tools/build_config.js +++ b/tools/build_config.js @@ -9,13 +9,13 @@ module.exports = { level: 2 }, rollup: { - node: { - output: { format: "cjs", strict: false, exports: "auto" }, + core: { input: { plugins: [ cjsPlugin(), jsonPlugin(), nodeResolve(), + // TODO: remove with version 12 { transform: (x) => { if (/var module/.exec(x)) { @@ -27,6 +27,9 @@ module.exports = { ] } }, + node: { + output: { format: "cjs", strict: false, exports: "auto" } + }, browser_iife: { input: { plugins: [ @@ -41,15 +44,6 @@ module.exports = { footer: "if (typeof exports === 'object' && typeof module !== 'undefined') { module.exports = hljs; }", interop: false } - }, - browser: { - input: { - plugins: [] - }, - output: { - format: "es", - interop: false - } } }, terser: { diff --git a/tools/build_node.js b/tools/build_node.js index a95b53a5b4..235061a4fe 100644 --- a/tools/build_node.js +++ b/tools/build_node.js @@ -45,7 +45,7 @@ async function buildNodeLanguage(language, options) { const ES_STUB = `${EMIT} import lang from './%%%%.js'; export default lang;`; - const input = { ...config.rollup.node.input, input: language.path }; + const input = { ...config.rollup.core.input, input: language.path }; const output = { ...config.rollup.node.output, file: `${process.env.BUILD_DIR}/lib/languages/${language.name}.js` }; await rollupWrite(input, output); await fs.writeFile(`${process.env.BUILD_DIR}/lib/languages/${language.name}.js.js`, @@ -63,7 +63,7 @@ async function buildNodeLanguage(language, options) { const EXCLUDE = ["join"]; async function buildESMUtils() { - const input = { ...config.rollup.node.input, input: `src/lib/regex.js` }; + const input = { ...config.rollup.core.input, input: `src/lib/regex.js` }; input.plugins = [...input.plugins, { transform: (code) => { EXCLUDE.forEach((fn) => { @@ -80,7 +80,7 @@ async function buildESMUtils() { } async function buildNodeHighlightJS(options) { - const input = { ...config.rollup.node.input, input: `src/highlight.js` }; + const input = { ...config.rollup.core.input, input: `src/highlight.js` }; const output = { ...config.rollup.node.output, file: `${process.env.BUILD_DIR}/lib/core.js` }; await rollupWrite(input, output); if (options.esm) { From 8027a2d1dce94c620c16a2ab3c8296b02dcb2dc2 Mon Sep 17 00:00:00 2001 From: Josh Goebel Date: Fri, 2 Jul 2021 20:38:58 -0400 Subject: [PATCH 23/25] test that the CDN ESM build is importable (to node) --- .github/workflows/node.js.yml | 5 +++++ test/builds/cdn_build_as_esm.mjs | 24 ++++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100644 test/builds/cdn_build_as_esm.mjs diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index 1c265e0808..d3c86f96e5 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -33,6 +33,11 @@ jobs: run: | npm run test-browser node test/builds/browser_build_as_commonjs.js + # CDN build should be easily importable + - if: contains(matrix.build-how, 'cdn') + name: Test: Can import CDN esm build + run: | + node test/builds/cdn_build_as_esm.mjs - if: contains(matrix.build-how, 'node') name: Test Node.js build diff --git a/test/builds/cdn_build_as_esm.mjs b/test/builds/cdn_build_as_esm.mjs new file mode 100644 index 0000000000..b217e7d5ce --- /dev/null +++ b/test/builds/cdn_build_as_esm.mjs @@ -0,0 +1,24 @@ +import hljs from "../../build/es/highlight.js"; + +const API = [ + "getLanguage", + "registerLanguage", + "highlight", + "highlightAuto", + "highlightAll", + "highlightElement" +]; + +const assert = (f,msg) => { + if (!f()) { + console.error(msg); + process.exit(1); + } +}; +const keys = Object.keys(hljs); + +API.forEach(n => { + assert(_ => keys.includes(n), `API should include ${n}`); +}); + +console.log("Pass: browser build works with Node.js just fine.") From c23ef0a9d27567c8a3945337a389a2757da209d8 Mon Sep 17 00:00:00 2001 From: Josh Goebel Date: Fri, 2 Jul 2021 21:01:24 -0400 Subject: [PATCH 24/25] run tests From a2a6b33b8432fb94631f17c24f6fce3b7965cc4c Mon Sep 17 00:00:00 2001 From: Josh Goebel Date: Fri, 2 Jul 2021 21:07:17 -0400 Subject: [PATCH 25/25] try to fix tests --- .github/workflows/node.js.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/node.js.yml b/.github/workflows/node.js.yml index d3c86f96e5..55c5aa64d6 100644 --- a/.github/workflows/node.js.yml +++ b/.github/workflows/node.js.yml @@ -35,7 +35,7 @@ jobs: node test/builds/browser_build_as_commonjs.js # CDN build should be easily importable - if: contains(matrix.build-how, 'cdn') - name: Test: Can import CDN esm build + name: Test that we can import CDN esm build run: | node test/builds/cdn_build_as_esm.mjs