diff --git a/lib/marked.cjs b/lib/marked.cjs index bcbd9c67fe..c5ffa1011e 100644 --- a/lib/marked.cjs +++ b/lib/marked.cjs @@ -412,7 +412,7 @@ function outputLink(cap, link, raw, lexer) { href: href, title: title, text: text, - tokens: lexer.inlineTokens(text, []) + tokens: lexer.inlineTokens(text) }; lexer.state.inLink = false; return token; @@ -520,15 +520,13 @@ var Tokenizer = /*#__PURE__*/function () { } } - var token = { + return { type: 'heading', raw: cap[0], depth: cap[1].length, text: text, - tokens: [] + tokens: this.lexer.inline(text) }; - this.lexer.inline(token.text, token.tokens); - return token; } }; @@ -746,10 +744,10 @@ var Tokenizer = /*#__PURE__*/function () { }; if (this.options.sanitize) { + var text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]); token.type = 'paragraph'; - token.text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]); - token.tokens = []; - this.lexer.inline(token.text, token.tokens); + token.text = text; + token.tokens = this.lexer.inline(text); } return token; @@ -819,8 +817,7 @@ var Tokenizer = /*#__PURE__*/function () { l = item.header.length; for (j = 0; j < l; j++) { - item.header[j].tokens = []; - this.lexer.inline(item.header[j].text, item.header[j].tokens); + item.header[j].tokens = this.lexer.inline(item.header[j].text); } // cell child tokens @@ -830,8 +827,7 @@ var Tokenizer = /*#__PURE__*/function () { row = item.rows[j]; for (k = 0; k < row.length; k++) { - row[k].tokens = []; - this.lexer.inline(row[k].text, row[k].tokens); + row[k].tokens = this.lexer.inline(row[k].text); } } @@ -844,15 +840,13 @@ var Tokenizer = /*#__PURE__*/function () { var cap = this.rules.block.lheading.exec(src); if (cap) { - var token = { + return { type: 'heading', raw: cap[0], depth: cap[2].charAt(0) === '=' ? 1 : 2, text: cap[1], - tokens: [] + tokens: this.lexer.inline(cap[1]) }; - this.lexer.inline(token.text, token.tokens); - return token; } }; @@ -860,14 +854,13 @@ var Tokenizer = /*#__PURE__*/function () { var cap = this.rules.block.paragraph.exec(src); if (cap) { - var token = { + var text = cap[1].charAt(cap[1].length - 1) === '\n' ? cap[1].slice(0, -1) : cap[1]; + return { type: 'paragraph', raw: cap[0], - text: cap[1].charAt(cap[1].length - 1) === '\n' ? cap[1].slice(0, -1) : cap[1], - tokens: [] + text: text, + tokens: this.lexer.inline(text) }; - this.lexer.inline(token.text, token.tokens); - return token; } }; @@ -875,14 +868,12 @@ var Tokenizer = /*#__PURE__*/function () { var cap = this.rules.block.text.exec(src); if (cap) { - var token = { + return { type: 'text', raw: cap[0], text: cap[0], - tokens: [] + tokens: this.lexer.inline(cap[0]) }; - this.lexer.inline(token.text, token.tokens); - return token; } }; @@ -1061,7 +1052,7 @@ var Tokenizer = /*#__PURE__*/function () { type: 'em', raw: src.slice(0, lLength + match.index + rLength + 1), text: _text, - tokens: this.lexer.inlineTokens(_text, []) + tokens: this.lexer.inlineTokens(_text) }; } // Create 'strong' if smallest delimiter has even char count. **a*** @@ -1071,7 +1062,7 @@ var Tokenizer = /*#__PURE__*/function () { type: 'strong', raw: src.slice(0, lLength + match.index + rLength + 1), text: text, - tokens: this.lexer.inlineTokens(text, []) + tokens: this.lexer.inlineTokens(text) }; } } @@ -1117,7 +1108,7 @@ var Tokenizer = /*#__PURE__*/function () { type: 'del', raw: cap[0], text: cap[2], - tokens: this.lexer.inlineTokens(cap[2], []) + tokens: this.lexer.inlineTokens(cap[2]) }; } }; @@ -1735,10 +1726,15 @@ var Lexer = /*#__PURE__*/function () { }; _proto.inline = function inline(src, tokens) { + if (tokens === void 0) { + tokens = []; + } + this.inlineQueue.push({ src: src, tokens: tokens }); + return tokens; } /** * Lexing/Compiling @@ -2711,22 +2707,32 @@ function marked(src, opt, callback) { return; } + function onError(e) { + e.message += '\nPlease report this to https://github.com/markedjs/marked.'; + + if (opt.silent) { + return '
An error occurred:
' + escape(e.message + '', true) + ''; + } + + throw e; + } + try { var _tokens = Lexer.lex(src, opt); if (opt.walkTokens) { + if (opt.async) { + return Promise.all(marked.walkTokens(_tokens, opt.walkTokens)).then(function () { + return Parser.parse(_tokens, opt); + })["catch"](onError); + } + marked.walkTokens(_tokens, opt.walkTokens); } return Parser.parse(_tokens, opt); } catch (e) { - e.message += '\nPlease report this to https://github.com/markedjs/marked.'; - - if (opt.silent) { - return '
An error occurred:
' + escape(e.message + '', true) + ''; - } - - throw e; + onError(e); } } /** @@ -2892,11 +2898,14 @@ marked.use = function () { var _walkTokens = marked.defaults.walkTokens; opts.walkTokens = function (token) { - pack.walkTokens.call(this, token); + var values = []; + values.concat(pack.walkTokens.call(this, token)); if (_walkTokens) { - _walkTokens.call(this, token); + values.concat(_walkTokens.call(this, token)); } + + return values; }; } @@ -2913,16 +2922,18 @@ marked.use = function () { marked.walkTokens = function (tokens, callback) { + var values = []; + var _loop3 = function _loop3() { var token = _step.value; - callback.call(marked, token); + values.concat(callback.call(marked, token)); switch (token.type) { case 'table': { for (var _iterator2 = _createForOfIteratorHelperLoose(token.header), _step2; !(_step2 = _iterator2()).done;) { var cell = _step2.value; - marked.walkTokens(cell.tokens, callback); + values.concat(marked.walkTokens(cell.tokens, callback)); } for (var _iterator3 = _createForOfIteratorHelperLoose(token.rows), _step3; !(_step3 = _iterator3()).done;) { @@ -2930,7 +2941,7 @@ marked.walkTokens = function (tokens, callback) { for (var _iterator4 = _createForOfIteratorHelperLoose(row), _step4; !(_step4 = _iterator4()).done;) { var _cell = _step4.value; - marked.walkTokens(_cell.tokens, callback); + values.concat(marked.walkTokens(_cell.tokens, callback)); } } @@ -2939,7 +2950,7 @@ marked.walkTokens = function (tokens, callback) { case 'list': { - marked.walkTokens(token.items, callback); + values.concat(marked.walkTokens(token.items, callback)); break; } @@ -2948,10 +2959,10 @@ marked.walkTokens = function (tokens, callback) { if (marked.defaults.extensions && marked.defaults.extensions.childTokens && marked.defaults.extensions.childTokens[token.type]) { // Walk any extensions marked.defaults.extensions.childTokens[token.type].forEach(function (childTokens) { - marked.walkTokens(token[childTokens], callback); + values.concat(marked.walkTokens(token[childTokens], callback)); }); } else if (token.tokens) { - marked.walkTokens(token.tokens, callback); + values.concat(marked.walkTokens(token.tokens, callback)); } } } @@ -2960,6 +2971,8 @@ marked.walkTokens = function (tokens, callback) { for (var _iterator = _createForOfIteratorHelperLoose(tokens), _step; !(_step = _iterator()).done;) { _loop3(); } + + return values; }; /** * Parse Inline diff --git a/lib/marked.esm.js b/lib/marked.esm.js index 21074efd07..9b06967ffd 100644 --- a/lib/marked.esm.js +++ b/lib/marked.esm.js @@ -329,7 +329,7 @@ function outputLink(cap, link, raw, lexer) { href, title, text, - tokens: lexer.inlineTokens(text, []) + tokens: lexer.inlineTokens(text) }; lexer.state.inLink = false; return token; @@ -435,15 +435,13 @@ class Tokenizer { } } - const token = { + return { type: 'heading', raw: cap[0], depth: cap[1].length, text, - tokens: [] + tokens: this.lexer.inline(text) }; - this.lexer.inline(token.text, token.tokens); - return token; } } @@ -653,10 +651,10 @@ class Tokenizer { text: cap[0] }; if (this.options.sanitize) { + const text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]); token.type = 'paragraph'; - token.text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]); - token.tokens = []; - this.lexer.inline(token.text, token.tokens); + token.text = text; + token.tokens = this.lexer.inline(text); } return token; } @@ -714,8 +712,7 @@ class Tokenizer { // header child tokens l = item.header.length; for (j = 0; j < l; j++) { - item.header[j].tokens = []; - this.lexer.inline(item.header[j].text, item.header[j].tokens); + item.header[j].tokens = this.lexer.inline(item.header[j].text); } // cell child tokens @@ -723,8 +720,7 @@ class Tokenizer { for (j = 0; j < l; j++) { row = item.rows[j]; for (k = 0; k < row.length; k++) { - row[k].tokens = []; - this.lexer.inline(row[k].text, row[k].tokens); + row[k].tokens = this.lexer.inline(row[k].text); } } @@ -736,45 +732,40 @@ class Tokenizer { lheading(src) { const cap = this.rules.block.lheading.exec(src); if (cap) { - const token = { + return { type: 'heading', raw: cap[0], depth: cap[2].charAt(0) === '=' ? 1 : 2, text: cap[1], - tokens: [] + tokens: this.lexer.inline(cap[1]) }; - this.lexer.inline(token.text, token.tokens); - return token; } } paragraph(src) { const cap = this.rules.block.paragraph.exec(src); if (cap) { - const token = { + const text = cap[1].charAt(cap[1].length - 1) === '\n' + ? cap[1].slice(0, -1) + : cap[1]; + return { type: 'paragraph', raw: cap[0], - text: cap[1].charAt(cap[1].length - 1) === '\n' - ? cap[1].slice(0, -1) - : cap[1], - tokens: [] + text, + tokens: this.lexer.inline(text) }; - this.lexer.inline(token.text, token.tokens); - return token; } } text(src) { const cap = this.rules.block.text.exec(src); if (cap) { - const token = { + return { type: 'text', raw: cap[0], text: cap[0], - tokens: [] + tokens: this.lexer.inline(cap[0]) }; - this.lexer.inline(token.text, token.tokens); - return token; } } @@ -943,7 +934,7 @@ class Tokenizer { type: 'em', raw: src.slice(0, lLength + match.index + rLength + 1), text, - tokens: this.lexer.inlineTokens(text, []) + tokens: this.lexer.inlineTokens(text) }; } @@ -953,7 +944,7 @@ class Tokenizer { type: 'strong', raw: src.slice(0, lLength + match.index + rLength + 1), text, - tokens: this.lexer.inlineTokens(text, []) + tokens: this.lexer.inlineTokens(text) }; } } @@ -994,7 +985,7 @@ class Tokenizer { type: 'del', raw: cap[0], text: cap[2], - tokens: this.lexer.inlineTokens(cap[2], []) + tokens: this.lexer.inlineTokens(cap[2]) }; } } @@ -1692,8 +1683,9 @@ class Lexer { return tokens; } - inline(src, tokens) { + inline(src, tokens = []) { this.inlineQueue.push({ src, tokens }); + return tokens; } /** @@ -2542,13 +2534,7 @@ function marked(src, opt, callback) { return; } - try { - const tokens = Lexer.lex(src, opt); - if (opt.walkTokens) { - marked.walkTokens(tokens, opt.walkTokens); - } - return Parser.parse(tokens, opt); - } catch (e) { + function onError(e) { e.message += '\nPlease report this to https://github.com/markedjs/marked.'; if (opt.silent) { return '
An error occurred:
' @@ -2557,6 +2543,23 @@ function marked(src, opt, callback) { } throw e; } + + try { + const tokens = Lexer.lex(src, opt); + if (opt.walkTokens) { + if (opt.async) { + return Promise.all(marked.walkTokens(tokens, opt.walkTokens)) + .then(() => { + return Parser.parse(tokens, opt); + }) + .catch(onError); + } + marked.walkTokens(tokens, opt.walkTokens); + } + return Parser.parse(tokens, opt); + } catch (e) { + onError(e); + } } /** @@ -2673,10 +2676,12 @@ marked.use = function(...args) { if (pack.walkTokens) { const walkTokens = marked.defaults.walkTokens; opts.walkTokens = function(token) { - pack.walkTokens.call(this, token); + const values = []; + values.concat(pack.walkTokens.call(this, token)); if (walkTokens) { - walkTokens.call(this, token); + values.concat(walkTokens.call(this, token)); } + return values; }; } @@ -2693,35 +2698,37 @@ marked.use = function(...args) { */ marked.walkTokens = function(tokens, callback) { + const values = []; for (const token of tokens) { - callback.call(marked, token); + values.concat(callback.call(marked, token)); switch (token.type) { case 'table': { for (const cell of token.header) { - marked.walkTokens(cell.tokens, callback); + values.concat(marked.walkTokens(cell.tokens, callback)); } for (const row of token.rows) { for (const cell of row) { - marked.walkTokens(cell.tokens, callback); + values.concat(marked.walkTokens(cell.tokens, callback)); } } break; } case 'list': { - marked.walkTokens(token.items, callback); + values.concat(marked.walkTokens(token.items, callback)); break; } default: { if (marked.defaults.extensions && marked.defaults.extensions.childTokens && marked.defaults.extensions.childTokens[token.type]) { // Walk any extensions marked.defaults.extensions.childTokens[token.type].forEach(function(childTokens) { - marked.walkTokens(token[childTokens], callback); + values.concat(marked.walkTokens(token[childTokens], callback)); }); } else if (token.tokens) { - marked.walkTokens(token.tokens, callback); + values.concat(marked.walkTokens(token.tokens, callback)); } } } } + return values; }; /** diff --git a/lib/marked.umd.js b/lib/marked.umd.js index 0d974a7c6a..d812bfe1a1 100644 --- a/lib/marked.umd.js +++ b/lib/marked.umd.js @@ -414,7 +414,7 @@ href: href, title: title, text: text, - tokens: lexer.inlineTokens(text, []) + tokens: lexer.inlineTokens(text) }; lexer.state.inLink = false; return token; @@ -522,15 +522,13 @@ } } - var token = { + return { type: 'heading', raw: cap[0], depth: cap[1].length, text: text, - tokens: [] + tokens: this.lexer.inline(text) }; - this.lexer.inline(token.text, token.tokens); - return token; } }; @@ -748,10 +746,10 @@ }; if (this.options.sanitize) { + var text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]); token.type = 'paragraph'; - token.text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]); - token.tokens = []; - this.lexer.inline(token.text, token.tokens); + token.text = text; + token.tokens = this.lexer.inline(text); } return token; @@ -821,8 +819,7 @@ l = item.header.length; for (j = 0; j < l; j++) { - item.header[j].tokens = []; - this.lexer.inline(item.header[j].text, item.header[j].tokens); + item.header[j].tokens = this.lexer.inline(item.header[j].text); } // cell child tokens @@ -832,8 +829,7 @@ row = item.rows[j]; for (k = 0; k < row.length; k++) { - row[k].tokens = []; - this.lexer.inline(row[k].text, row[k].tokens); + row[k].tokens = this.lexer.inline(row[k].text); } } @@ -846,15 +842,13 @@ var cap = this.rules.block.lheading.exec(src); if (cap) { - var token = { + return { type: 'heading', raw: cap[0], depth: cap[2].charAt(0) === '=' ? 1 : 2, text: cap[1], - tokens: [] + tokens: this.lexer.inline(cap[1]) }; - this.lexer.inline(token.text, token.tokens); - return token; } }; @@ -862,14 +856,13 @@ var cap = this.rules.block.paragraph.exec(src); if (cap) { - var token = { + var text = cap[1].charAt(cap[1].length - 1) === '\n' ? cap[1].slice(0, -1) : cap[1]; + return { type: 'paragraph', raw: cap[0], - text: cap[1].charAt(cap[1].length - 1) === '\n' ? cap[1].slice(0, -1) : cap[1], - tokens: [] + text: text, + tokens: this.lexer.inline(text) }; - this.lexer.inline(token.text, token.tokens); - return token; } }; @@ -877,14 +870,12 @@ var cap = this.rules.block.text.exec(src); if (cap) { - var token = { + return { type: 'text', raw: cap[0], text: cap[0], - tokens: [] + tokens: this.lexer.inline(cap[0]) }; - this.lexer.inline(token.text, token.tokens); - return token; } }; @@ -1063,7 +1054,7 @@ type: 'em', raw: src.slice(0, lLength + match.index + rLength + 1), text: _text, - tokens: this.lexer.inlineTokens(_text, []) + tokens: this.lexer.inlineTokens(_text) }; } // Create 'strong' if smallest delimiter has even char count. **a*** @@ -1073,7 +1064,7 @@ type: 'strong', raw: src.slice(0, lLength + match.index + rLength + 1), text: text, - tokens: this.lexer.inlineTokens(text, []) + tokens: this.lexer.inlineTokens(text) }; } } @@ -1119,7 +1110,7 @@ type: 'del', raw: cap[0], text: cap[2], - tokens: this.lexer.inlineTokens(cap[2], []) + tokens: this.lexer.inlineTokens(cap[2]) }; } }; @@ -1737,10 +1728,15 @@ }; _proto.inline = function inline(src, tokens) { + if (tokens === void 0) { + tokens = []; + } + this.inlineQueue.push({ src: src, tokens: tokens }); + return tokens; } /** * Lexing/Compiling @@ -2713,22 +2709,32 @@ return; } + function onError(e) { + e.message += '\nPlease report this to https://github.com/markedjs/marked.'; + + if (opt.silent) { + return 'An error occurred:
' + escape(e.message + '', true) + ''; + } + + throw e; + } + try { var _tokens = Lexer.lex(src, opt); if (opt.walkTokens) { + if (opt.async) { + return Promise.all(marked.walkTokens(_tokens, opt.walkTokens)).then(function () { + return Parser.parse(_tokens, opt); + })["catch"](onError); + } + marked.walkTokens(_tokens, opt.walkTokens); } return Parser.parse(_tokens, opt); } catch (e) { - e.message += '\nPlease report this to https://github.com/markedjs/marked.'; - - if (opt.silent) { - return 'An error occurred:
' + escape(e.message + '', true) + ''; - } - - throw e; + onError(e); } } /** @@ -2894,11 +2900,14 @@ var _walkTokens = marked.defaults.walkTokens; opts.walkTokens = function (token) { - pack.walkTokens.call(this, token); + var values = []; + values.concat(pack.walkTokens.call(this, token)); if (_walkTokens) { - _walkTokens.call(this, token); + values.concat(_walkTokens.call(this, token)); } + + return values; }; } @@ -2915,16 +2924,18 @@ marked.walkTokens = function (tokens, callback) { + var values = []; + var _loop3 = function _loop3() { var token = _step.value; - callback.call(marked, token); + values.concat(callback.call(marked, token)); switch (token.type) { case 'table': { for (var _iterator2 = _createForOfIteratorHelperLoose(token.header), _step2; !(_step2 = _iterator2()).done;) { var cell = _step2.value; - marked.walkTokens(cell.tokens, callback); + values.concat(marked.walkTokens(cell.tokens, callback)); } for (var _iterator3 = _createForOfIteratorHelperLoose(token.rows), _step3; !(_step3 = _iterator3()).done;) { @@ -2932,7 +2943,7 @@ for (var _iterator4 = _createForOfIteratorHelperLoose(row), _step4; !(_step4 = _iterator4()).done;) { var _cell = _step4.value; - marked.walkTokens(_cell.tokens, callback); + values.concat(marked.walkTokens(_cell.tokens, callback)); } } @@ -2941,7 +2952,7 @@ case 'list': { - marked.walkTokens(token.items, callback); + values.concat(marked.walkTokens(token.items, callback)); break; } @@ -2950,10 +2961,10 @@ if (marked.defaults.extensions && marked.defaults.extensions.childTokens && marked.defaults.extensions.childTokens[token.type]) { // Walk any extensions marked.defaults.extensions.childTokens[token.type].forEach(function (childTokens) { - marked.walkTokens(token[childTokens], callback); + values.concat(marked.walkTokens(token[childTokens], callback)); }); } else if (token.tokens) { - marked.walkTokens(token.tokens, callback); + values.concat(marked.walkTokens(token.tokens, callback)); } } } @@ -2962,6 +2973,8 @@ for (var _iterator = _createForOfIteratorHelperLoose(tokens), _step; !(_step = _iterator()).done;) { _loop3(); } + + return values; }; /** * Parse Inline diff --git a/src/Lexer.js b/src/Lexer.js index 3c0a2c7e01..c4bbf41a83 100644 --- a/src/Lexer.js +++ b/src/Lexer.js @@ -316,8 +316,9 @@ export class Lexer { return tokens; } - inline(src, tokens) { + inline(src, tokens = []) { this.inlineQueue.push({ src, tokens }); + return tokens; } /** diff --git a/src/Tokenizer.js b/src/Tokenizer.js index e8a69b6759..17f650e5a6 100644 --- a/src/Tokenizer.js +++ b/src/Tokenizer.js @@ -19,7 +19,7 @@ function outputLink(cap, link, raw, lexer) { href, title, text, - tokens: lexer.inlineTokens(text, []) + tokens: lexer.inlineTokens(text) }; lexer.state.inLink = false; return token; @@ -125,15 +125,13 @@ export class Tokenizer { } } - const token = { + return { type: 'heading', raw: cap[0], depth: cap[1].length, text, - tokens: [] + tokens: this.lexer.inline(text) }; - this.lexer.inline(token.text, token.tokens); - return token; } } @@ -343,10 +341,10 @@ export class Tokenizer { text: cap[0] }; if (this.options.sanitize) { + const text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]); token.type = 'paragraph'; - token.text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]); - token.tokens = []; - this.lexer.inline(token.text, token.tokens); + token.text = text; + token.tokens = this.lexer.inline(text); } return token; } @@ -404,8 +402,7 @@ export class Tokenizer { // header child tokens l = item.header.length; for (j = 0; j < l; j++) { - item.header[j].tokens = []; - this.lexer.inline(item.header[j].text, item.header[j].tokens); + item.header[j].tokens = this.lexer.inline(item.header[j].text); } // cell child tokens @@ -413,8 +410,7 @@ export class Tokenizer { for (j = 0; j < l; j++) { row = item.rows[j]; for (k = 0; k < row.length; k++) { - row[k].tokens = []; - this.lexer.inline(row[k].text, row[k].tokens); + row[k].tokens = this.lexer.inline(row[k].text); } } @@ -426,45 +422,40 @@ export class Tokenizer { lheading(src) { const cap = this.rules.block.lheading.exec(src); if (cap) { - const token = { + return { type: 'heading', raw: cap[0], depth: cap[2].charAt(0) === '=' ? 1 : 2, text: cap[1], - tokens: [] + tokens: this.lexer.inline(cap[1]) }; - this.lexer.inline(token.text, token.tokens); - return token; } } paragraph(src) { const cap = this.rules.block.paragraph.exec(src); if (cap) { - const token = { + const text = cap[1].charAt(cap[1].length - 1) === '\n' + ? cap[1].slice(0, -1) + : cap[1]; + return { type: 'paragraph', raw: cap[0], - text: cap[1].charAt(cap[1].length - 1) === '\n' - ? cap[1].slice(0, -1) - : cap[1], - tokens: [] + text, + tokens: this.lexer.inline(text) }; - this.lexer.inline(token.text, token.tokens); - return token; } } text(src) { const cap = this.rules.block.text.exec(src); if (cap) { - const token = { + return { type: 'text', raw: cap[0], text: cap[0], - tokens: [] + tokens: this.lexer.inline(cap[0]) }; - this.lexer.inline(token.text, token.tokens); - return token; } } @@ -633,7 +624,7 @@ export class Tokenizer { type: 'em', raw: src.slice(0, lLength + match.index + rLength + 1), text, - tokens: this.lexer.inlineTokens(text, []) + tokens: this.lexer.inlineTokens(text) }; } @@ -643,7 +634,7 @@ export class Tokenizer { type: 'strong', raw: src.slice(0, lLength + match.index + rLength + 1), text, - tokens: this.lexer.inlineTokens(text, []) + tokens: this.lexer.inlineTokens(text) }; } } @@ -684,7 +675,7 @@ export class Tokenizer { type: 'del', raw: cap[0], text: cap[2], - tokens: this.lexer.inlineTokens(cap[2], []) + tokens: this.lexer.inlineTokens(cap[2]) }; } } diff --git a/src/marked.js b/src/marked.js index 10f543336c..f91d30459f 100644 --- a/src/marked.js +++ b/src/marked.js @@ -105,13 +105,7 @@ export function marked(src, opt, callback) { return; } - try { - const tokens = Lexer.lex(src, opt); - if (opt.walkTokens) { - marked.walkTokens(tokens, opt.walkTokens); - } - return Parser.parse(tokens, opt); - } catch (e) { + function onError(e) { e.message += '\nPlease report this to https://github.com/markedjs/marked.'; if (opt.silent) { return 'An error occurred:
' @@ -120,6 +114,23 @@ export function marked(src, opt, callback) { } throw e; } + + try { + const tokens = Lexer.lex(src, opt); + if (opt.walkTokens) { + if (opt.async) { + return Promise.all(marked.walkTokens(tokens, opt.walkTokens)) + .then(() => { + return Parser.parse(tokens, opt); + }) + .catch(onError); + } + marked.walkTokens(tokens, opt.walkTokens); + } + return Parser.parse(tokens, opt); + } catch (e) { + onError(e); + } } /** @@ -236,10 +247,12 @@ marked.use = function(...args) { if (pack.walkTokens) { const walkTokens = marked.defaults.walkTokens; opts.walkTokens = function(token) { - pack.walkTokens.call(this, token); + const values = []; + values.concat(pack.walkTokens.call(this, token)); if (walkTokens) { - walkTokens.call(this, token); + values.concat(walkTokens.call(this, token)); } + return values; }; } @@ -256,35 +269,37 @@ marked.use = function(...args) { */ marked.walkTokens = function(tokens, callback) { + const values = []; for (const token of tokens) { - callback.call(marked, token); + values.concat(callback.call(marked, token)); switch (token.type) { case 'table': { for (const cell of token.header) { - marked.walkTokens(cell.tokens, callback); + values.concat(marked.walkTokens(cell.tokens, callback)); } for (const row of token.rows) { for (const cell of row) { - marked.walkTokens(cell.tokens, callback); + values.concat(marked.walkTokens(cell.tokens, callback)); } } break; } case 'list': { - marked.walkTokens(token.items, callback); + values.concat(marked.walkTokens(token.items, callback)); break; } default: { if (marked.defaults.extensions && marked.defaults.extensions.childTokens && marked.defaults.extensions.childTokens[token.type]) { // Walk any extensions marked.defaults.extensions.childTokens[token.type].forEach(function(childTokens) { - marked.walkTokens(token[childTokens], callback); + values.concat(marked.walkTokens(token[childTokens], callback)); }); } else if (token.tokens) { - marked.walkTokens(token.tokens, callback); + values.concat(marked.walkTokens(token.tokens, callback)); } } } } + return values; }; /** diff --git a/test/bench.js b/test/bench.js index 7afd24f0e6..7b3d9e71b2 100644 --- a/test/bench.js +++ b/test/bench.js @@ -3,6 +3,7 @@ import { fileURLToPath } from 'url'; import { isEqual } from './helpers/html-differ.js'; import { loadFiles } from './helpers/load.js'; +import { marked as cjsMarked } from '../lib/marked.cjs'; import { marked as esmMarked } from '../lib/marked.esm.js'; const __dirname = dirname(fileURLToPath(import.meta.url)); @@ -30,9 +31,10 @@ export function load() { export async function runBench(options) { options = options || {}; const specs = load(); + const tests = {}; // Non-GFM, Non-pedantic - marked.setOptions({ + cjsMarked.setOptions({ gfm: false, breaks: false, pedantic: false, @@ -40,9 +42,9 @@ export async function runBench(options) { smartLists: false }); if (options.marked) { - marked.setOptions(options.marked); + cjsMarked.setOptions(options.marked); } - await bench('cjs marked', specs, marked.parse); + tests['cjs marked'] = cjsMarked.parse; esmMarked.setOptions({ gfm: false, @@ -54,113 +56,76 @@ export async function runBench(options) { if (options.marked) { esmMarked.setOptions(options.marked); } - await bench('esm marked', specs, esmMarked.parse); + tests['esm marked'] = esmMarked.parse; - // GFM - marked.setOptions({ - gfm: true, - breaks: false, - pedantic: false, - sanitize: false, - smartLists: false - }); - if (options.marked) { - marked.setOptions(options.marked); - } - await bench('cjs marked (gfm)', specs, marked.parse); - - esmMarked.setOptions({ - gfm: true, - breaks: false, - pedantic: false, - sanitize: false, - smartLists: false - }); - if (options.marked) { - esmMarked.setOptions(options.marked); - } - await bench('esm marked (gfm)', specs, esmMarked.parse); - - // Pedantic - marked.setOptions({ - gfm: false, - breaks: false, - pedantic: true, - sanitize: false, - smartLists: false - }); - if (options.marked) { - marked.setOptions(options.marked); - } - await bench('cjs marked (pedantic)', specs, marked.parse); - - esmMarked.setOptions({ - gfm: false, - breaks: false, - pedantic: true, - sanitize: false, - smartLists: false - }); - if (options.marked) { - esmMarked.setOptions(options.marked); - } - await bench('esm marked (pedantic)', specs, esmMarked.parse); + // esmMarked.setOptions({ + // gfm: true, + // breaks: false, + // pedantic: false, + // sanitize: false, + // smartLists: false + // }); + // if (options.marked) { + // esmMarked.setOptions(options.marked); + // } + // tests['esm marked (gfm)'] = esmMarked.parse; try { - await bench('commonmark', specs, (await (async() => { + tests.commonmark = (await (async() => { const { Parser, HtmlRenderer } = await import('commonmark'); const parser = new Parser(); const writer = new HtmlRenderer(); return function(text) { return writer.render(parser.parse(text)); }; - })())); + })()); } catch (e) { console.error('Could not bench commonmark. (Error: %s)', e.message); } try { - await bench('markdown-it', specs, (await (async() => { + tests['markdown-it'] = (await (async() => { const MarkdownIt = (await import('markdown-it')).default; const md = new MarkdownIt(); return md.render.bind(md); - })())); + })()); } catch (e) { console.error('Could not bench markdown-it. (Error: %s)', e.message); } + + await bench(tests, specs); } -export async function bench(name, specs, engine) { - const before = process.hrtime(); - for (let i = 0; i < 1e3; i++) { - for (const spec of specs) { - await engine(spec.markdown); +export async function bench(tests, specs) { + const stats = {}; + for (const name in tests) { + stats[name] = { + elapsed: 0n, + correct: 0 + }; + } + + console.log(); + for (let i = 0; i < specs.length; i++) { + const spec = specs[i]; + process.stdout.write(`${(i * 100 / specs.length).toFixed(1).padStart(5)}% ${i.toString().padStart(specs.length.toString().length)} of ${specs.length}\r`); + for (const name in tests) { + const test = tests[name]; + const before = process.hrtime.bigint(); + for (let n = 0; n < 1e3; n++) { + await test(spec.markdown); + } + const after = process.hrtime.bigint(); + stats[name].elapsed += after - before; + stats[name].correct += (await isEqual(spec.html, await test(spec.markdown)) ? 1 : 0); } } - const elapsed = process.hrtime(before); - const ms = prettyElapsedTime(elapsed).toFixed(); - let correct = 0; - for (const spec of specs) { - if (await isEqual(spec.html, await engine(spec.markdown))) { - correct++; - } + for (const name in tests) { + const ms = prettyElapsedTime(stats[name].elapsed); + const percent = (stats[name].correct / specs.length * 100).toFixed(2); + console.log(`${name} completed in ${ms}ms and passed ${percent}%`); } - const percent = (correct / specs.length * 100).toFixed(2); - - console.log('%s completed in %sms and passed %s%', name, ms, percent); -} - -/** - * A simple one-time benchmark - */ -export async function time(options) { - options = options || {}; - const specs = load(); - if (options.marked) { - marked.setOptions(options.marked); - } - await bench('marked', specs, marked); } /** @@ -204,35 +169,23 @@ function parseArg(argv) { while (argv.length) { const arg = getarg(); - switch (arg) { - case '-t': - case '--time': - options.time = true; - break; - case '-m': - case '--minified': - options.minified = true; - break; - default: - if (arg.indexOf('--') === 0) { - const opt = camelize(arg.replace(/^--(no-)?/, '')); - if (!defaults.hasOwnProperty(opt)) { - continue; - } - options.marked = options.marked || {}; - if (arg.indexOf('--no-') === 0) { - options.marked[opt] = typeof defaults[opt] !== 'boolean' - ? null - : false; - } else { - options.marked[opt] = typeof defaults[opt] !== 'boolean' - ? argv.shift() - : true; - } - } else { - orphans.push(arg); - } - break; + if (arg.indexOf('--') === 0) { + const opt = camelize(arg.replace(/^--(no-)?/, '')); + if (!defaults.hasOwnProperty(opt)) { + continue; + } + options.marked = options.marked || {}; + if (arg.indexOf('--no-') === 0) { + options.marked[opt] = typeof defaults[opt] !== 'boolean' + ? null + : false; + } else { + options.marked[opt] = typeof defaults[opt] !== 'boolean' + ? argv.shift() + : true; + } + } else { + orphans.push(arg); } } @@ -257,28 +210,19 @@ function camelize(text) { * Main */ export default async function main(argv) { - marked = (await import('../lib/marked.cjs')).marked; + marked = cjsMarked; const opt = parseArg(argv); - if (opt.minified) { - marked = (await import('../marked.min.js')).marked; - } - - if (opt.time) { - await time(opt); - } else { - await runBench(opt); - } + await runBench(opt); } /** * returns time to millisecond granularity + * @param hrtimeElapsed {bigint} */ function prettyElapsedTime(hrtimeElapsed) { - const seconds = hrtimeElapsed[0]; - const frac = Math.round(hrtimeElapsed[1] / 1e3) / 1e3; - return seconds * 1e3 + frac; + return Number(hrtimeElapsed / 1_000_000n); } process.title = 'marked bench';