diff --git a/docs/USING_ADVANCED.md b/docs/USING_ADVANCED.md index 3abf6427f0..23224d1687 100644 --- a/docs/USING_ADVANCED.md +++ b/docs/USING_ADVANCED.md @@ -58,6 +58,7 @@ console.log(marked(markdownString)); |smartLists |`boolean` |`false` |v0.2.8 |If true, use smarter list behavior than those found in `markdown.pl`.| |smartypants |`boolean` |`false` |v0.2.9 |If true, use "smart" typographic punctuation for things like quotes and dashes.| |tokenizer |`object` |`new Tokenizer()`|v1.0.0|An object containing functions to create tokens from markdown. See [extensibility](/#/USING_PRO.md) for more details.| +|walkTokens |`function` |`null`|v1.1.0|A function which is called for every token. See [extensibility](/#/USING_PRO.md) for more details.| |xhtml |`boolean` |`false` |v0.3.2 |If true, emit self-closing HTML tags for void elements (<br/>, <img/>, etc.) with a "/" as required by XHTML.|

Asynchronous highlighting

diff --git a/docs/USING_PRO.md b/docs/USING_PRO.md index 9b6bd7ac7c..5912d2b5e6 100644 --- a/docs/USING_PRO.md +++ b/docs/USING_PRO.md @@ -10,6 +10,8 @@ The `renderer` and `tokenizer` options can be an object with functions that will The `renderer` and `tokenizer` functions can return false to fallback to the previous function. +The `walkTokens` option can be a function that will be called with every token before rendering. When calling `use` multiple times with different `walkTokens` functions each function will be called in the **reverse** order in which they were assigned. + All other options will overwrite previously set options.

The renderer

@@ -188,6 +190,35 @@ smartypants('"this ... string"') // "“this … string”" ``` +

Walk Tokens

+ +The walkTokens function gets called with every token. Child tokens are called before moving on to sibling tokens. Each token is passed by reference so updates are persisted when passed to the parser. The return value of the function is ignored. + +**Example:** Overriding heading tokens to start at h2. + +```js +const marked = require('marked'); + +// Override function +const walkTokens = (token) => { + if (token.type === 'heading') { + token.depth += 1; + } +}; + +marked.use({ walkTokens }); + +// Run marked +console.log(marked('# heading 2\n\n## heading 3')); +``` + +**Output:** + +```html +

heading 2

+

heading 3

+``` +

The lexer

The lexer takes a markdown string and calls the tokenizer functions. diff --git a/docs/index.html b/docs/index.html index a7de1818c0..a6c293083e 100644 --- a/docs/index.html +++ b/docs/index.html @@ -157,6 +157,7 @@

Marked.js Documentation

  • marked.use()
  • Renderer
  • Tokenizer
    • +
  • Walk Tokens
  • Lexer
  • Parser
    • diff --git a/src/Tokenizer.js b/src/Tokenizer.js index 512b45dc50..e3ff002b77 100644 --- a/src/Tokenizer.js +++ b/src/Tokenizer.js @@ -269,6 +269,7 @@ module.exports = class Tokenizer { } list.items.push({ + type: 'list_item', raw, task: istask, checked: ischecked, diff --git a/src/defaults.js b/src/defaults.js index 0153bb4334..fe376563da 100644 --- a/src/defaults.js +++ b/src/defaults.js @@ -16,6 +16,7 @@ function getDefaults() { smartLists: false, smartypants: false, tokenizer: null, + walkTokens: null, xhtml: false }; } diff --git a/src/marked.js b/src/marked.js index 8488e79f03..83ffc00394 100644 --- a/src/marked.js +++ b/src/marked.js @@ -28,18 +28,17 @@ function marked(src, opt, callback) { + Object.prototype.toString.call(src) + ', string expected'); } - if (callback || typeof opt === 'function') { - if (!callback) { - callback = opt; - opt = null; - } + if (typeof opt === 'function') { + callback = opt; + opt = null; + } - opt = merge({}, marked.defaults, opt || {}); - checkSanitizeDeprecation(opt); + opt = merge({}, marked.defaults, opt || {}); + checkSanitizeDeprecation(opt); + + if (callback) { const highlight = opt.highlight; - let tokens, - pending, - i = 0; + let tokens; try { tokens = Lexer.lex(src, opt); @@ -47,20 +46,15 @@ function marked(src, opt, callback) { return callback(e); } - pending = tokens.length; - const done = function(err) { - if (err) { - opt.highlight = highlight; - return callback(err); - } - let out; - try { - out = Parser.parse(tokens, opt); - } catch (e) { - err = e; + if (!err) { + try { + out = Parser.parse(tokens, opt); + } catch (e) { + err = e; + } } opt.highlight = highlight; @@ -76,34 +70,45 @@ function marked(src, opt, callback) { delete opt.highlight; - if (!pending) return done(); + if (!tokens.length) return done(); - for (; i < tokens.length; i++) { - (function(token) { - if (token.type !== 'code') { - return --pending || done(); - } - return highlight(token.text, token.lang, function(err, code) { - if (err) return done(err); - if (code == null || code === token.text) { - return --pending || done(); + let pending = 0; + marked.walkTokens(tokens, function(token) { + if (token.type === 'code') { + pending++; + highlight(token.text, token.lang, function(err, code) { + if (err) { + return done(err); + } + if (code != null && code !== token.text) { + token.text = code; + token.escaped = true; + } + + pending--; + if (pending === 0) { + done(); } - token.text = code; - token.escaped = true; - --pending || done(); }); - })(tokens[i]); + } + }); + + if (pending === 0) { + done(); } return; } + try { - opt = merge({}, marked.defaults, opt || {}); - checkSanitizeDeprecation(opt); - return Parser.parse(Lexer.lex(src, opt), opt); + const tokens = Lexer.lex(src, opt); + if (opt.walkTokens) { + marked.walkTokens(tokens, opt.walkTokens); + } + return Parser.parse(tokens, opt); } catch (e) { e.message += '\nPlease report this to https://github.com/markedjs/marked.'; - if ((opt || marked.defaults).silent) { + if (opt.silent) { return '

    An error occurred:

    '
         + escape(e.message + '', true)
         + '
    '; @@ -161,9 +166,50 @@ marked.use = function(extension) { } opts.tokenizer = tokenizer; } + if (extension.walkTokens) { + const walkTokens = marked.defaults.walkTokens; + opts.walkTokens = (token) => { + extension.walkTokens(token); + if (walkTokens) { + walkTokens(token); + } + }; + } marked.setOptions(opts); }; +/** + * Run callback for every token + */ + +marked.walkTokens = function(tokens, callback) { + for (const token of tokens) { + callback(token); + switch (token.type) { + case 'table': { + for (const cell of token.tokens.header) { + marked.walkTokens(cell, callback); + } + for (const row of token.tokens.cells) { + for (const cell of row) { + marked.walkTokens(cell, callback); + } + } + break; + } + case 'list': { + marked.walkTokens(token.items, callback); + break; + } + default: { + if (token.tokens) { + marked.walkTokens(token.tokens, callback); + } + } + } + } +}; + /** * Expose */ diff --git a/test/unit/Lexer-spec.js b/test/unit/Lexer-spec.js index 9b2b5d7612..fea19c9ff2 100644 --- a/test/unit/Lexer-spec.js +++ b/test/unit/Lexer-spec.js @@ -307,6 +307,7 @@ a | b loose: false, items: [ { + type: 'list_item', raw: '- item 1', task: false, checked: undefined, @@ -320,6 +321,7 @@ a | b }] }, { + type: 'list_item', raw: '- item 2\n', task: false, checked: undefined, diff --git a/test/unit/marked-spec.js b/test/unit/marked-spec.js index 1b6beafdd6..f1253b49bf 100644 --- a/test/unit/marked-spec.js +++ b/test/unit/marked-spec.js @@ -132,6 +132,18 @@ describe('use extension', () => { expect(html).toBe('

    extension

    \n'); }); + it('should use walkTokens', () => { + let walked = 0; + const extension = { + walkTokens(token) { + walked++; + } + }; + marked.use(extension); + marked('text'); + expect(walked).toBe(2); + }); + it('should use options from extension', () => { const extension = { headerIds: false @@ -141,6 +153,29 @@ describe('use extension', () => { expect(html).toBe('

    heading

    \n'); }); + it('should call all walkTokens in reverse order', () => { + let walkedOnce = 0; + let walkedTwice = 0; + const extension1 = { + walkTokens(token) { + if (token.walkedOnce) { + walkedTwice++; + } + } + }; + const extension2 = { + walkTokens(token) { + walkedOnce++; + token.walkedOnce = true; + } + }; + marked.use(extension1); + marked.use(extension2); + marked('text'); + expect(walkedOnce).toBe(2); + expect(walkedTwice).toBe(2); + }); + it('should use last extension function and not override others', () => { const extension1 = { renderer: { @@ -229,3 +264,167 @@ paragraph expect(html).toBe('arrow no options\nfunction options\nshorthand options\n'); }); }); + +describe('async highlight', () => { + let highlight, markdown; + beforeEach(() => { + highlight = jasmine.createSpy('highlight', (text, lang, callback) => { + setImmediate(() => { + callback(null, `async ${text || ''}`); + }); + }); + markdown = ` +\`\`\`lang1 +text 1 +\`\`\` + +> \`\`\`lang2 +> text 2 +> \`\`\` + +- \`\`\`lang3 + text 3 + \`\`\` +`; + }); + + it('should highlight codeblocks async', (done) => { + highlight.and.callThrough(); + + marked(markdown, { highlight }, (err, html) => { + if (err) { + fail(err); + } + + expect(html).toBe(`
    async text 1
    +
    +
    async text 2
    +
    + +`); + done(); + }); + }); + + it('should call callback for each error in highlight', (done) => { + highlight.and.callFake((lang, text, callback) => { + callback(new Error('highlight error')); + }); + + let numErrors = 0; + marked(markdown, { highlight }, (err, html) => { + expect(err).toBeTruthy(); + expect(html).toBeUndefined(); + + if (err) { + numErrors++; + } + + if (numErrors === 3) { + done(); + } + }); + }); +}); + +describe('walkTokens', () => { + it('should walk over every token', () => { + const markdown = ` +paragraph + +--- + +# heading + +\`\`\` +code +\`\`\` + +| a | b | +|---|---| +| 1 | 2 | +| 3 | 4 | + +> blockquote + +- list + +
    html
    + +[link](https://example.com) + +![image](https://example.com/image.jpg) + +**strong** + +*em* + +\`codespan\` + +~~del~~ + +br +br +`; + const tokens = marked.lexer(markdown, { ...marked.getDefaults(), breaks: true }); + const tokensSeen = []; + marked.walkTokens(tokens, (token) => { + tokensSeen.push([token.type, (token.raw || '').replace(/\n/g, '')]); + }); + + expect(tokensSeen).toEqual([ + ['paragraph', 'paragraph'], + ['text', 'paragraph'], + ['space', ''], + ['hr', '---'], + ['heading', '# heading'], + ['text', 'heading'], + ['code', '```code```'], + ['table', '| a | b ||---|---|| 1 | 2 || 3 | 4 |'], + ['text', 'a'], + ['text', 'b'], + ['text', '1'], + ['text', '2'], + ['text', '3'], + ['text', '4'], + ['blockquote', '> blockquote'], + ['paragraph', 'blockquote'], + ['text', 'blockquote'], + ['list', '- list'], + ['list_item', '- list'], + ['text', 'list'], + ['text', 'list'], + ['space', ''], + ['html', '
    html
    '], + ['paragraph', '[link](https://example.com)'], + ['link', '[link](https://example.com)'], + ['text', 'link'], + ['space', ''], + ['paragraph', '![image](https://example.com/image.jpg)'], + ['image', '![image](https://example.com/image.jpg)'], + ['space', ''], + ['paragraph', '**strong**'], + ['strong', '**strong**'], + ['text', 'strong'], + ['space', ''], + ['paragraph', '*em*'], + ['em', '*em*'], + ['text', 'em'], + ['space', ''], + ['paragraph', '`codespan`'], + ['codespan', '`codespan`'], + ['space', ''], + ['paragraph', '~~del~~'], + ['del', '~~del~~'], + ['text', 'del'], + ['space', ''], + ['paragraph', 'brbr'], + ['text', 'br'], + ['br', ''], + ['text', 'br'] + ]); + }); +});