add walkTokens and fix highlight async

docs/USING_ADVANCED.md

@@ -58,6 +58,7 @@ console.log(marked(markdownString));
 |smartLists  |`boolean` |`false`  |v0.2.8   |If true, use smarter list behavior than those found in `markdown.pl`.|
 |smartypants |`boolean` |`false`  |v0.2.9   |If true, use "smart" typographic punctuation for things like quotes and dashes.|
 |tokenizer    |`object`  |`new Tokenizer()`|v1.0.0|An object containing functions to create tokens from markdown. See [extensibility](/#/USING_PRO.md) for more details.|
+|walkTokens   |`function`  |`null`|v1.1.0|A function which is called for every token. See [extensibility](/#/USING_PRO.md) for more details.|
 |xhtml       |`boolean` |`false`  |v0.3.2   |If true, emit self-closing HTML tags for void elements (<br/>, <img/>, etc.) with a "/" as required by XHTML.|
 
 <h2 id="highlight">Asynchronous highlighting</h2>

docs/USING_PRO.md

@@ -10,6 +10,8 @@ The `renderer` and `tokenizer` options can be an object with functions that will
 
 The `renderer` and `tokenizer` functions can return false to fallback to the previous function.
 
+The `walkTokens` option can be a function that will be called with every token before rendering. When calling `use` multiple times with different `walkTokens` functions each function will be called in the **reverse** order in which they were assigned.
+
 All other options will overwrite previously set options.
 
 <h2 id="renderer">The renderer</h2>
@@ -188,6 +190,35 @@ smartypants('"this ... string"')
 // "“this … string”"
 ```
 
+<h2 id="walk-tokens">Walk Tokens</h2>
+
+The walkTokens function gets called with every token. Child tokens are called before moving on to sibling tokens. Each token is passed by reference so updates are persisted when passed to the parser. The return value of the function is ignored.
+
+**Example:** Overriding heading tokens to start at h2.
+
+```js
+const marked = require('marked');
+
+// Override function
+const walkTokens = (token) => {
+  if (token.type === 'heading') {
+    token.depth += 1;
+  }
+};
+
+marked.use({ walkTokens });
+
+// Run marked
+console.log(marked('# heading 2\n\n## heading 3'));
+```
+
+**Output:**
+
+```html
+<h2 id="heading-2">heading 2</h2>
+<h3 id="heading-3">heading 3</h3>
+```
+
 <h2 id="lexer">The lexer</h2>
 
 The lexer takes a markdown string and calls the tokenizer functions.

docs/index.html

@@ -157,6 +157,7 @@ <h1>Marked.js Documentation</h1>
                             <li><a href="#/USING_PRO.md#use">marked.use()</a></li>
                             <li><a href="#/USING_PRO.md#renderer">Renderer</a></li>
                             <li><a href="#/USING_PRO.md#tokenizer">Tokenizer</a></li>
+                            <li><a href="#/USING_PRO.md#walk-tokens">Walk Tokens</a></li>
                             <li><a href="#/USING_PRO.md#lexer">Lexer</a></li>
                             <li><a href="#/USING_PRO.md#parser">Parser</a></li>
                         </ul>

src/Tokenizer.js

@@ -269,6 +269,7 @@ module.exports = class Tokenizer {
         }
 
         list.items.push({
+          type: 'list_item',
           raw,
           task: istask,
           checked: ischecked,

src/defaults.js

@@ -16,6 +16,7 @@ function getDefaults() {
     smartLists: false,
     smartypants: false,
     tokenizer: null,
+    walkTokens: null,
     xhtml: false
   };
 }

src/marked.js

@@ -28,39 +28,33 @@ function marked(src, opt, callback) {
       + Object.prototype.toString.call(src) + ', string expected');
   }
 
-  if (callback || typeof opt === 'function') {
-    if (!callback) {
-      callback = opt;
-      opt = null;
-    }
+  if (typeof opt === 'function') {
+    callback = opt;
+    opt = null;
+  }
 
-    opt = merge({}, marked.defaults, opt || {});
-    checkSanitizeDeprecation(opt);
+  opt = merge({}, marked.defaults, opt || {});
+  checkSanitizeDeprecation(opt);
+
+  if (callback) {
     const highlight = opt.highlight;
-    let tokens,
-      pending,
-      i = 0;
+    let tokens;
 
     try {
       tokens = Lexer.lex(src, opt);
     } catch (e) {
       return callback(e);
     }
 
-    pending = tokens.length;
-
     const done = function(err) {
-      if (err) {
-        opt.highlight = highlight;
-        return callback(err);
-      }
-
       let out;
 
-      try {
-        out = Parser.parse(tokens, opt);
-      } catch (e) {
-        err = e;
+      if (!err) {
+        try {
+          out = Parser.parse(tokens, opt);
+        } catch (e) {
+          err = e;
+        }
       }
 
       opt.highlight = highlight;
@@ -76,34 +70,45 @@ function marked(src, opt, callback) {
 
     delete opt.highlight;
 
-    if (!pending) return done();
+    if (!tokens.length) return done();
 
-    for (; i < tokens.length; i++) {
-      (function(token) {
-        if (token.type !== 'code') {
-          return --pending || done();
-        }
-        return highlight(token.text, token.lang, function(err, code) {
-          if (err) return done(err);
-          if (code == null || code === token.text) {
-            return --pending || done();
+    let pending = 0;
+    marked.walkTokens(tokens, function(token) {
+      if (token.type === 'code') {
+        pending++;
+        highlight(token.text, token.lang, function(err, code) {
+          if (err) {
+            return done(err);
+          }
+          if (code != null && code !== token.text) {
+            token.text = code;
+            token.escaped = true;
+          }
+
+          pending--;
+          if (pending === 0) {
+            done();
           }
-          token.text = code;
-          token.escaped = true;
-          --pending || done();
         });
-      })(tokens[i]);
+      }
+    });
+
+    if (pending === 0) {
+      done();
     }
 
     return;
   }
+
   try {
-    opt = merge({}, marked.defaults, opt || {});
-    checkSanitizeDeprecation(opt);
-    return Parser.parse(Lexer.lex(src, opt), opt);
+    const tokens = Lexer.lex(src, opt);
+    if (opt.walkTokens) {
+      marked.walkTokens(tokens, opt.walkTokens);
+    }
+    return Parser.parse(tokens, opt);
   } catch (e) {
     e.message += '\nPlease report this to https://github.com/markedjs/marked.';
-    if ((opt || marked.defaults).silent) {
+    if (opt.silent) {
       return '<p>An error occurred:</p><pre>'
         + escape(e.message + '', true)
         + '</pre>';
@@ -161,9 +166,50 @@ marked.use = function(extension) {
     }
     opts.tokenizer = tokenizer;
   }
+  if (extension.walkTokens) {
+    const walkTokens = marked.defaults.walkTokens;
+    opts.walkTokens = (token) => {
+      extension.walkTokens(token);
+      if (walkTokens) {
+        walkTokens(token);
+      }
+    };
+  }
   marked.setOptions(opts);
 };
 
+/**
+ * Run callback for every token
+ */
+
+marked.walkTokens = function(tokens, callback) {
+  for (const token of tokens) {
+    callback(token);
+    switch (token.type) {
+      case 'table': {
+        for (const cell of token.tokens.header) {
+          marked.walkTokens(cell, callback);
+        }
+        for (const row of token.tokens.cells) {
+          for (const cell of row) {
+            marked.walkTokens(cell, callback);
+          }
+        }
+        break;
+      }
+      case 'list': {
+        marked.walkTokens(token.items, callback);
+        break;
+      }
+      default: {
+        if (token.tokens) {
+          marked.walkTokens(token.tokens, callback);
+        }
+      }
+    }
+  }
+};
+
 /**
  * Expose
  */

test/unit/Lexer-spec.js

@@ -307,6 +307,7 @@ a | b
             loose: false,
             items: [
               {
+                type: 'list_item',
                 raw: '- item 1',
                 task: false,
                 checked: undefined,
@@ -320,6 +321,7 @@ a | b
                 }]
               },
               {
+                type: 'list_item',
                 raw: '- item 2\n',
                 task: false,
                 checked: undefined,