From 67a2343a1f9037d5e0bfeb6c9cc59304eb99adc7 Mon Sep 17 00:00:00 2001
From: Tony Brix <tony@brix.ninja>
Date: Fri, 3 Jun 2022 21:36:03 -0500
Subject: [PATCH] fix: return values from walkTokens

---
 lib/marked.cjs    |  99 +++++++++++++----------
 lib/marked.esm.js |  99 ++++++++++++-----------
 lib/marked.umd.js |  99 +++++++++++++----------
 src/Lexer.js      |   3 +-
 src/Tokenizer.js  |  51 +++++-------
 src/marked.js     |  45 +++++++----
 test/bench.js     | 196 +++++++++++++++++-----------------------------
 7 files changed, 288 insertions(+), 304 deletions(-)

diff --git a/lib/marked.cjs b/lib/marked.cjs
index bcbd9c67fe..c5ffa1011e 100644
--- a/lib/marked.cjs
+++ b/lib/marked.cjs
@@ -412,7 +412,7 @@ function outputLink(cap, link, raw, lexer) {
       href: href,
       title: title,
       text: text,
-      tokens: lexer.inlineTokens(text, [])
+      tokens: lexer.inlineTokens(text)
     };
     lexer.state.inLink = false;
     return token;
@@ -520,15 +520,13 @@ var Tokenizer = /*#__PURE__*/function () {
         }
       }
 
-      var token = {
+      return {
         type: 'heading',
         raw: cap[0],
         depth: cap[1].length,
         text: text,
-        tokens: []
+        tokens: this.lexer.inline(text)
       };
-      this.lexer.inline(token.text, token.tokens);
-      return token;
     }
   };
 
@@ -746,10 +744,10 @@ var Tokenizer = /*#__PURE__*/function () {
       };
 
       if (this.options.sanitize) {
+        var text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]);
         token.type = 'paragraph';
-        token.text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]);
-        token.tokens = [];
-        this.lexer.inline(token.text, token.tokens);
+        token.text = text;
+        token.tokens = this.lexer.inline(text);
       }
 
       return token;
@@ -819,8 +817,7 @@ var Tokenizer = /*#__PURE__*/function () {
         l = item.header.length;
 
         for (j = 0; j < l; j++) {
-          item.header[j].tokens = [];
-          this.lexer.inline(item.header[j].text, item.header[j].tokens);
+          item.header[j].tokens = this.lexer.inline(item.header[j].text);
         } // cell child tokens
 
 
@@ -830,8 +827,7 @@ var Tokenizer = /*#__PURE__*/function () {
           row = item.rows[j];
 
           for (k = 0; k < row.length; k++) {
-            row[k].tokens = [];
-            this.lexer.inline(row[k].text, row[k].tokens);
+            row[k].tokens = this.lexer.inline(row[k].text);
           }
         }
 
@@ -844,15 +840,13 @@ var Tokenizer = /*#__PURE__*/function () {
     var cap = this.rules.block.lheading.exec(src);
 
     if (cap) {
-      var token = {
+      return {
         type: 'heading',
         raw: cap[0],
         depth: cap[2].charAt(0) === '=' ? 1 : 2,
         text: cap[1],
-        tokens: []
+        tokens: this.lexer.inline(cap[1])
       };
-      this.lexer.inline(token.text, token.tokens);
-      return token;
     }
   };
 
@@ -860,14 +854,13 @@ var Tokenizer = /*#__PURE__*/function () {
     var cap = this.rules.block.paragraph.exec(src);
 
     if (cap) {
-      var token = {
+      var text = cap[1].charAt(cap[1].length - 1) === '\n' ? cap[1].slice(0, -1) : cap[1];
+      return {
         type: 'paragraph',
         raw: cap[0],
-        text: cap[1].charAt(cap[1].length - 1) === '\n' ? cap[1].slice(0, -1) : cap[1],
-        tokens: []
+        text: text,
+        tokens: this.lexer.inline(text)
       };
-      this.lexer.inline(token.text, token.tokens);
-      return token;
     }
   };
 
@@ -875,14 +868,12 @@ var Tokenizer = /*#__PURE__*/function () {
     var cap = this.rules.block.text.exec(src);
 
     if (cap) {
-      var token = {
+      return {
         type: 'text',
         raw: cap[0],
         text: cap[0],
-        tokens: []
+        tokens: this.lexer.inline(cap[0])
       };
-      this.lexer.inline(token.text, token.tokens);
-      return token;
     }
   };
 
@@ -1061,7 +1052,7 @@ var Tokenizer = /*#__PURE__*/function () {
             type: 'em',
             raw: src.slice(0, lLength + match.index + rLength + 1),
             text: _text,
-            tokens: this.lexer.inlineTokens(_text, [])
+            tokens: this.lexer.inlineTokens(_text)
           };
         } // Create 'strong' if smallest delimiter has even char count. **a***
 
@@ -1071,7 +1062,7 @@ var Tokenizer = /*#__PURE__*/function () {
           type: 'strong',
           raw: src.slice(0, lLength + match.index + rLength + 1),
           text: text,
-          tokens: this.lexer.inlineTokens(text, [])
+          tokens: this.lexer.inlineTokens(text)
         };
       }
     }
@@ -1117,7 +1108,7 @@ var Tokenizer = /*#__PURE__*/function () {
         type: 'del',
         raw: cap[0],
         text: cap[2],
-        tokens: this.lexer.inlineTokens(cap[2], [])
+        tokens: this.lexer.inlineTokens(cap[2])
       };
     }
   };
@@ -1735,10 +1726,15 @@ var Lexer = /*#__PURE__*/function () {
   };
 
   _proto.inline = function inline(src, tokens) {
+    if (tokens === void 0) {
+      tokens = [];
+    }
+
     this.inlineQueue.push({
       src: src,
       tokens: tokens
     });
+    return tokens;
   }
   /**
    * Lexing/Compiling
@@ -2711,22 +2707,32 @@ function marked(src, opt, callback) {
     return;
   }
 
+  function onError(e) {
+    e.message += '\nPlease report this to https://github.com/markedjs/marked.';
+
+    if (opt.silent) {
+      return '<p>An error occurred:</p><pre>' + escape(e.message + '', true) + '</pre>';
+    }
+
+    throw e;
+  }
+
   try {
     var _tokens = Lexer.lex(src, opt);
 
     if (opt.walkTokens) {
+      if (opt.async) {
+        return Promise.all(marked.walkTokens(_tokens, opt.walkTokens)).then(function () {
+          return Parser.parse(_tokens, opt);
+        })["catch"](onError);
+      }
+
       marked.walkTokens(_tokens, opt.walkTokens);
     }
 
     return Parser.parse(_tokens, opt);
   } catch (e) {
-    e.message += '\nPlease report this to https://github.com/markedjs/marked.';
-
-    if (opt.silent) {
-      return '<p>An error occurred:</p><pre>' + escape(e.message + '', true) + '</pre>';
-    }
-
-    throw e;
+    onError(e);
   }
 }
 /**
@@ -2892,11 +2898,14 @@ marked.use = function () {
       var _walkTokens = marked.defaults.walkTokens;
 
       opts.walkTokens = function (token) {
-        pack.walkTokens.call(this, token);
+        var values = [];
+        values.concat(pack.walkTokens.call(this, token));
 
         if (_walkTokens) {
-          _walkTokens.call(this, token);
+          values.concat(_walkTokens.call(this, token));
         }
+
+        return values;
       };
     }
 
@@ -2913,16 +2922,18 @@ marked.use = function () {
 
 
 marked.walkTokens = function (tokens, callback) {
+  var values = [];
+
   var _loop3 = function _loop3() {
     var token = _step.value;
-    callback.call(marked, token);
+    values.concat(callback.call(marked, token));
 
     switch (token.type) {
       case 'table':
         {
           for (var _iterator2 = _createForOfIteratorHelperLoose(token.header), _step2; !(_step2 = _iterator2()).done;) {
             var cell = _step2.value;
-            marked.walkTokens(cell.tokens, callback);
+            values.concat(marked.walkTokens(cell.tokens, callback));
           }
 
           for (var _iterator3 = _createForOfIteratorHelperLoose(token.rows), _step3; !(_step3 = _iterator3()).done;) {
@@ -2930,7 +2941,7 @@ marked.walkTokens = function (tokens, callback) {
 
             for (var _iterator4 = _createForOfIteratorHelperLoose(row), _step4; !(_step4 = _iterator4()).done;) {
               var _cell = _step4.value;
-              marked.walkTokens(_cell.tokens, callback);
+              values.concat(marked.walkTokens(_cell.tokens, callback));
             }
           }
 
@@ -2939,7 +2950,7 @@ marked.walkTokens = function (tokens, callback) {
 
       case 'list':
         {
-          marked.walkTokens(token.items, callback);
+          values.concat(marked.walkTokens(token.items, callback));
           break;
         }
 
@@ -2948,10 +2959,10 @@ marked.walkTokens = function (tokens, callback) {
           if (marked.defaults.extensions && marked.defaults.extensions.childTokens && marked.defaults.extensions.childTokens[token.type]) {
             // Walk any extensions
             marked.defaults.extensions.childTokens[token.type].forEach(function (childTokens) {
-              marked.walkTokens(token[childTokens], callback);
+              values.concat(marked.walkTokens(token[childTokens], callback));
             });
           } else if (token.tokens) {
-            marked.walkTokens(token.tokens, callback);
+            values.concat(marked.walkTokens(token.tokens, callback));
           }
         }
     }
@@ -2960,6 +2971,8 @@ marked.walkTokens = function (tokens, callback) {
   for (var _iterator = _createForOfIteratorHelperLoose(tokens), _step; !(_step = _iterator()).done;) {
     _loop3();
   }
+
+  return values;
 };
 /**
  * Parse Inline
diff --git a/lib/marked.esm.js b/lib/marked.esm.js
index 21074efd07..9b06967ffd 100644
--- a/lib/marked.esm.js
+++ b/lib/marked.esm.js
@@ -329,7 +329,7 @@ function outputLink(cap, link, raw, lexer) {
       href,
       title,
       text,
-      tokens: lexer.inlineTokens(text, [])
+      tokens: lexer.inlineTokens(text)
     };
     lexer.state.inLink = false;
     return token;
@@ -435,15 +435,13 @@ class Tokenizer {
         }
       }
 
-      const token = {
+      return {
         type: 'heading',
         raw: cap[0],
         depth: cap[1].length,
         text,
-        tokens: []
+        tokens: this.lexer.inline(text)
       };
-      this.lexer.inline(token.text, token.tokens);
-      return token;
     }
   }
 
@@ -653,10 +651,10 @@ class Tokenizer {
         text: cap[0]
       };
       if (this.options.sanitize) {
+        const text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]);
         token.type = 'paragraph';
-        token.text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]);
-        token.tokens = [];
-        this.lexer.inline(token.text, token.tokens);
+        token.text = text;
+        token.tokens = this.lexer.inline(text);
       }
       return token;
     }
@@ -714,8 +712,7 @@ class Tokenizer {
         // header child tokens
         l = item.header.length;
         for (j = 0; j < l; j++) {
-          item.header[j].tokens = [];
-          this.lexer.inline(item.header[j].text, item.header[j].tokens);
+          item.header[j].tokens = this.lexer.inline(item.header[j].text);
         }
 
         // cell child tokens
@@ -723,8 +720,7 @@ class Tokenizer {
         for (j = 0; j < l; j++) {
           row = item.rows[j];
           for (k = 0; k < row.length; k++) {
-            row[k].tokens = [];
-            this.lexer.inline(row[k].text, row[k].tokens);
+            row[k].tokens = this.lexer.inline(row[k].text);
           }
         }
 
@@ -736,45 +732,40 @@ class Tokenizer {
   lheading(src) {
     const cap = this.rules.block.lheading.exec(src);
     if (cap) {
-      const token = {
+      return {
         type: 'heading',
         raw: cap[0],
         depth: cap[2].charAt(0) === '=' ? 1 : 2,
         text: cap[1],
-        tokens: []
+        tokens: this.lexer.inline(cap[1])
       };
-      this.lexer.inline(token.text, token.tokens);
-      return token;
     }
   }
 
   paragraph(src) {
     const cap = this.rules.block.paragraph.exec(src);
     if (cap) {
-      const token = {
+      const text = cap[1].charAt(cap[1].length - 1) === '\n'
+        ? cap[1].slice(0, -1)
+        : cap[1];
+      return {
         type: 'paragraph',
         raw: cap[0],
-        text: cap[1].charAt(cap[1].length - 1) === '\n'
-          ? cap[1].slice(0, -1)
-          : cap[1],
-        tokens: []
+        text,
+        tokens: this.lexer.inline(text)
       };
-      this.lexer.inline(token.text, token.tokens);
-      return token;
     }
   }
 
   text(src) {
     const cap = this.rules.block.text.exec(src);
     if (cap) {
-      const token = {
+      return {
         type: 'text',
         raw: cap[0],
         text: cap[0],
-        tokens: []
+        tokens: this.lexer.inline(cap[0])
       };
-      this.lexer.inline(token.text, token.tokens);
-      return token;
     }
   }
 
@@ -943,7 +934,7 @@ class Tokenizer {
             type: 'em',
             raw: src.slice(0, lLength + match.index + rLength + 1),
             text,
-            tokens: this.lexer.inlineTokens(text, [])
+            tokens: this.lexer.inlineTokens(text)
           };
         }
 
@@ -953,7 +944,7 @@ class Tokenizer {
           type: 'strong',
           raw: src.slice(0, lLength + match.index + rLength + 1),
           text,
-          tokens: this.lexer.inlineTokens(text, [])
+          tokens: this.lexer.inlineTokens(text)
         };
       }
     }
@@ -994,7 +985,7 @@ class Tokenizer {
         type: 'del',
         raw: cap[0],
         text: cap[2],
-        tokens: this.lexer.inlineTokens(cap[2], [])
+        tokens: this.lexer.inlineTokens(cap[2])
       };
     }
   }
@@ -1692,8 +1683,9 @@ class Lexer {
     return tokens;
   }
 
-  inline(src, tokens) {
+  inline(src, tokens = []) {
     this.inlineQueue.push({ src, tokens });
+    return tokens;
   }
 
   /**
@@ -2542,13 +2534,7 @@ function marked(src, opt, callback) {
     return;
   }
 
-  try {
-    const tokens = Lexer.lex(src, opt);
-    if (opt.walkTokens) {
-      marked.walkTokens(tokens, opt.walkTokens);
-    }
-    return Parser.parse(tokens, opt);
-  } catch (e) {
+  function onError(e) {
     e.message += '\nPlease report this to https://github.com/markedjs/marked.';
     if (opt.silent) {
       return '<p>An error occurred:</p><pre>'
@@ -2557,6 +2543,23 @@ function marked(src, opt, callback) {
     }
     throw e;
   }
+
+  try {
+    const tokens = Lexer.lex(src, opt);
+    if (opt.walkTokens) {
+      if (opt.async) {
+        return Promise.all(marked.walkTokens(tokens, opt.walkTokens))
+          .then(() => {
+            return Parser.parse(tokens, opt);
+          })
+          .catch(onError);
+      }
+      marked.walkTokens(tokens, opt.walkTokens);
+    }
+    return Parser.parse(tokens, opt);
+  } catch (e) {
+    onError(e);
+  }
 }
 
 /**
@@ -2673,10 +2676,12 @@ marked.use = function(...args) {
     if (pack.walkTokens) {
       const walkTokens = marked.defaults.walkTokens;
       opts.walkTokens = function(token) {
-        pack.walkTokens.call(this, token);
+        const values = [];
+        values.concat(pack.walkTokens.call(this, token));
         if (walkTokens) {
-          walkTokens.call(this, token);
+          values.concat(walkTokens.call(this, token));
         }
+        return values;
       };
     }
 
@@ -2693,35 +2698,37 @@ marked.use = function(...args) {
  */
 
 marked.walkTokens = function(tokens, callback) {
+  const values = [];
   for (const token of tokens) {
-    callback.call(marked, token);
+    values.concat(callback.call(marked, token));
     switch (token.type) {
       case 'table': {
         for (const cell of token.header) {
-          marked.walkTokens(cell.tokens, callback);
+          values.concat(marked.walkTokens(cell.tokens, callback));
         }
         for (const row of token.rows) {
           for (const cell of row) {
-            marked.walkTokens(cell.tokens, callback);
+            values.concat(marked.walkTokens(cell.tokens, callback));
           }
         }
         break;
       }
       case 'list': {
-        marked.walkTokens(token.items, callback);
+        values.concat(marked.walkTokens(token.items, callback));
         break;
       }
       default: {
         if (marked.defaults.extensions && marked.defaults.extensions.childTokens && marked.defaults.extensions.childTokens[token.type]) { // Walk any extensions
           marked.defaults.extensions.childTokens[token.type].forEach(function(childTokens) {
-            marked.walkTokens(token[childTokens], callback);
+            values.concat(marked.walkTokens(token[childTokens], callback));
           });
         } else if (token.tokens) {
-          marked.walkTokens(token.tokens, callback);
+          values.concat(marked.walkTokens(token.tokens, callback));
         }
       }
     }
   }
+  return values;
 };
 
 /**
diff --git a/lib/marked.umd.js b/lib/marked.umd.js
index 0d974a7c6a..d812bfe1a1 100644
--- a/lib/marked.umd.js
+++ b/lib/marked.umd.js
@@ -414,7 +414,7 @@
         href: href,
         title: title,
         text: text,
-        tokens: lexer.inlineTokens(text, [])
+        tokens: lexer.inlineTokens(text)
       };
       lexer.state.inLink = false;
       return token;
@@ -522,15 +522,13 @@
           }
         }
 
-        var token = {
+        return {
           type: 'heading',
           raw: cap[0],
           depth: cap[1].length,
           text: text,
-          tokens: []
+          tokens: this.lexer.inline(text)
         };
-        this.lexer.inline(token.text, token.tokens);
-        return token;
       }
     };
 
@@ -748,10 +746,10 @@
         };
 
         if (this.options.sanitize) {
+          var text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]);
           token.type = 'paragraph';
-          token.text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]);
-          token.tokens = [];
-          this.lexer.inline(token.text, token.tokens);
+          token.text = text;
+          token.tokens = this.lexer.inline(text);
         }
 
         return token;
@@ -821,8 +819,7 @@
           l = item.header.length;
 
           for (j = 0; j < l; j++) {
-            item.header[j].tokens = [];
-            this.lexer.inline(item.header[j].text, item.header[j].tokens);
+            item.header[j].tokens = this.lexer.inline(item.header[j].text);
           } // cell child tokens
 
 
@@ -832,8 +829,7 @@
             row = item.rows[j];
 
             for (k = 0; k < row.length; k++) {
-              row[k].tokens = [];
-              this.lexer.inline(row[k].text, row[k].tokens);
+              row[k].tokens = this.lexer.inline(row[k].text);
             }
           }
 
@@ -846,15 +842,13 @@
       var cap = this.rules.block.lheading.exec(src);
 
       if (cap) {
-        var token = {
+        return {
           type: 'heading',
           raw: cap[0],
           depth: cap[2].charAt(0) === '=' ? 1 : 2,
           text: cap[1],
-          tokens: []
+          tokens: this.lexer.inline(cap[1])
         };
-        this.lexer.inline(token.text, token.tokens);
-        return token;
       }
     };
 
@@ -862,14 +856,13 @@
       var cap = this.rules.block.paragraph.exec(src);
 
       if (cap) {
-        var token = {
+        var text = cap[1].charAt(cap[1].length - 1) === '\n' ? cap[1].slice(0, -1) : cap[1];
+        return {
           type: 'paragraph',
           raw: cap[0],
-          text: cap[1].charAt(cap[1].length - 1) === '\n' ? cap[1].slice(0, -1) : cap[1],
-          tokens: []
+          text: text,
+          tokens: this.lexer.inline(text)
         };
-        this.lexer.inline(token.text, token.tokens);
-        return token;
       }
     };
 
@@ -877,14 +870,12 @@
       var cap = this.rules.block.text.exec(src);
 
       if (cap) {
-        var token = {
+        return {
           type: 'text',
           raw: cap[0],
           text: cap[0],
-          tokens: []
+          tokens: this.lexer.inline(cap[0])
         };
-        this.lexer.inline(token.text, token.tokens);
-        return token;
       }
     };
 
@@ -1063,7 +1054,7 @@
               type: 'em',
               raw: src.slice(0, lLength + match.index + rLength + 1),
               text: _text,
-              tokens: this.lexer.inlineTokens(_text, [])
+              tokens: this.lexer.inlineTokens(_text)
             };
           } // Create 'strong' if smallest delimiter has even char count. **a***
 
@@ -1073,7 +1064,7 @@
             type: 'strong',
             raw: src.slice(0, lLength + match.index + rLength + 1),
             text: text,
-            tokens: this.lexer.inlineTokens(text, [])
+            tokens: this.lexer.inlineTokens(text)
           };
         }
       }
@@ -1119,7 +1110,7 @@
           type: 'del',
           raw: cap[0],
           text: cap[2],
-          tokens: this.lexer.inlineTokens(cap[2], [])
+          tokens: this.lexer.inlineTokens(cap[2])
         };
       }
     };
@@ -1737,10 +1728,15 @@
     };
 
     _proto.inline = function inline(src, tokens) {
+      if (tokens === void 0) {
+        tokens = [];
+      }
+
       this.inlineQueue.push({
         src: src,
         tokens: tokens
       });
+      return tokens;
     }
     /**
      * Lexing/Compiling
@@ -2713,22 +2709,32 @@
       return;
     }
 
+    function onError(e) {
+      e.message += '\nPlease report this to https://github.com/markedjs/marked.';
+
+      if (opt.silent) {
+        return '<p>An error occurred:</p><pre>' + escape(e.message + '', true) + '</pre>';
+      }
+
+      throw e;
+    }
+
     try {
       var _tokens = Lexer.lex(src, opt);
 
       if (opt.walkTokens) {
+        if (opt.async) {
+          return Promise.all(marked.walkTokens(_tokens, opt.walkTokens)).then(function () {
+            return Parser.parse(_tokens, opt);
+          })["catch"](onError);
+        }
+
         marked.walkTokens(_tokens, opt.walkTokens);
       }
 
       return Parser.parse(_tokens, opt);
     } catch (e) {
-      e.message += '\nPlease report this to https://github.com/markedjs/marked.';
-
-      if (opt.silent) {
-        return '<p>An error occurred:</p><pre>' + escape(e.message + '', true) + '</pre>';
-      }
-
-      throw e;
+      onError(e);
     }
   }
   /**
@@ -2894,11 +2900,14 @@
         var _walkTokens = marked.defaults.walkTokens;
 
         opts.walkTokens = function (token) {
-          pack.walkTokens.call(this, token);
+          var values = [];
+          values.concat(pack.walkTokens.call(this, token));
 
           if (_walkTokens) {
-            _walkTokens.call(this, token);
+            values.concat(_walkTokens.call(this, token));
           }
+
+          return values;
         };
       }
 
@@ -2915,16 +2924,18 @@
 
 
   marked.walkTokens = function (tokens, callback) {
+    var values = [];
+
     var _loop3 = function _loop3() {
       var token = _step.value;
-      callback.call(marked, token);
+      values.concat(callback.call(marked, token));
 
       switch (token.type) {
         case 'table':
           {
             for (var _iterator2 = _createForOfIteratorHelperLoose(token.header), _step2; !(_step2 = _iterator2()).done;) {
               var cell = _step2.value;
-              marked.walkTokens(cell.tokens, callback);
+              values.concat(marked.walkTokens(cell.tokens, callback));
             }
 
             for (var _iterator3 = _createForOfIteratorHelperLoose(token.rows), _step3; !(_step3 = _iterator3()).done;) {
@@ -2932,7 +2943,7 @@
 
               for (var _iterator4 = _createForOfIteratorHelperLoose(row), _step4; !(_step4 = _iterator4()).done;) {
                 var _cell = _step4.value;
-                marked.walkTokens(_cell.tokens, callback);
+                values.concat(marked.walkTokens(_cell.tokens, callback));
               }
             }
 
@@ -2941,7 +2952,7 @@
 
         case 'list':
           {
-            marked.walkTokens(token.items, callback);
+            values.concat(marked.walkTokens(token.items, callback));
             break;
           }
 
@@ -2950,10 +2961,10 @@
             if (marked.defaults.extensions && marked.defaults.extensions.childTokens && marked.defaults.extensions.childTokens[token.type]) {
               // Walk any extensions
               marked.defaults.extensions.childTokens[token.type].forEach(function (childTokens) {
-                marked.walkTokens(token[childTokens], callback);
+                values.concat(marked.walkTokens(token[childTokens], callback));
               });
             } else if (token.tokens) {
-              marked.walkTokens(token.tokens, callback);
+              values.concat(marked.walkTokens(token.tokens, callback));
             }
           }
       }
@@ -2962,6 +2973,8 @@
     for (var _iterator = _createForOfIteratorHelperLoose(tokens), _step; !(_step = _iterator()).done;) {
       _loop3();
     }
+
+    return values;
   };
   /**
    * Parse Inline
diff --git a/src/Lexer.js b/src/Lexer.js
index 3c0a2c7e01..c4bbf41a83 100644
--- a/src/Lexer.js
+++ b/src/Lexer.js
@@ -316,8 +316,9 @@ export class Lexer {
     return tokens;
   }
 
-  inline(src, tokens) {
+  inline(src, tokens = []) {
     this.inlineQueue.push({ src, tokens });
+    return tokens;
   }
 
   /**
diff --git a/src/Tokenizer.js b/src/Tokenizer.js
index e8a69b6759..17f650e5a6 100644
--- a/src/Tokenizer.js
+++ b/src/Tokenizer.js
@@ -19,7 +19,7 @@ function outputLink(cap, link, raw, lexer) {
       href,
       title,
       text,
-      tokens: lexer.inlineTokens(text, [])
+      tokens: lexer.inlineTokens(text)
     };
     lexer.state.inLink = false;
     return token;
@@ -125,15 +125,13 @@ export class Tokenizer {
         }
       }
 
-      const token = {
+      return {
         type: 'heading',
         raw: cap[0],
         depth: cap[1].length,
         text,
-        tokens: []
+        tokens: this.lexer.inline(text)
       };
-      this.lexer.inline(token.text, token.tokens);
-      return token;
     }
   }
 
@@ -343,10 +341,10 @@ export class Tokenizer {
         text: cap[0]
       };
       if (this.options.sanitize) {
+        const text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]);
         token.type = 'paragraph';
-        token.text = this.options.sanitizer ? this.options.sanitizer(cap[0]) : escape(cap[0]);
-        token.tokens = [];
-        this.lexer.inline(token.text, token.tokens);
+        token.text = text;
+        token.tokens = this.lexer.inline(text);
       }
       return token;
     }
@@ -404,8 +402,7 @@ export class Tokenizer {
         // header child tokens
         l = item.header.length;
         for (j = 0; j < l; j++) {
-          item.header[j].tokens = [];
-          this.lexer.inline(item.header[j].text, item.header[j].tokens);
+          item.header[j].tokens = this.lexer.inline(item.header[j].text);
         }
 
         // cell child tokens
@@ -413,8 +410,7 @@ export class Tokenizer {
         for (j = 0; j < l; j++) {
           row = item.rows[j];
           for (k = 0; k < row.length; k++) {
-            row[k].tokens = [];
-            this.lexer.inline(row[k].text, row[k].tokens);
+            row[k].tokens = this.lexer.inline(row[k].text);
           }
         }
 
@@ -426,45 +422,40 @@ export class Tokenizer {
   lheading(src) {
     const cap = this.rules.block.lheading.exec(src);
     if (cap) {
-      const token = {
+      return {
         type: 'heading',
         raw: cap[0],
         depth: cap[2].charAt(0) === '=' ? 1 : 2,
         text: cap[1],
-        tokens: []
+        tokens: this.lexer.inline(cap[1])
       };
-      this.lexer.inline(token.text, token.tokens);
-      return token;
     }
   }
 
   paragraph(src) {
     const cap = this.rules.block.paragraph.exec(src);
     if (cap) {
-      const token = {
+      const text = cap[1].charAt(cap[1].length - 1) === '\n'
+        ? cap[1].slice(0, -1)
+        : cap[1];
+      return {
         type: 'paragraph',
         raw: cap[0],
-        text: cap[1].charAt(cap[1].length - 1) === '\n'
-          ? cap[1].slice(0, -1)
-          : cap[1],
-        tokens: []
+        text,
+        tokens: this.lexer.inline(text)
       };
-      this.lexer.inline(token.text, token.tokens);
-      return token;
     }
   }
 
   text(src) {
     const cap = this.rules.block.text.exec(src);
     if (cap) {
-      const token = {
+      return {
         type: 'text',
         raw: cap[0],
         text: cap[0],
-        tokens: []
+        tokens: this.lexer.inline(cap[0])
       };
-      this.lexer.inline(token.text, token.tokens);
-      return token;
     }
   }
 
@@ -633,7 +624,7 @@ export class Tokenizer {
             type: 'em',
             raw: src.slice(0, lLength + match.index + rLength + 1),
             text,
-            tokens: this.lexer.inlineTokens(text, [])
+            tokens: this.lexer.inlineTokens(text)
           };
         }
 
@@ -643,7 +634,7 @@ export class Tokenizer {
           type: 'strong',
           raw: src.slice(0, lLength + match.index + rLength + 1),
           text,
-          tokens: this.lexer.inlineTokens(text, [])
+          tokens: this.lexer.inlineTokens(text)
         };
       }
     }
@@ -684,7 +675,7 @@ export class Tokenizer {
         type: 'del',
         raw: cap[0],
         text: cap[2],
-        tokens: this.lexer.inlineTokens(cap[2], [])
+        tokens: this.lexer.inlineTokens(cap[2])
       };
     }
   }
diff --git a/src/marked.js b/src/marked.js
index 10f543336c..f91d30459f 100644
--- a/src/marked.js
+++ b/src/marked.js
@@ -105,13 +105,7 @@ export function marked(src, opt, callback) {
     return;
   }
 
-  try {
-    const tokens = Lexer.lex(src, opt);
-    if (opt.walkTokens) {
-      marked.walkTokens(tokens, opt.walkTokens);
-    }
-    return Parser.parse(tokens, opt);
-  } catch (e) {
+  function onError(e) {
     e.message += '\nPlease report this to https://github.com/markedjs/marked.';
     if (opt.silent) {
       return '<p>An error occurred:</p><pre>'
@@ -120,6 +114,23 @@ export function marked(src, opt, callback) {
     }
     throw e;
   }
+
+  try {
+    const tokens = Lexer.lex(src, opt);
+    if (opt.walkTokens) {
+      if (opt.async) {
+        return Promise.all(marked.walkTokens(tokens, opt.walkTokens))
+          .then(() => {
+            return Parser.parse(tokens, opt);
+          })
+          .catch(onError);
+      }
+      marked.walkTokens(tokens, opt.walkTokens);
+    }
+    return Parser.parse(tokens, opt);
+  } catch (e) {
+    onError(e);
+  }
 }
 
 /**
@@ -236,10 +247,12 @@ marked.use = function(...args) {
     if (pack.walkTokens) {
       const walkTokens = marked.defaults.walkTokens;
       opts.walkTokens = function(token) {
-        pack.walkTokens.call(this, token);
+        const values = [];
+        values.concat(pack.walkTokens.call(this, token));
         if (walkTokens) {
-          walkTokens.call(this, token);
+          values.concat(walkTokens.call(this, token));
         }
+        return values;
       };
     }
 
@@ -256,35 +269,37 @@ marked.use = function(...args) {
  */
 
 marked.walkTokens = function(tokens, callback) {
+  const values = [];
   for (const token of tokens) {
-    callback.call(marked, token);
+    values.concat(callback.call(marked, token));
     switch (token.type) {
       case 'table': {
         for (const cell of token.header) {
-          marked.walkTokens(cell.tokens, callback);
+          values.concat(marked.walkTokens(cell.tokens, callback));
         }
         for (const row of token.rows) {
           for (const cell of row) {
-            marked.walkTokens(cell.tokens, callback);
+            values.concat(marked.walkTokens(cell.tokens, callback));
           }
         }
         break;
       }
       case 'list': {
-        marked.walkTokens(token.items, callback);
+        values.concat(marked.walkTokens(token.items, callback));
         break;
       }
       default: {
         if (marked.defaults.extensions && marked.defaults.extensions.childTokens && marked.defaults.extensions.childTokens[token.type]) { // Walk any extensions
           marked.defaults.extensions.childTokens[token.type].forEach(function(childTokens) {
-            marked.walkTokens(token[childTokens], callback);
+            values.concat(marked.walkTokens(token[childTokens], callback));
           });
         } else if (token.tokens) {
-          marked.walkTokens(token.tokens, callback);
+          values.concat(marked.walkTokens(token.tokens, callback));
         }
       }
     }
   }
+  return values;
 };
 
 /**
diff --git a/test/bench.js b/test/bench.js
index 7afd24f0e6..7b3d9e71b2 100644
--- a/test/bench.js
+++ b/test/bench.js
@@ -3,6 +3,7 @@ import { fileURLToPath } from 'url';
 import { isEqual } from './helpers/html-differ.js';
 import { loadFiles } from './helpers/load.js';
 
+import { marked as cjsMarked } from '../lib/marked.cjs';
 import { marked as esmMarked } from '../lib/marked.esm.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -30,9 +31,10 @@ export function load() {
 export async function runBench(options) {
   options = options || {};
   const specs = load();
+  const tests = {};
 
   // Non-GFM, Non-pedantic
-  marked.setOptions({
+  cjsMarked.setOptions({
     gfm: false,
     breaks: false,
     pedantic: false,
@@ -40,9 +42,9 @@ export async function runBench(options) {
     smartLists: false
   });
   if (options.marked) {
-    marked.setOptions(options.marked);
+    cjsMarked.setOptions(options.marked);
   }
-  await bench('cjs marked', specs, marked.parse);
+  tests['cjs marked'] = cjsMarked.parse;
 
   esmMarked.setOptions({
     gfm: false,
@@ -54,113 +56,76 @@ export async function runBench(options) {
   if (options.marked) {
     esmMarked.setOptions(options.marked);
   }
-  await bench('esm marked', specs, esmMarked.parse);
+  tests['esm marked'] = esmMarked.parse;
 
-  // GFM
-  marked.setOptions({
-    gfm: true,
-    breaks: false,
-    pedantic: false,
-    sanitize: false,
-    smartLists: false
-  });
-  if (options.marked) {
-    marked.setOptions(options.marked);
-  }
-  await bench('cjs marked (gfm)', specs, marked.parse);
-
-  esmMarked.setOptions({
-    gfm: true,
-    breaks: false,
-    pedantic: false,
-    sanitize: false,
-    smartLists: false
-  });
-  if (options.marked) {
-    esmMarked.setOptions(options.marked);
-  }
-  await bench('esm marked (gfm)', specs, esmMarked.parse);
-
-  // Pedantic
-  marked.setOptions({
-    gfm: false,
-    breaks: false,
-    pedantic: true,
-    sanitize: false,
-    smartLists: false
-  });
-  if (options.marked) {
-    marked.setOptions(options.marked);
-  }
-  await bench('cjs marked (pedantic)', specs, marked.parse);
-
-  esmMarked.setOptions({
-    gfm: false,
-    breaks: false,
-    pedantic: true,
-    sanitize: false,
-    smartLists: false
-  });
-  if (options.marked) {
-    esmMarked.setOptions(options.marked);
-  }
-  await bench('esm marked (pedantic)', specs, esmMarked.parse);
+  // esmMarked.setOptions({
+  //   gfm: true,
+  //   breaks: false,
+  //   pedantic: false,
+  //   sanitize: false,
+  //   smartLists: false
+  // });
+  // if (options.marked) {
+  //   esmMarked.setOptions(options.marked);
+  // }
+  // tests['esm marked (gfm)'] = esmMarked.parse;
 
   try {
-    await bench('commonmark', specs, (await (async() => {
+    tests.commonmark = (await (async() => {
       const { Parser, HtmlRenderer } = await import('commonmark');
       const parser = new Parser();
       const writer = new HtmlRenderer();
       return function(text) {
         return writer.render(parser.parse(text));
       };
-    })()));
+    })());
   } catch (e) {
     console.error('Could not bench commonmark. (Error: %s)', e.message);
   }
 
   try {
-    await bench('markdown-it', specs, (await (async() => {
+    tests['markdown-it'] = (await (async() => {
       const MarkdownIt = (await import('markdown-it')).default;
       const md = new MarkdownIt();
       return md.render.bind(md);
-    })()));
+    })());
   } catch (e) {
     console.error('Could not bench markdown-it. (Error: %s)', e.message);
   }
+
+  await bench(tests, specs);
 }
 
-export async function bench(name, specs, engine) {
-  const before = process.hrtime();
-  for (let i = 0; i < 1e3; i++) {
-    for (const spec of specs) {
-      await engine(spec.markdown);
+export async function bench(tests, specs) {
+  const stats = {};
+  for (const name in tests) {
+    stats[name] = {
+      elapsed: 0n,
+      correct: 0
+    };
+  }
+
+  console.log();
+  for (let i = 0; i < specs.length; i++) {
+    const spec = specs[i];
+    process.stdout.write(`${(i * 100 / specs.length).toFixed(1).padStart(5)}% ${i.toString().padStart(specs.length.toString().length)} of ${specs.length}\r`);
+    for (const name in tests) {
+      const test = tests[name];
+      const before = process.hrtime.bigint();
+      for (let n = 0; n < 1e3; n++) {
+        await test(spec.markdown);
+      }
+      const after = process.hrtime.bigint();
+      stats[name].elapsed += after - before;
+      stats[name].correct += (await isEqual(spec.html, await test(spec.markdown)) ? 1 : 0);
     }
   }
-  const elapsed = process.hrtime(before);
-  const ms = prettyElapsedTime(elapsed).toFixed();
 
-  let correct = 0;
-  for (const spec of specs) {
-    if (await isEqual(spec.html, await engine(spec.markdown))) {
-      correct++;
-    }
+  for (const name in tests) {
+    const ms = prettyElapsedTime(stats[name].elapsed);
+    const percent = (stats[name].correct / specs.length * 100).toFixed(2);
+    console.log(`${name} completed in ${ms}ms and passed ${percent}%`);
   }
-  const percent = (correct / specs.length * 100).toFixed(2);
-
-  console.log('%s completed in %sms and passed %s%', name, ms, percent);
-}
-
-/**
- * A simple one-time benchmark
- */
-export async function time(options) {
-  options = options || {};
-  const specs = load();
-  if (options.marked) {
-    marked.setOptions(options.marked);
-  }
-  await bench('marked', specs, marked);
 }
 
 /**
@@ -204,35 +169,23 @@ function parseArg(argv) {
 
   while (argv.length) {
     const arg = getarg();
-    switch (arg) {
-      case '-t':
-      case '--time':
-        options.time = true;
-        break;
-      case '-m':
-      case '--minified':
-        options.minified = true;
-        break;
-      default:
-        if (arg.indexOf('--') === 0) {
-          const opt = camelize(arg.replace(/^--(no-)?/, ''));
-          if (!defaults.hasOwnProperty(opt)) {
-            continue;
-          }
-          options.marked = options.marked || {};
-          if (arg.indexOf('--no-') === 0) {
-            options.marked[opt] = typeof defaults[opt] !== 'boolean'
-              ? null
-              : false;
-          } else {
-            options.marked[opt] = typeof defaults[opt] !== 'boolean'
-              ? argv.shift()
-              : true;
-          }
-        } else {
-          orphans.push(arg);
-        }
-        break;
+    if (arg.indexOf('--') === 0) {
+      const opt = camelize(arg.replace(/^--(no-)?/, ''));
+      if (!defaults.hasOwnProperty(opt)) {
+        continue;
+      }
+      options.marked = options.marked || {};
+      if (arg.indexOf('--no-') === 0) {
+        options.marked[opt] = typeof defaults[opt] !== 'boolean'
+          ? null
+          : false;
+      } else {
+        options.marked[opt] = typeof defaults[opt] !== 'boolean'
+          ? argv.shift()
+          : true;
+      }
+    } else {
+      orphans.push(arg);
     }
   }
 
@@ -257,28 +210,19 @@ function camelize(text) {
  * Main
  */
 export default async function main(argv) {
-  marked = (await import('../lib/marked.cjs')).marked;
+  marked = cjsMarked;
 
   const opt = parseArg(argv);
 
-  if (opt.minified) {
-    marked = (await import('../marked.min.js')).marked;
-  }
-
-  if (opt.time) {
-    await time(opt);
-  } else {
-    await runBench(opt);
-  }
+  await runBench(opt);
 }
 
 /**
  * returns time to millisecond granularity
+ * @param hrtimeElapsed {bigint}
  */
 function prettyElapsedTime(hrtimeElapsed) {
-  const seconds = hrtimeElapsed[0];
-  const frac = Math.round(hrtimeElapsed[1] / 1e3) / 1e3;
-  return seconds * 1e3 + frac;
+  return Number(hrtimeElapsed / 1_000_000n);
 }
 
 process.title = 'marked bench';