fix: remove space token from list items

markedjs · Feb 22, 2021 · a38564e · a38564e
1 parent 97278e4
commit a38564e
Show file tree

Hide file tree

Showing 7 changed files with 111 additions and 29 deletions.
diff --git a/lib/marked.esm.js b/lib/marked.esm.js
@@ -532,39 +532,51 @@ var Tokenizer_1 = class Tokenizer {
         addBack,
         loose,
         istask,
-        ischecked;
+        ischecked,
+        endMatch;
 
       let l = itemMatch.length;
       bcurr = this.rules.block.listItemStart.exec(itemMatch[0]);
       for (let i = 0; i < l; i++) {
         item = itemMatch[i];
         raw = item;
 
+        if (!this.options.pedantic) {
+          // Determine if current item contains the end of the list
+          endMatch = item.match(new RegExp('\\n\\s*\\n {0,' + (bcurr[0].length - 1) + '}\\S'));
+          if (endMatch) {
+            addBack = item.length - endMatch.index + itemMatch.slice(i + 1).join('\n').length;
+            list.raw = list.raw.substring(0, list.raw.length - addBack);
+
+            item = item.substring(0, endMatch.index);
+            raw = item;
+            l = i + 1;
+          }
+        }
+
         // Determine whether the next list item belongs here.
         // Backpedal if it does not belong in this list.
         if (i !== l - 1) {
           bnext = this.rules.block.listItemStart.exec(itemMatch[i + 1]);
           if (
             !this.options.pedantic
-              ? bnext[1].length > bcurr[0].length || bnext[1].length > 3
+              ? bnext[1].length >= bcurr[0].length || bnext[1].length > 3
               : bnext[1].length > bcurr[1].length
           ) {
-            // nested list
-            itemMatch.splice(i, 2, itemMatch[i] + '\n' + itemMatch[i + 1]);
+            // nested list or continuation
+            itemMatch.splice(i, 2, itemMatch[i] + (!this.options.pedantic && bnext[1].length < bcurr[0].length && !itemMatch[i].match(/\n$/) ? '' : '\n') + itemMatch[i + 1]);
             i--;
             l--;
             continue;
-          } else {
-            if (
-              // different bullet style
-              !this.options.pedantic || this.options.smartLists
-                ? bnext[2][bnext[2].length - 1] !== bull[bull.length - 1]
-                : isordered === (bnext[2].length === 1)
-            ) {
-              addBack = itemMatch.slice(i + 1).join('\n');
-              list.raw = list.raw.substring(0, list.raw.length - addBack.length);
-              i = l - 1;
-            }
+          } else if (
+            // different bullet style
+            !this.options.pedantic || this.options.smartLists
+              ? bnext[2][bnext[2].length - 1] !== bull[bull.length - 1]
+              : isordered === (bnext[2].length === 1)
+          ) {
+            addBack = itemMatch.slice(i + 1).join('\n').length;
+            list.raw = list.raw.substring(0, list.raw.length - addBack);
+            i = l - 1;
           }
           bcurr = bnext;
         }
@@ -583,12 +595,15 @@ var Tokenizer_1 = class Tokenizer {
             : item.replace(/^ {1,4}/gm, '');
         }
 
+        // trim item newlines at end
+        item = item.replace(/\n+$/, '');
+
         // Determine whether item is loose or not.
         // Use: /(^|\n)(?! )[^\n]+\n\n(?!\s*$)/
         // for discount behavior.
-        loose = next || /\n\n(?!\s*$)/.test(item);
+        loose = next || /\n\n(?!\s*$)/.test(raw);
         if (i !== l - 1) {
-          next = item.charAt(item.length - 1) === '\n';
+          next = raw.charAt(raw.length - 1) === '\n';
           if (!loose) loose = next;
         }
 
@@ -1072,7 +1087,7 @@ block.item = edit$1(block.item, 'gm')
   .replace(/bull/g, block.bullet)
   .getRegex();
 
-block.listItemStart = edit$1(/^( *)(bull)/)
+block.listItemStart = edit$1(/^( *)(bull) */)
   .replace('bull', block.bullet)
   .getRegex();
 

diff --git a/lib/marked.js b/lib/marked.js
@@ -647,7 +647,7 @@
             if (endMatch) {
               addBack = item.length - endMatch.index + itemMatch.slice(i + 1).join('\n').length;
               list.raw = list.raw.substring(0, list.raw.length - addBack);
-              item = item.substring(0, endMatch.index + 1);
+              item = item.substring(0, endMatch.index);
               raw = item;
               l = i + 1;
             }
@@ -683,15 +683,17 @@
           if (~item.indexOf('\n ')) {
             space -= item.length;
             item = !this.options.pedantic ? item.replace(new RegExp('^ {1,' + space + '}', 'gm'), '') : item.replace(/^ {1,4}/gm, '');
-          } // Determine whether item is loose or not.
+          } // trim item newlines at end
+
+
+          item = item.replace(/\n+$/, ''); // Determine whether item is loose or not.
           // Use: /(^|\n)(?! )[^\n]+\n\n(?!\s*$)/
           // for discount behavior.
 
-
-          loose = next || /\n\n(?!\s*$)/.test(item);
+          loose = next || /\n\n(?!\s*$)/.test(raw);
 
           if (i !== l - 1) {
-            next = item.charAt(item.length - 1) === '\n';
+            next = raw.charAt(raw.length - 1) === '\n';
             if (!loose) loose = next;
           }
 

diff --git a/marked.min.js b/marked.min.js
diff --git a/src/Tokenizer.js b/src/Tokenizer.js
@@ -236,7 +236,7 @@ module.exports = class Tokenizer {
             addBack = item.length - endMatch.index + itemMatch.slice(i + 1).join('\n').length;
             list.raw = list.raw.substring(0, list.raw.length - addBack);
 
-            item = item.substring(0, endMatch.index + 1);
+            item = item.substring(0, endMatch.index);
             raw = item;
             l = i + 1;
           }
@@ -283,12 +283,15 @@ module.exports = class Tokenizer {
             : item.replace(/^ {1,4}/gm, '');
         }
 
+        // trim item newlines at end
+        item = item.replace(/\n+$/, '');
+
         // Determine whether item is loose or not.
         // Use: /(^|\n)(?! )[^\n]+\n\n(?!\s*$)/
         // for discount behavior.
-        loose = next || /\n\n(?!\s*$)/.test(item);
+        loose = next || /\n\n(?!\s*$)/.test(raw);
         if (i !== l - 1) {
-          next = item.charAt(item.length - 1) === '\n';
+          next = raw.charAt(raw.length - 1) === '\n';
           if (!loose) loose = next;
         }
 

diff --git a/test.js b/test.js
@@ -0,0 +1,3 @@
+const marked = require("./");
+
+console.log(marked("`a\n\n b\n`"));
diff --git a/test/unit/Lexer-spec.js b/test/unit/Lexer-spec.js
@@ -326,7 +326,7 @@ a | b
                 task: false,
                 checked: undefined,
                 loose: false,
-                text: 'item 2\n',
+                text: 'item 2',
                 tokens: [{
                   type: 'text',
                   raw: 'item 2',
@@ -390,6 +390,66 @@ a | b
       });
     });
 
+    it('space after list', () => {
+      expectTokens({
+        md: `
+- item 1
+- item 2
+
+paragraph
+`,
+        tokens: [
+          {
+            type: 'list',
+            raw: '- item 1\n- item 2\n\n',
+            ordered: false,
+            start: '',
+            loose: false,
+            items: [
+              {
+                type: 'list_item',
+                raw: '- item 1',
+                task: false,
+                checked: undefined,
+                loose: false,
+                text: 'item 1',
+                tokens: [{
+                  type: 'text',
+                  raw: 'item 1',
+                  text: 'item 1',
+                  tokens: [{ type: 'text', raw: 'item 1', text: 'item 1' }]
+                }]
+              },
+              {
+                type: 'list_item',
+                raw: '- item 2\n\n',
+                task: false,
+                checked: undefined,
+                loose: false,
+                text: 'item 2',
+                tokens: [{
+                  type: 'text',
+                  raw: 'item 2',
+                  text: 'item 2',
+                  tokens: [{ type: 'text', raw: 'item 2', text: 'item 2' }]
+                }]
+              }
+            ]
+          },
+          {
+            type: 'paragraph',
+            raw: 'paragraph',
+            text: 'paragraph',
+            tokens: [{
+              type: 'text',
+              raw: 'paragraph',
+              text: 'paragraph'
+            }]
+          }
+        ]
+      });
+    });
+
     it('start', () => {
       expectTokens({
         md: `

diff --git a/test/unit/marked-spec.js b/test/unit/marked-spec.js
@@ -465,7 +465,6 @@ br
       ['list_item', '- list'],
       ['text', 'list'],
       ['text', 'list'],
-      ['space', ''],
       ['html', '<div>html</div>'],
       ['paragraph', '[link](https://example.com)'],
       ['link', '[link](https://example.com)'],