diff --git a/docs/USING_PRO.md b/docs/USING_PRO.md index 40aadea661..9b6bd7ac7c 100644 --- a/docs/USING_PRO.md +++ b/docs/USING_PRO.md @@ -155,7 +155,7 @@ console.log(marked('$ latex code $\n\n` other code `')); - table(*string* src) - lheading(*string* src) - paragraph(*string* src) -- text(*string* src) +- text(*string* src, *array* tokens) ### Inline level tokenizer methods diff --git a/src/Lexer.js b/src/Lexer.js index 577f40f966..b961706ee7 100644 --- a/src/Lexer.js +++ b/src/Lexer.js @@ -112,7 +112,7 @@ module.exports = class Lexer { */ blockTokens(src, tokens = [], top = true) { src = src.replace(/^ +$/gm, ''); - let token, i, l; + let token, i, l, lastToken; while (src) { // newline @@ -127,7 +127,13 @@ module.exports = class Lexer { // code if (token = this.tokenizer.code(src, tokens)) { src = src.substring(token.raw.length); - tokens.push(token); + if (token.type) { + tokens.push(token); + } else { + lastToken = tokens[tokens.length - 1]; + lastToken.raw += '\n' + token.raw; + lastToken.text += '\n' + token.text; + } continue; } @@ -219,9 +225,15 @@ module.exports = class Lexer { } // text - if (token = this.tokenizer.text(src)) { + if (token = this.tokenizer.text(src, tokens)) { src = src.substring(token.raw.length); - tokens.push(token); + if (token.type) { + tokens.push(token); + } else { + lastToken = tokens[tokens.length - 1]; + lastToken.raw += '\n' + token.raw; + lastToken.text += '\n' + token.text; + } continue; } diff --git a/src/Tokenizer.js b/src/Tokenizer.js index 38f1328bc9..512b45dc50 100644 --- a/src/Tokenizer.js +++ b/src/Tokenizer.js @@ -84,21 +84,21 @@ module.exports = class Tokenizer { const lastToken = tokens[tokens.length - 1]; // An indented code block cannot interrupt a paragraph. if (lastToken && lastToken.type === 'paragraph') { - tokens.pop(); - lastToken.text += '\n' + cap[0].trimRight(); - lastToken.raw += '\n' + cap[0]; - return lastToken; - } else { - const text = cap[0].replace(/^ {4}/gm, ''); return { - type: 'code', raw: cap[0], - codeBlockStyle: 'indented', - text: !this.options.pedantic - ? rtrim(text, '\n') - : text + text: cap[0].trimRight() }; } + + const text = cap[0].replace(/^ {4}/gm, ''); + return { + type: 'code', + raw: cap[0], + codeBlockStyle: 'indented', + text: !this.options.pedantic + ? rtrim(text, '\n') + : text + }; } } @@ -374,9 +374,17 @@ module.exports = class Tokenizer { } } - text(src) { + text(src, tokens) { const cap = this.rules.block.text.exec(src); if (cap) { + const lastToken = tokens[tokens.length - 1]; + if (lastToken && lastToken.type === 'text') { + return { + raw: cap[0], + text: cap[0] + }; + } + return { type: 'text', raw: cap[0], @@ -504,10 +512,17 @@ module.exports = class Tokenizer { codespan(src) { const cap = this.rules.inline.code.exec(src); if (cap) { + let text = cap[2].replace(/\n/g, ' '); + const hasNonSpaceChars = /[^ ]/.test(text); + const hasSpaceCharsOnBothEnds = text.startsWith(' ') && text.endsWith(' '); + if (hasNonSpaceChars && hasSpaceCharsOnBothEnds) { + text = text.substring(1, text.length - 1); + } + text = escape(text, true); return { type: 'codespan', raw: cap[0], - text: escape(cap[2].trim(), true) + text }; } } diff --git a/test/specs/new/codespan_newline.html b/test/specs/new/codespan_newline.html new file mode 100644 index 0000000000..2ebfcdd713 --- /dev/null +++ b/test/specs/new/codespan_newline.html @@ -0,0 +1,5 @@ +

code code

+ + diff --git a/test/specs/new/codespan_newline.md b/test/specs/new/codespan_newline.md new file mode 100644 index 0000000000..4e5c7c13a7 --- /dev/null +++ b/test/specs/new/codespan_newline.md @@ -0,0 +1,5 @@ +`code +code` + +- `code +code` diff --git a/test/unit/Lexer-spec.js b/test/unit/Lexer-spec.js index d8a802061b..9b2b5d7612 100644 --- a/test/unit/Lexer-spec.js +++ b/test/unit/Lexer-spec.js @@ -752,12 +752,95 @@ a | b }); }); - it('code', () => { - expectInlineTokens({ - md: '`code`', - tokens: [ - { type: 'codespan', raw: '`code`', text: 'code' } - ] + describe('codespan', () => { + it('code', () => { + expectInlineTokens({ + md: '`code`', + tokens: [ + { type: 'codespan', raw: '`code`', text: 'code' } + ] + }); + }); + + it('only spaces not stripped', () => { + expectInlineTokens({ + md: '` `', + tokens: [ + { type: 'codespan', raw: '` `', text: ' ' } + ] + }); + }); + + it('beginning space only not stripped', () => { + expectInlineTokens({ + md: '` a`', + tokens: [ + { type: 'codespan', raw: '` a`', text: ' a' } + ] + }); + }); + + it('end space only not stripped', () => { + expectInlineTokens({ + md: '`a `', + tokens: [ + { type: 'codespan', raw: '`a `', text: 'a ' } + ] + }); + }); + + it('begin and end spaces are stripped', () => { + expectInlineTokens({ + md: '` a `', + tokens: [ + { type: 'codespan', raw: '` a `', text: 'a' } + ] + }); + }); + + it('begin and end newlines are stripped', () => { + expectInlineTokens({ + md: '`\na\n`', + tokens: [ + { type: 'codespan', raw: '`\na\n`', text: 'a' } + ] + }); + }); + + it('begin and end tabs are not stripped', () => { + expectInlineTokens({ + md: '`\ta\t`', + tokens: [ + { type: 'codespan', raw: '`\ta\t`', text: '\ta\t' } + ] + }); + }); + + it('begin and end newlines', () => { + expectInlineTokens({ + md: '`\na\n`', + tokens: [ + { type: 'codespan', raw: '`\na\n`', text: 'a' } + ] + }); + }); + + it('begin and end multiple spaces only one stripped', () => { + expectInlineTokens({ + md: '` a `', + tokens: [ + { type: 'codespan', raw: '` a `', text: ' a ' } + ] + }); + }); + + it('newline to space', () => { + expectInlineTokens({ + md: '`a\nb`', + tokens: [ + { type: 'codespan', raw: '`a\nb`', text: 'a b' } + ] + }); }); });