fix: prevent heading and def tokens from greedily capturing multiple newlines (#3925)

Author: Puxhkar

src/Tokenizer.ts

@@ -126,7 +126,7 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
 
       return {
         type: 'heading',
-        raw: cap[0],
+        raw: rtrim(cap[0], '\n'),
         depth: cap[1].length,
         text,
         tokens: this.lexer.inline(text),
@@ -501,7 +501,7 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
       return {
         type: 'def',
         tag,
-        raw: cap[0],
+        raw: rtrim(cap[0], '\n'),
         href,
         title,
       };
@@ -577,7 +577,7 @@ export class _Tokenizer<ParserOutput = string, RendererOutput = string> {
       const text = cap[1].trim();
       return {
         type: 'heading',
-        raw: cap[0],
+        raw: rtrim(cap[0], '\n'),
         depth: cap[2].charAt(0) === '=' ? 1 : 2,
         text,
         tokens: this.lexer.inline(text),

test/unit/Lexer.test.js

@@ -107,53 +107,81 @@ lheading 2
           },
           {
             type: 'heading',
-            raw: '# heading 1\n\n',
+            raw: '# heading 1',
             depth: 1,
             text: 'heading 1',
             tokens: [{ type: 'text', raw: 'heading 1', text: 'heading 1', escaped: false }],
           },
+          {
+            type: 'space',
+            raw: '\n\n',
+          },
           {
             type: 'heading',
-            raw: '## heading 2\n\n',
+            raw: '## heading 2',
             depth: 2,
             text: 'heading 2',
             tokens: [{ type: 'text', raw: 'heading 2', text: 'heading 2', escaped: false }],
           },
+          {
+            type: 'space',
+            raw: '\n\n',
+          },
           {
             type: 'heading',
-            raw: '### heading 3\n\n',
+            raw: '### heading 3',
             depth: 3,
             text: 'heading 3',
             tokens: [{ type: 'text', raw: 'heading 3', text: 'heading 3', escaped: false }],
           },
+          {
+            type: 'space',
+            raw: '\n\n',
+          },
           {
             type: 'heading',
-            raw: '#### heading 4\n\n',
+            raw: '#### heading 4',
             depth: 4,
             text: 'heading 4',
             tokens: [{ type: 'text', raw: 'heading 4', text: 'heading 4', escaped: false }],
           },
+          {
+            type: 'space',
+            raw: '\n\n',
+          },
           {
             type: 'heading',
-            raw: '##### heading 5\n\n',
+            raw: '##### heading 5',
             depth: 5,
             text: 'heading 5',
             tokens: [{ type: 'text', raw: 'heading 5', text: 'heading 5', escaped: false }],
           },
+          {
+            type: 'space',
+            raw: '\n\n',
+          },
           {
             type: 'heading',
-            raw: '###### heading 6\n\n',
+            raw: '###### heading 6',
             depth: 6,
             text: 'heading 6',
             tokens: [{ type: 'text', raw: 'heading 6', text: 'heading 6', escaped: false }],
           },
+          {
+            type: 'space',
+            raw: '\n\n',
+          },
           {
             type: 'heading',
-            raw: 'lheading 1\n==========\n\n',
+            raw: 'lheading 1\n==========',
             depth: 1,
             text: 'lheading 1',
             tokens: [{ type: 'text', raw: 'lheading 1', text: 'lheading 1', escaped: false }],
           },
+          {
+            type: 'space',
+            raw: '\n\n',
+          },
           {
             type: 'heading',
             raw: 'lheading 2\n----------\n',

test/unit/marked.test.js

@@ -1006,6 +1006,7 @@ br
         ['space', ''],
         ['heading', '# heading'],
         ['text', 'heading'],
+        ['space', ''],
         ['code', '```code```'],
         ['space', ''],
         ['table', '| a | b ||---|---|| 1 | 2 || 3 | 4 |'],