🐛 FIX: build-aborting crashes on hostile input, and docutils 0.23 compat

Three defects where malformed or unlucky input aborted the whole build
instead of degrading gracefully:

- Deeply nested HTML: rendering the parsed HTML AST recurses once per
  nesting level, and the resulting RecursionError escaped the
  html_to_nodes try/except (which only covered tokenization), killing
  the build. Now caught, warning as myst.html_parse and falling back to
  the raw HTML.

- Hostile YAML front matter: only ParserError/ScannerError were caught,
  so a ConstructorError (e.g. `title: !Ref X`), ComposerError, or the
  bare ValueError PyYAML leaks for out-of-range timestamps
  (`2021-99-99`) crashed the parse. Both read_topmatter and
  render_front_matter now catch yaml.YAMLError + ValueError and emit
  the intended myst.topmatter warning.

- A footnote whose label matches any heading or target name was
  silently dropped: the duplicate-definition check tested against
  document.nameids, which holds every target name, not just footnote
  labels. It now checks against the names of previously rendered
  footnotes only.

Also docutils 0.23 support (the only failure was test-side, docutils
now attaches info-level system messages to the doctree when
report_level allows):

- normalize them out of the tree in test_link_resolution (they are
  still asserted via the captured report stream)
- extend the pin to docutils<0.24 and add 0.23 to the myst-docutils CI
  test matrix

Author: chrisjsewell

.github/workflows/tests.yml

@@ -74,7 +74,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        docutils-version: ["0.20", "0.21", "0.22"]
+        docutils-version: ["0.20", "0.21", "0.22", "0.23"]
 
     steps:
     - name: Checkout source

myst_parser/config/main.py

@@ -609,7 +609,9 @@ def read_topmatter(text: str | Iterator[str]) -> dict[str, Any] | None:
         top_matter.append(line.rstrip() + "\n")
     try:
         metadata = yaml.safe_load("".join(top_matter))
-    except (yaml.parser.ParserError, yaml.scanner.ScannerError) as err:
+    # ValueError: PyYAML lets it escape for some invalid scalars,
+    # e.g. out-of-range timestamps like `2021-99-99`
+    except (yaml.YAMLError, ValueError) as err:
         raise TopmatterReadError("Malformed YAML") from err
     if not isinstance(metadata, dict):
         raise TopmatterReadError(f"YAML is not a dict: {type(metadata)}")

myst_parser/mdit_to_docutils/base.py

@@ -1262,7 +1262,9 @@ def render_front_matter(self, token: SyntaxTreeNode) -> None:
         if isinstance(token.content, str):
             try:
                 data = yaml.safe_load(token.content)
-            except (yaml.parser.ParserError, yaml.scanner.ScannerError):
+            # ValueError: PyYAML lets it escape for some invalid scalars,
+            # e.g. out-of-range timestamps like `2021-99-99`
+            except (yaml.YAMLError, ValueError):
                 self.create_warning(
                     "Malformed YAML",
                     MystWarnings.MD_TOPMATTER,
@@ -1505,7 +1507,15 @@ def render_footnote_reference(self, token: SyntaxTreeNode) -> None:
         """Despite the name, this is actually a footnote definition, e.g. `[^a]: ...`"""
         target = token.meta["label"]
 
-        if target in self.document.nameids:
+        # check against existing footnote labels only, not `document.nameids`,
+        # which holds every target name (headings, `(name)=` targets, ...) and
+        # so would wrongly drop a footnote that merely shares a heading's name
+        existing_labels = {
+            name
+            for footnote in self.document.autofootnotes + self.document.footnotes
+            for name in footnote["names"]
+        }
+        if target in existing_labels:
             # note we chose to directly omit these footnotes in the parser,
             # rather than let docutils/sphinx handle them, since otherwise you end up with a confusing warning:
             # WARNING: Duplicate explicit target name: "x". [docutils]

myst_parser/mdit_to_docutils/html_to_nodes.py

@@ -7,7 +7,7 @@
 
 from docutils import nodes
 
-from myst_parser.parsers.parse_html import Data, tokenize_html
+from myst_parser.parsers.parse_html import Data, Element, tokenize_html
 from myst_parser.warnings_ import MystWarnings
 
 if TYPE_CHECKING:
@@ -81,6 +81,22 @@ def html_to_nodes(
     ):
         return default_html(text, renderer.document["source"], line_number)
 
+    try:
+        return _render_nodes(root, line_number, renderer)
+    except RecursionError:
+        msg_node = renderer.create_warning(
+            "HTML is too deeply nested to process",
+            MystWarnings.HTML_PARSE,
+            line=line_number,
+        )
+        return ([msg_node] if msg_node else []) + default_html(
+            text, renderer.document["source"], line_number
+        )
+
+
+def _render_nodes(
+    root: Element, line_number: int, renderer: DocutilsRenderer
+) -> list[nodes.Element]:
     nodes_list = []
     for child in root:
         if child.name == "img":

pyproject.toml

@@ -34,7 +34,7 @@ keywords = [
 ]
 requires-python = ">=3.11"
 dependencies = [
-    "docutils>=0.20,<0.23",
+    "docutils>=0.20,<0.24",
     "jinja2", # required for substitutions, but let sphinx choose version
     "markdown-it-py~=4.2",
     "mdit-py-plugins~=0.6,>=0.6.1",

tests/test_docutils.py

@@ -1,6 +1,7 @@
 import contextlib
 import importlib.util
 import io
+import sys
 from dataclasses import dataclass, field, fields
 from textwrap import dedent
 from typing import Literal
@@ -210,6 +211,89 @@ def test_linkify_no_warning_when_available():
     assert "[myst.linkify]" not in stream.getvalue()
 
 
+def test_html_deep_nesting_warns():
+    """Deeply nested HTML degrades to raw output with a warning.
+
+    Regression: rendering the HTML AST recurses once per nesting level,
+    and the resulting ``RecursionError`` escaped and aborted the build.
+    """
+    depth = sys.getrecursionlimit()
+    # tags on separate lines, to stay under docutils' line-length-limit
+    source = (
+        '<div class="admonition"><p class="title">Title</p>\n'
+        + "<div>\n" * depth
+        + "content\n"
+        + "</div>\n" * depth
+        + "</div>\n"
+    )
+    stream = io.StringIO()
+    doctree = publish_doctree(
+        source=source,
+        parser=Parser(),
+        settings_overrides={
+            "myst_enable_extensions": ["html_admonition"],
+            "warning_stream": stream,
+        },
+    )
+    assert "HTML is too deeply nested" in stream.getvalue()
+    assert "[myst.html]" in stream.getvalue()
+    # the original text is preserved as a raw HTML node
+    assert list(doctree.findall(nodes.raw))
+
+
+@pytest.mark.parametrize(
+    "yaml_line",
+    [
+        "title: !UnknownTag value",  # yaml.constructor.ConstructorError
+        "date: 2021-99-99",  # bare ValueError from an out-of-range timestamp
+    ],
+)
+def test_topmatter_hostile_yaml_warns(yaml_line):
+    """Front matter YAML errors beyond parser/scanner ones warn, not crash.
+
+    Regression: only ``ParserError``/``ScannerError`` were caught, so e.g. a
+    ``ConstructorError`` from an unknown tag aborted the whole build.
+    """
+    stream = io.StringIO()
+    doctree = publish_doctree(
+        source=f"---\n{yaml_line}\n---\n\ncontent\n",
+        parser=Parser(),
+        settings_overrides={"warning_stream": stream},
+    )
+    assert "[myst.topmatter]" in stream.getvalue()
+    assert "content" in doctree.pformat()
+
+
+def test_footnote_label_matching_heading_name():
+    """A footnote label sharing a heading's name is not a duplicate.
+
+    Regression: the duplicate check used ``document.nameids``, which holds
+    every target name, so the footnote definition was silently dropped.
+    """
+    stream = io.StringIO()
+    doctree = publish_doctree(
+        source="# Note\n\n[^note]\n\n[^note]: the definition\n",
+        parser=Parser(),
+        settings_overrides={"warning_stream": stream},
+    )
+    footnotes = list(doctree.findall(nodes.footnote))
+    assert footnotes, "expected the footnote definition to be kept"
+    assert "the definition" in footnotes[0].astext()
+    assert "Duplicate footnote" not in stream.getvalue()
+
+
+def test_footnote_duplicate_definition_warns():
+    """A genuinely duplicated footnote definition still warns and is dropped."""
+    stream = io.StringIO()
+    doctree = publish_doctree(
+        source="[^a]\n\n[^a]: first\n\n[^a]: second\n",
+        parser=Parser(),
+        settings_overrides={"warning_stream": stream},
+    )
+    assert "Duplicate footnote definition" in stream.getvalue()
+    assert len(list(doctree.findall(nodes.footnote))) == 1
+
+
 def test_definition_list_orphan_definition():
     """A definition with no preceding term errors, but keeps its content.
 

tests/test_renderers/test_fixtures_docutils.py

@@ -12,6 +12,7 @@
 
 import pytest
 from docutils import __version_info__ as docutils_version
+from docutils import nodes
 from docutils.core import Publisher, publish_doctree
 from pytest_param_files import ParamTestData
 
@@ -64,6 +65,12 @@ def test_link_resolution(file_params: ParamTestData, normalize_doctree_xml):
         parser=Parser(),
         settings_overrides=settings,
     )
+    # docutils >=0.23 also inserts info-level (severity 1) system messages
+    # into the doctree; they are still written to the report stream,
+    # which is what the fixtures capture, so drop them from the tree
+    for msg in list(doctree.findall(nodes.system_message)):
+        if msg["level"] < 2:
+            msg.parent.remove(msg)
     outcome = normalize_doctree_xml(doctree.pformat())
     if report_stream.getvalue().strip():
         outcome += "\n\n" + report_stream.getvalue().strip()