From 79e76e3874dbde97d2d3e593ea4b709496787a18 Mon Sep 17 00:00:00 2001 From: Adam Dangoor Date: Sat, 14 Nov 2026 00:50:03 +0000 Subject: [PATCH] =?UTF-8?q?=E2=9C=A8=20NEW:=20Allow=20directive=20parsing?= =?UTF-8?q?=20without=20a=20directive=20schema?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 7 ++ docs/develop/architecture.md | 16 ++++ myst_parser/parsers/directives.py | 63 +++++++++++--- tests/test_renderers/test_parse_directives.py | 82 +++++++++++++++++++ 4 files changed, 157 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d79c7010..c98b04e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## Unreleased + +- Allow `parse_directive_text(None, first_line, content)` to extract arbitrary + string options and body offsets without defining a docutils directive class. + Type the result's option values as strings in this mode, and allow arbitrary + converter result values when a directive class or full YAML parsing is used. + ## 5.2.0 - 2026-11-10 Heading anchors are now stable and documented: diff --git a/docs/develop/architecture.md b/docs/develop/architecture.md index b1ef1503..3ed4ac09 100644 --- a/docs/develop/architecture.md +++ b/docs/develop/architecture.md @@ -25,3 +25,19 @@ system. This parser does the following: * Parse markdown files with the markdown-it parser, including MyST specific plugins * Convert these files into docutils objects using the MyST docutils renderer * Provide these to Sphinx in order to use in building your site. + +## Extracting directive text + +Tools that inspect directive source without rendering it can use +`myst_parser.parsers.directives.parse_directive_text(None, "", content)`. +Passing `None` instead of a docutils directive class extracts arbitrary option +names as strings, including empty flags, without applying a directive's option +converters. Both colon-prefixed options and YAML-style option blocks are supported. +Syntax errors still produce warnings with source locations. + +The result includes `options`, normalized `body` lines and `body_offset`. +For tools that need exact source whitespace, use `body_offset` to locate the +body in the original content rather than joining the normalized lines. +The empty second argument means there is no body text on the directive's opening +line. If supplied, that text is included in the body, as with a directive that +takes no arguments. diff --git a/myst_parser/parsers/directives.py b/myst_parser/parsers/directives.py index e72057dc..816e5011 100644 --- a/myst_parser/parsers/directives.py +++ b/myst_parser/parsers/directives.py @@ -39,7 +39,7 @@ from collections.abc import Callable from dataclasses import dataclass from textwrap import dedent -from typing import Any +from typing import Any, Generic, Literal, TypeVar, overload import yaml from docutils.parsers.rst import Directive @@ -66,11 +66,14 @@ class ParseWarnings: type: MystWarnings = MystWarnings.DIRECTIVE_PARSING +_OptionValue = TypeVar("_OptionValue") + + @dataclass -class DirectiveParsingResult: +class DirectiveParsingResult(Generic[_OptionValue]): arguments: list[str] """The arguments parsed from the first line.""" - options: dict + options: dict[str, _OptionValue] """Options parsed from the YAML block.""" body: list[str] """The lines of body content""" @@ -91,17 +94,47 @@ class DirectiveParsingResult: """ +@overload +def parse_directive_text( + directive_class: None, + first_line: str, + content: str, + *, + line: int | None = None, + validate_options: Literal[True] = True, + additional_options: dict[str, str] | None = None, +) -> DirectiveParsingResult[str]: ... + + +@overload +def parse_directive_text( + directive_class: type[Directive] | None, + first_line: str, + content: str, + *, + line: int | None = None, + validate_options: bool = True, + additional_options: dict[str, str] | None = None, +) -> DirectiveParsingResult[Any]: ... + + def parse_directive_text( - directive_class: type[Directive], + directive_class: type[Directive] | None, first_line: str, content: str, *, line: int | None = None, validate_options: bool = True, additional_options: dict[str, str] | None = None, -) -> DirectiveParsingResult: +) -> DirectiveParsingResult[Any]: """Parse (and validate) the full directive text. + :param directive_class: The directive class defining arguments and options. + Pass ``None`` to extract arbitrary string options without a directive + schema. In this mode, text on the opening line is body content, as for + a directive with no arguments, and body content is always allowed. + With ``validate_options=False``, full YAML values are returned instead, + as when a directive class is supplied. :param first_line: The text on the same line as the directive name. May be an argument or body text, dependent on the directive :param content: All text after the first line. Can include options. @@ -119,7 +152,7 @@ def parse_directive_text( content_offset: int has_options_block: bool - if directive_class.option_spec: + if directive_class is None or directive_class.option_spec: # only look for an option block if there are possible options # body, options, option_errors = _parse_directive_options( result = _parse_directive_options( @@ -145,7 +178,9 @@ def parse_directive_text( body_lines = content.splitlines() content_offset = 0 - if not (directive_class.required_arguments or directive_class.optional_arguments): + if directive_class is None or not ( + directive_class.required_arguments or directive_class.optional_arguments + ): # If there are no possible arguments, then the body can start on the argument line if first_line.strip(): if has_options_block and any(body_lines): @@ -171,7 +206,7 @@ def parse_directive_text( content_offset += 1 # check for body content - if body_lines and not directive_class.has_content: + if directive_class is not None and body_lines and not directive_class.has_content: parse_warnings.append(ParseWarnings("Has content, but none permitted")) return DirectiveParsingResult( @@ -212,7 +247,7 @@ def is_option_line(line: str) -> bool: def _parse_directive_options( content: str, - directive_class: type[Directive], + directive_class: type[Directive] | None, as_yaml: bool, line: int | None, additional_options: dict[str, str] | None = None, @@ -221,7 +256,8 @@ def _parse_directive_options( :param content: All text after the directive's opening line, possibly starting with an option block - :param directive_class: The directive class to validate options against + :param directive_class: The directive class to validate options against, + or None to retain arbitrary string options without validation :param as_yaml: Whether to parse the options block with the full YAML spec, rather than validating against the directive's option specification (used by myst-nb) @@ -358,7 +394,7 @@ def _option_line(name: str) -> int | None: return None return options_position + option_lines[name] - if issubclass(directive_class, TestDirective): + if directive_class is not None and issubclass(directive_class, TestDirective): # technically this directive spec only accepts one option ('option') # but since its for testing only we accept all options return _DirectiveOptions( @@ -369,6 +405,11 @@ def _option_line(name: str) -> int | None: # The options block takes priority over additional options options = {**additional_options, **options} + if directive_class is None: + return _DirectiveOptions( + content, options, validation_errors, has_options_block, option_line_count + ) + # check options against spec options_spec: dict[str, Callable] = directive_class.option_spec new_options: dict[str, Any] = {} diff --git a/tests/test_renderers/test_parse_directives.py b/tests/test_renderers/test_parse_directives.py index 3c543577..f07b2a7f 100644 --- a/tests/test_renderers/test_parse_directives.py +++ b/tests/test_renderers/test_parse_directives.py @@ -20,6 +20,88 @@ FIXTURE_PATH = Path(__file__).parent.joinpath("fixtures") +@pytest.mark.parametrize( + "content,expected_options,expected_body,expected_offset", + [ + ("body\n", {}, ["body"], 0), + ( + ':ctx: {"name": "World"}\n:custom: true\n:flag:\n\nbody\n\n\n', + {"ctx": '{"name": "World"}', "custom": "true", "flag": ""}, + ["body", ""], + 4, + ), + ( + '---\nctx: {"name": "World"}\ncustom: true\nflag:\n---\n\nbody\n\n\n', + {"ctx": '{"name": "World"}', "custom": "true", "flag": ""}, + ["body", ""], + 6, + ), + ( + "---\ncustom: |\n first\n second\n---\n\nbody\n", + {"custom": "first\nsecond"}, + ["body"], + 6, + ), + ( + ":custom: value\n:::{note}\nbody\n:::\n", + {"custom": "value"}, + [":::{note}", "body", ":::"], + 1, + ), + (":flag:\n\n", {"flag": ""}, [], 1), + ], +) +def test_parsing_without_directive_class( + content: str, + expected_options: dict[str, str], + expected_body: list[str], + expected_offset: int, +) -> None: + """Tools can extract string options without defining a directive.""" + result = parse_directive_text(None, "", content) + assert result.options == expected_options + assert result.body == expected_body + assert result.body_offset == expected_offset + assert result.arguments == [] + assert not result.warnings + + +def test_parsing_without_directive_class_additional_options() -> None: + result = parse_directive_text( + None, + "", + ":custom: authored\n\nbody", + additional_options={"custom": "default", "other": "injected"}, + ) + assert result.options == {"custom": "authored", "other": "injected"} + assert not result.warnings + + +def test_parsing_without_directive_class_first_line() -> None: + result = parse_directive_text(None, "body on opening line", "next line") + assert result.body == ["body on opening line", "next line"] + assert result.body_offset == -1 + assert not result.warnings + + +def test_parsing_without_directive_class_reports_invalid_options() -> None: + result = parse_directive_text(None, "", '---\ncustom: "unclosed\n---\nbody', line=5) + assert result.options == {} + assert result.body == ["body"] + assert len(result.warnings) == 1 + assert result.warnings[0].lineno == 7 + + +def test_parsing_without_directive_class_full_yaml() -> None: + """The existing full-YAML mode remains available without a schema.""" + result = parse_directive_text( + None, "", "---\ncustom: [1]\n---\nbody", validate_options=False + ) + assert result.options == {"custom": [1]} + assert result.body == ["body"] + assert not result.warnings + + @pytest.mark.param_file(FIXTURE_PATH / "option_parsing.yaml", "yaml") def test_option_parsing(file_params): """Test parsing of directive options."""