diff --git a/docs/syntax/optional.md b/docs/syntax/optional.md index ae054b51..efe0e759 100644 --- a/docs/syntax/optional.md +++ b/docs/syntax/optional.md @@ -64,6 +64,8 @@ Adding `"smartquotes"` to `myst_enable_extensions` (in the {{ confpy }}) will au - `'single quotes'`: 'single quotes' - `"double quotes"`: "double quotes" +Quote characters follow the configured document language (such as Sphinx's `language` option or Docutils' `--language`, `--smart-quotes`, and `--smartquote-locales` settings), falling back to English typographic quotes when unsupported. + Adding `"replacements"` to `myst_enable_extensions` (in the {{ confpy }}) will automatically convert some common typographic texts: text | converted diff --git a/myst_parser/parsers/docutils_.py b/myst_parser/parsers/docutils_.py index 91ae74e3..a89c8868 100644 --- a/myst_parser/parsers/docutils_.py +++ b/myst_parser/parsers/docutils_.py @@ -32,7 +32,11 @@ SortFootnotes, UnreferencedFootnotesDetector, ) -from myst_parser.parsers.mdit import create_md_parser, linkify_available +from myst_parser.parsers.mdit import ( + create_md_parser, + document_smartquotes, + linkify_available, +) from myst_parser.warnings_ import MystWarnings, create_warning @@ -329,7 +333,12 @@ def parse(self, inputstring: str, document: nodes.document) -> None: config = merge_file_level(config, topmatter, warning) # parse content - parser = create_md_parser(config, DocutilsRenderer) + quotes = ( + document_smartquotes(document) + if "smartquotes" in config.enable_extensions + else None + ) + parser = create_md_parser(config, DocutilsRenderer, quotes=quotes) parser.options["document"] = document parser.render(inputstring) diff --git a/myst_parser/parsers/mdit.py b/myst_parser/parsers/mdit.py index b248096b..0cc124e0 100644 --- a/myst_parser/parsers/mdit.py +++ b/myst_parser/parsers/mdit.py @@ -4,7 +4,8 @@ from __future__ import annotations -from collections.abc import Callable +from collections.abc import Callable, Iterable, Sequence +from typing import TYPE_CHECKING from markdown_it import MarkdownIt from markdown_it.renderer import RendererProtocol @@ -26,6 +27,90 @@ from myst_parser.config.main import MdParserConfig +if TYPE_CHECKING: + from docutils import nodes + +DEFAULT_SMARTQUOTES: tuple[str, str, str, str] = ( + "\u201c", + "\u201d", + "\u2018", + "\u2019", +) + + +def _coerce_quotes(quotes: Sequence[str]) -> tuple[str, str, str, str] | None: + """Validate and normalize a 4-item quote sequence into a 4-tuple of strings.""" + if len(quotes) != 4: + return None + return (quotes[0], quotes[1], quotes[2], quotes[3]) + + +def resolve_smartquotes( + language_code: str | None = None, + *, + smart_quotes: bool | str = False, + smartquotes_locales: Iterable[tuple[str, Sequence[str]]] | None = None, +) -> tuple[str, str, str, str]: + """Resolve localized smart quote characters for ``markdown-it-py``. + + Follows Docutils' ``SmartQuotes`` locale lookup (``smartchars.quotes``, + ``smartquotes_locales``, ``smart_quotes="alt"``, and BCP 47 subtag fallback + via ``normalize_language_tag``) without mutating global Docutils state. + Falls back to ``DEFAULT_SMARTQUOTES`` when ``language_code`` is unset or + has no matching quote definition. + + :param language_code: Document language tag (e.g. ``"fr"``, ``"de_DE"``). + :param smart_quotes: Docutils ``smart_quotes`` setting; strings starting + with ``"alt"`` select the ``-x-altquot`` variant. + :param smartquotes_locales: Optional ``(language_tag, quotes)`` overrides + from Docutils' ``smartquotes_locales`` setting. + :return: A 4-tuple ``(open_double, close_double, open_single, close_single)``. + """ + if not language_code or not language_code.strip(): + return DEFAULT_SMARTQUOTES + + from docutils.utils import normalize_language_tag + from docutils.utils.smartquotes import smartchars + + quotes_map: dict[str, Sequence[str]] = dict(smartchars.quotes) + if smartquotes_locales: + for locale_tag, locale_quotes in smartquotes_locales: + coerced = _coerce_quotes(locale_quotes) + if coerced is None or not locale_tag: + continue + normalized = normalize_language_tag(locale_tag) + if normalized and normalized[0]: + quotes_map[normalized[0]] = coerced + + lang = language_code.strip() + if isinstance(smart_quotes, str) and smart_quotes.startswith("alt"): + if "-x-altquot" in lang: + lang = lang.replace("-x-altquot", "") + else: + lang = f"{lang}-x-altquot" + + for tag in normalize_language_tag(lang): + if tag in quotes_map: + coerced = _coerce_quotes(quotes_map[tag]) + if coerced is not None: + return coerced + + return DEFAULT_SMARTQUOTES + + +def document_smartquotes(document: nodes.document, /) -> tuple[str, str, str, str]: + """Resolve localized smart quote characters from a Docutils document. + + :param document: The Docutils document whose runtime settings are read. + :return: A 4-tuple ``(open_double, close_double, open_single, close_single)``. + """ + settings = document.settings + return resolve_smartquotes( + getattr(settings, "language_code", None), + smart_quotes=getattr(settings, "smart_quotes", False), + smartquotes_locales=getattr(settings, "smartquotes_locales", None), + ) + def linkify_available() -> bool: """Return whether the optional ``linkify-it-py`` dependency is installed.""" @@ -36,9 +121,20 @@ def linkify_available() -> bool: def create_md_parser( - config: MdParserConfig, renderer: Callable[[MarkdownIt], RendererProtocol] + config: MdParserConfig, + renderer: Callable[[MarkdownIt], RendererProtocol], + *, + quotes: Sequence[str] | None = None, ) -> MarkdownIt: - """Return a Markdown parser with the required MyST configuration.""" + """Return a Markdown parser with the required MyST configuration. + + :param config: The MyST parser configuration. + :param renderer: Renderer factory passed to ``MarkdownIt``. + :param quotes: Optional 4-item sequence ``(open_double, close_double, + open_single, close_single)`` overriding ``markdown-it-py``'s default + typographic quote characters when ``smartquotes`` is enabled. + :return: Configured ``MarkdownIt`` parser instance. + """ if config.commonmark_only: # see https://spec.commonmark.org/ @@ -73,6 +169,11 @@ def create_md_parser( if "smartquotes" in config.enable_extensions: md.enable("smartquotes") typographer = True + if quotes is not None: + coerced_quotes = _coerce_quotes(quotes) + if coerced_quotes is None: + raise ValueError("`quotes` must contain exactly 4 quote strings.") + md.options["quotes"] = coerced_quotes if "replacements" in config.enable_extensions: md.enable("replacements") typographer = True diff --git a/myst_parser/parsers/sphinx_.py b/myst_parser/parsers/sphinx_.py index b2a83e41..17d00add 100644 --- a/myst_parser/parsers/sphinx_.py +++ b/myst_parser/parsers/sphinx_.py @@ -22,7 +22,7 @@ ResolveSectionRefs, SortFootnotes, ) -from myst_parser.parsers.mdit import create_md_parser +from myst_parser.parsers.mdit import create_md_parser, document_smartquotes from myst_parser.warnings_ import create_warning SPHINX_LOGGER = logging.getLogger(__name__) @@ -84,6 +84,11 @@ def parse(self, inputstring: str, document: nodes.document) -> None: ) config = merge_file_level(config, topmatter, warning) - parser = create_md_parser(config, SphinxRenderer) + quotes = ( + document_smartquotes(document) + if "smartquotes" in config.enable_extensions + else None + ) + parser = create_md_parser(config, SphinxRenderer, quotes=quotes) parser.options["document"] = document parser.render(inputstring) diff --git a/tests/test_docutils.py b/tests/test_docutils.py index cabb2c03..63c7140e 100644 --- a/tests/test_docutils.py +++ b/tests/test_docutils.py @@ -28,7 +28,11 @@ cli_xml, to_html5_demo, ) -from myst_parser.parsers.mdit import create_md_parser +from myst_parser.parsers.mdit import ( + DEFAULT_SMARTQUOTES, + create_md_parser, + resolve_smartquotes, +) def test_attr_to_optparse_option(): @@ -1218,3 +1222,149 @@ def test_math_inline_double_latex_no_blank_lines(): ) assert "\\begin{align*}" in output assert_no_blank_line_in_math_env(output) + + +@pytest.mark.parametrize( + ("language_code", "smart_quotes", "locales", "expected"), + [ + pytest.param(None, False, None, DEFAULT_SMARTQUOTES, id="unset"), + pytest.param("", False, None, DEFAULT_SMARTQUOTES, id="empty"), + pytest.param(" ", False, None, DEFAULT_SMARTQUOTES, id="whitespace"), + pytest.param("en", False, None, DEFAULT_SMARTQUOTES, id="english"), + pytest.param( + "fr", + False, + None, + ("\u00ab\u00a0", "\u00a0\u00bb", "\u201c", "\u201d"), + id="french", + ), + pytest.param( + "fr", + "alt", + None, + ("\u00ab\u202f", "\u202f\u00bb", "\u201c", "\u201d"), + id="french-alt", + ), + pytest.param( + "fr-x-altquot", + "alt", + None, + ("\u00ab\u00a0", "\u00a0\u00bb", "\u201c", "\u201d"), + id="french-alt-toggle", + ), + pytest.param( + "fr_CH", + False, + None, + ("\u00ab", "\u00bb", "\u2039", "\u203a"), + id="swiss-french", + ), + pytest.param( + "de_AT", + False, + None, + ("\u201e", "\u201c", "\u201a", "\u2018"), + id="austrian-german-fallback", + ), + pytest.param( + "zh_TW", + False, + None, + ("\u300c", "\u300d", "\u300e", "\u300f"), + id="traditional-chinese", + ), + pytest.param( + "tlh", + False, + None, + DEFAULT_SMARTQUOTES, + id="unsupported-fallback", + ), + pytest.param( + "fr_FR", + False, + [("fr_FR", ("<<", ">>", "<", ">")), ("", "abcd"), ("de", "bad")], + ("<<", ">>", "<", ">"), + id="custom-locale-override", + ), + ], +) +def test_resolve_smartquotes(language_code, smart_quotes, locales, expected): + """``resolve_smartquotes`` honors Docutils locale rules without mutating globals (#806).""" + from docutils.utils.smartquotes import smartchars + + snapshot = dict(smartchars.quotes) + assert ( + resolve_smartquotes( + language_code, + smart_quotes=smart_quotes, + smartquotes_locales=locales, + ) + == expected + ) + assert smartchars.quotes == snapshot + + +def test_create_md_parser_invalid_quotes(): + """``create_md_parser`` rejects quote overrides that do not specify 4 entries.""" + with pytest.raises(ValueError, match="exactly 4 quote strings"): + create_md_parser( + MdParserConfig(enable_extensions={"smartquotes"}), + DocutilsRenderer, + quotes=("<<", ">>"), + ) + + +@pytest.mark.parametrize( + ("source", "settings", "expected"), + [ + pytest.param( + 'Test\'s "double" and \'single\' and "*emph*"', + { + "language_code": "fr", + "myst_enable_extensions": ["smartquotes"], + }, + "Test\u2019s \u00ab\u00a0double\u00a0\u00bb and \u201csingle\u201d and \u00ab\u00a0emph\u00a0\u00bb", + id="fr-default", + ), + pytest.param( + '"Test"', + { + "language_code": "fr", + "smart_quotes": "alt", + "myst_enable_extensions": ["smartquotes"], + }, + "\u00ab\u202fTest\u202f\u00bb", + id="fr-alt", + ), + pytest.param( + "---\nmyst:\n enable_extensions: [smartquotes]\n---\n\"Hallo\" und 'Welt'", + { + "language_code": "de", + }, + "\u201eHallo\u201c und \u201aWelt\u2018", + id="de-topmatter", + ), + pytest.param( + "\"Custom\" and 'quote'", + { + "language_code": "de", + "smartquotes_locales": [("de", ("[", "]", "{", "}"))], + "myst_enable_extensions": ["smartquotes"], + }, + "[Custom] and {quote}", + id="custom-locales", + ), + ], +) +def test_docutils_smartquotes_localized(source, settings, expected): + """Docutils ``Parser`` localizes MyST ``smartquotes`` using document settings (#806).""" + doctree = publish_doctree( + source=source, + parser=Parser(), + settings_overrides={ + "warning_stream": io.StringIO(), + **settings, + }, + ) + assert doctree.astext() == expected diff --git a/tests/test_renderers/test_smartquotes.py b/tests/test_renderers/test_smartquotes.py new file mode 100644 index 00000000..f38ce0d4 --- /dev/null +++ b/tests/test_renderers/test_smartquotes.py @@ -0,0 +1,145 @@ +"""Tests for localized quote rules with the MyST ``smartquotes`` extension (#806).""" + +from __future__ import annotations + +import pytest +from docutils import nodes +from docutils.utils.smartquotes import smartchars +from sphinx_pytest.plugin import CreateDoctree + + +@pytest.mark.parametrize( + ("language", "sphinx_smartquotes", "expected"), + [ + pytest.param( + "fr", + True, + "Welcome to Test\u2019s documentation!\n\n\u00ab\u00a0Test\u00a0\u00bb and \u201csingle\u201d", + id="fr-with-sphinx-smartquotes", + ), + pytest.param( + "fr", + False, + "Welcome to Test\u2019s documentation!\n\n\u00ab\u00a0Test\u00a0\u00bb and \u201csingle\u201d", + id="fr-myst-only", + ), + pytest.param( + "de", + False, + "Welcome to Test\u2019s documentation!\n\n\u201eTest\u201c and \u201asingle\u2018", + id="de-myst-only", + ), + pytest.param( + "fr_CH", + False, + "Welcome to Test\u2019s documentation!\n\n\u00abTest\u00bb and \u2039single\u203a", + id="fr-ch-subtag", + ), + pytest.param( + "ja", + False, + "Welcome to Test\u2019s documentation!\n\n\u300cTest\u300d and \u300esingle\u300f", + id="ja-myst-only", + ), + pytest.param( + "ar", + False, + "Welcome to Test\u2019s documentation!\n\n\u201cTest\u201d and \u2018single\u2019", + id="unsupported-smartquotes-language-fallback", + ), + ], +) +def test_sphinx_smartquotes_language( + sphinx_doctree: CreateDoctree, + language: str, + sphinx_smartquotes: bool, + expected: str, +) -> None: + """Sphinx ``language`` localizes MyST's ``smartquotes`` extension (#806).""" + sphinx_doctree.set_conf( + { + "extensions": ["myst_parser"], + "myst_enable_extensions": ["smartquotes"], + "language": language, + "smartquotes": sphinx_smartquotes, + } + ) + result = sphinx_doctree( + "# Welcome to Test's documentation!\n\n\"Test\" and 'single'\n", + "index.md", + ) + assert not result.warnings + assert result.doctrees["index"].astext() == expected + + +def test_sphinx_smartquotes_topmatter_and_include( + sphinx_doctree: CreateDoctree, +) -> None: + """Per-file front matter and ``{include}`` directives honor Sphinx ``language`` (#806).""" + sphinx_doctree.srcdir.joinpath("included.md").write_text( + "\"Included\" and 'nested'\n", + encoding="utf8", + ) + sphinx_doctree.set_conf( + { + "extensions": ["myst_parser"], + "language": "fr", + "smartquotes": False, + "exclude_patterns": ["included.md"], + } + ) + source = ( + "---\n" + "myst:\n" + " enable_extensions:\n" + " - smartquotes\n" + "---\n" + "# Heading\n\n" + '"Main"\n\n' + "```{include} included.md\n" + "```\n" + ) + result = sphinx_doctree(source, "index.md") + assert not result.warnings + paragraphs = [p.astext() for p in result.doctrees["index"].findall(nodes.paragraph)] + assert paragraphs == [ + "\u00ab\u00a0Main\u00a0\u00bb", + "\u00ab\u00a0Included\u00a0\u00bb and \u201cnested\u201d", + ] + + +def test_sphinx_smartquotes_env_locales_no_global_mutation( + sphinx_doctree: CreateDoctree, +) -> None: + """Custom ``env.settings['smartquotes_locales']`` works without mutating Docutils globals.""" + snapshot = dict(smartchars.quotes) + (sphinx_doctree.srcdir / "ext_locales.py").write_text( + "def setup(app):\n" + " app.connect(\n" + " 'builder-inited',\n" + " lambda app: app.env.settings.update({\n" + " 'smartquotes_locales': [('fr', ('[[', ']]', '{{', '}}'))],\n" + " }),\n" + " )\n", + encoding="utf8", + ) + import sys + + sys.path.insert(0, str(sphinx_doctree.srcdir)) + try: + sphinx_doctree.set_conf( + { + "extensions": ["myst_parser", "ext_locales"], + "myst_enable_extensions": ["smartquotes"], + "language": "fr", + "smartquotes": False, + } + ) + result = sphinx_doctree("# Title\n\n\"Double\" and 'single'\n", "index.md") + finally: + sys.path.remove(str(sphinx_doctree.srcdir)) + sys.modules.pop("ext_locales", None) + + assert not result.warnings + assert result.doctrees["index"].astext() == "Title\n\n[[Double]] and {{single}}" + assert smartchars.quotes == snapshot