From 72dd223df3e8baf7d15a7d49d911cf9c2264420a Mon Sep 17 00:00:00 2001 From: Oscar Levin Date: Sat, 1 Aug 2026 12:02:46 -0600 Subject: [PATCH 01/13] Improve logging --- pretext/__init__.py | 2 +- pretext/logger.py | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/pretext/__init__.py b/pretext/__init__.py index 05c4944b..8d4ed671 100644 --- a/pretext/__init__.py +++ b/pretext/__init__.py @@ -18,7 +18,7 @@ VERSION = get_version("pretext", Path(__file__).parent.parent) -CORE_COMMIT = "1939b8c6c1cef11fb1c582ff69f49ce2ea8cfd27" +CORE_COMMIT = "8877b5771ff4e20d369f07378489713a3ac57665" def activate() -> None: diff --git a/pretext/logger.py b/pretext/logger.py index e95d81ef..4ed95ce0 100644 --- a/pretext/logger.py +++ b/pretext/logger.py @@ -8,18 +8,33 @@ log = logging.getLogger("ptxlogger") +class ColorFormatter(click_log.ColorFormatter): + """click_log prefixes a message with its level name, but only for the levels + in its own `colors` table; an unrecognized level gets no label at all. Core + PreTeXt renames level 50 to FATAL and adds BUG (45) and FALLBACK (25), so + those messages arrived unlabeled. Extend the table to cover them. + """ + + colors = { + **click_log.ColorFormatter.colors, + "fatal": dict(fg="red", bold=True), + "bug": dict(fg="magenta"), + "fallback": dict(fg="cyan"), + } + + def add_log_stream_handler() -> None: # Set up logging: # click_handler logs all messages to stdout as the CLI runs click_handler = logging.StreamHandler(sys.stdout) - click_handler.setFormatter(click_log.ColorFormatter()) + click_handler.setFormatter(ColorFormatter()) log.addHandler(click_handler) def get_log_error_flush_handler() -> logging.handlers.MemoryHandler: # error_flush_handler captures error/critical logs for flushing to stderr at the end of a CLI run sh = logging.StreamHandler(sys.stderr) - sh.setFormatter(click_log.ColorFormatter()) + sh.setFormatter(ColorFormatter()) sh.setLevel(logging.ERROR) error_flush_handler = logging.handlers.MemoryHandler( capacity=1024 * 100, From a1e24b22472c67cdbfce4002af23f6b69120b2f9 Mon Sep 17 00:00:00 2001 From: Oscar Levin Date: Sat, 1 Aug 2026 12:03:08 -0600 Subject: [PATCH 02/13] No longer require external attribute in publication file --- pretext/project/__init__.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pretext/project/__init__.py b/pretext/project/__init__.py index 7ee295bf..51a0ba7a 100644 --- a/pretext/project/__init__.py +++ b/pretext/project/__init__.py @@ -72,12 +72,12 @@ class Format(str, Enum): } -# The CLI only needs two values from the publication file. Therefore, this class ignores the vast majority of a publication file's contents, loading and validating only a (small) relevant subset. +# The CLI only needs one value from the publication file. Therefore, this class ignores the vast majority of a publication file's contents, loading and validating only the relevant subset. +# Note that as of 2026-07-30, the external directory has moved to the docinfo element. # Since we will want to hash the baseurl for generating qr codes, we also load it here. class PublicationSubset( pxml.BaseXmlModel, tag="publication", search_mode=SearchMode.UNORDERED ): - external: Path = pxml.wrapped("source/directories", pxml.attr()) generated: Path = pxml.wrapped("source/directories", pxml.attr()) baseurl: t.Optional[str] = pxml.wrapped( "html/baseurl", pxml.attr(name="href", default=None) @@ -518,14 +518,21 @@ def _read_publication_file_subset(self) -> PublicationSubset: p_bytes = ET.tostring(p_et) return PublicationSubset.from_xml(p_bytes) + def _get_managed_directories(self) -> t.Dict[Path, Path]: + generated, external = core.get_managed_directories( + xml_source=self.source_abspath(), + pub_file=self.publication_abspath().as_posix(), + ) + return {"generated": generated, "external": external} + def external_dir(self) -> Path: - return self._read_publication_file_subset().external + return self._get_managed_directories()["external"] def external_dir_abspath(self) -> Path: return (self.source_abspath().parent / self.external_dir()).resolve() def generated_dir(self) -> Path: - return self._read_publication_file_subset().generated + return self._get_managed_directories()["generated"] def generated_dir_abspath(self) -> Path: return (self.source_abspath().parent / self.generated_dir()).resolve() From d8c97819a8c8a5c8ba7eaa1bd71739f0a7219559 Mon Sep 17 00:00:00 2001 From: Oscar Levin Date: Sat, 1 Aug 2026 12:03:18 -0600 Subject: [PATCH 03/13] Improvements to templates --- CHANGELOG.md | 4 + templates/article/publication/publication.ptx | 74 ++- templates/article/source/main.ptx | 2 +- templates/book/publication/publication.ptx | 407 +++++++++++++---- templates/book/source/docinfo.ptx | 24 +- .../publication-slides-annotated.ptx | 19 +- .../course/publication/publication-slides.ptx | 19 +- templates/course/publication/publication.ptx | 426 ++++++++++++++---- .../publication/publication_standalone.ptx | 424 +++++++++++++---- templates/course/source/docinfo.ptx | 2 + templates/demo/publication/publication.ptx | 419 +++++++++++++---- templates/demo/source/docinfo.ptx | 8 +- templates/hello/publication/publication.ptx | 2 +- templates/publication.ptx | 2 +- .../slideshow/publication/publication.ptx | 36 +- templates/slideshow/source/main.ptx | 1 + templates/standalone-publication.ptx | 2 +- 17 files changed, 1464 insertions(+), 407 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e71b78a1..9dd15196 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ Instructions: Add a subsection under `[Unreleased]` for additions, fixes, change ## [Unreleased] +### Changed + +- External directory specification should now go in the `` element rather than the publication file. + ## [2.45.0] - 2026-07-26 Includes updates to core through commit: [eec7477](https://github.com/PreTeXtBook/pretext/commit/eec74773ac46acb74bb423b7b216d64b2416ee27) diff --git a/templates/article/publication/publication.ptx b/templates/article/publication/publication.ptx index f36399d0..2be00d28 100644 --- a/templates/article/publication/publication.ptx +++ b/templates/article/publication/publication.ptx @@ -1,8 +1,25 @@ + + + + + + + + + + - + + + - + + + + + + @@ -12,5 +29,58 @@ https://pretextbook.org/doc/guide/html/appendix-journals.html#appendix-journals --> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/templates/article/source/main.ptx b/templates/article/source/main.ptx index a63fa4eb..44bf7983 100644 --- a/templates/article/source/main.ptx +++ b/templates/article/source/main.ptx @@ -2,7 +2,7 @@ - + authorslastnamelowercase-ShortTitleCamelCase \newcommand{\foo}{b^{ar}} diff --git a/templates/book/publication/publication.ptx b/templates/book/publication/publication.ptx index 92d6e51f..ad58e77e 100644 --- a/templates/book/publication/publication.ptx +++ b/templates/book/publication/publication.ptx @@ -7,80 +7,156 @@ - + + + - + + + - + + + - + + + - + + - + + + + + + - + + - + + + + + + + + + + + + + + + + + + + + - - - + + + + - - - + + + - + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + + + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + + - + + + + + + + + - - + + + - + + - + + + + + + + + + + + + + + + + + + + + + + - - + + + + + + + - - + + + + + - - + + - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + - + + - - + + + - + + + - - - + + + + - + + + - - - - - + + + + + + + + + + + + + + + + + + + + + diff --git a/templates/book/source/docinfo.ptx b/templates/book/source/docinfo.ptx index f5a76985..827e78e8 100644 --- a/templates/book/source/docinfo.ptx +++ b/templates/book/source/docinfo.ptx @@ -5,25 +5,31 @@ - changeme + - A simple string (no markup is allowed) to describe your book. + + - - - - - - \newcommand{\N}{\mathbb N} \newcommand{\Z}{\mathbb Z} \newcommand{\Q}{\mathbb Q} \newcommand{\R}{\mathbb R} + + \newcommand{\N}{\mathbb N} + \newcommand{\Z}{\mathbb Z} + \newcommand{\Q}{\mathbb Q} + \newcommand{\R}{\mathbb R} + - \usepackage{tikz, pgfplots} \usetikzlibrary{positioning,matrix,arrows} \usetikzlibrary{shapes,decorations,shadows,fadings,patterns} \usetikzlibrary{decorations.markings} + + \usepackage{tikz, pgfplots} + \usetikzlibrary{positioning,matrix,arrows} + \usetikzlibrary{shapes,decorations,shadows,fadings,patterns} + \usetikzlibrary{decorations.markings} + diff --git a/templates/course/publication/publication-slides-annotated.ptx b/templates/course/publication/publication-slides-annotated.ptx index 788808d4..4a3ac0df 100644 --- a/templates/course/publication/publication-slides-annotated.ptx +++ b/templates/course/publication/publication-slides-annotated.ptx @@ -4,21 +4,32 @@ - + + + + + + - - + + + + + + + + - + diff --git a/templates/course/publication/publication-slides.ptx b/templates/course/publication/publication-slides.ptx index 935aadc1..d2110489 100644 --- a/templates/course/publication/publication-slides.ptx +++ b/templates/course/publication/publication-slides.ptx @@ -2,7 +2,7 @@ - + @@ -10,15 +10,26 @@ + + + + + - - + + + + + + + + - + diff --git a/templates/course/publication/publication.ptx b/templates/course/publication/publication.ptx index a296d5b6..20d77558 100644 --- a/templates/course/publication/publication.ptx +++ b/templates/course/publication/publication.ptx @@ -7,119 +7,247 @@ - + + + - + + + - + + + - + + + - + + - + + + + + + - + + - + + + + + + + + + + + + + + + + + + + + - - - + + + + - - - + + + - + + + + + + + + - - - - - - - - - + + + + + + + + + - - - + + + - + + + + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + + - - - - - - - - - + > + --> + + + + + + + + + + + + + + - + + - - - - - - - - + + + - - + + + + + + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + - + + - - + + + - + + + - - - + + + + - + + + - - - - - + + + + + + + + + + + + + + + + + + + + + diff --git a/templates/course/publication/publication_standalone.ptx b/templates/course/publication/publication_standalone.ptx index 48973a46..abb1849e 100644 --- a/templates/course/publication/publication_standalone.ptx +++ b/templates/course/publication/publication_standalone.ptx @@ -7,116 +7,244 @@ - + + + - + + + - + + + - + + + - + + - + + + + + + - + + - + + + + + + + + + + + + + + + + + + + + - - - + + + + - - - + + + - + + + + + + + + - - - - - - - - - + + + + + + + + + - - - + + + - + + + + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + + - - - - - - - - - + > + --> + + + + + + + + + + + + + + - + + - + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + - + + - - + + + - + + + - - - + + + + - + + + - - - - - + + + + + + + + + + + + + + + + + + + + + diff --git a/templates/course/source/docinfo.ptx b/templates/course/source/docinfo.ptx index fb0b305b..ab35e2d7 100644 --- a/templates/course/source/docinfo.ptx +++ b/templates/course/source/docinfo.ptx @@ -5,6 +5,8 @@ + + diff --git a/templates/demo/publication/publication.ptx b/templates/demo/publication/publication.ptx index b07867ba..bfaabf97 100644 --- a/templates/demo/publication/publication.ptx +++ b/templates/demo/publication/publication.ptx @@ -7,116 +7,245 @@ - + + + - + + + - + + + - + + + - + + - + + + + + + - + + - + + + + + + + + + + + + + + + + + + + + - + + + + - + - + + + + + + + + - - - - - - - - - - - - - - + + + + + + + + + + + + + + - + + + + + + + + + + + - - - - + + + + + + + + + + + + + + + + + + - - - - - - - - - + > + --> + + + + + + + + + + + + + + - - - + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + + + + + + + + + - - - + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + - + + - - + + + - + + + - - - + + + + - + + + - - - - - + + + + + + + + + + + + + + + + + + + + + diff --git a/templates/demo/source/docinfo.ptx b/templates/demo/source/docinfo.ptx index e4ea3b0e..b93fe93b 100644 --- a/templates/demo/source/docinfo.ptx +++ b/templates/demo/source/docinfo.ptx @@ -12,17 +12,17 @@ https://pretextbook.org/doc/guide/html/sec-publishing-to-runestone-academy.html for more information. --> - - changeme + + - + diff --git a/templates/hello/publication/publication.ptx b/templates/hello/publication/publication.ptx index 35f12844..083b57a5 100644 --- a/templates/hello/publication/publication.ptx +++ b/templates/hello/publication/publication.ptx @@ -2,6 +2,6 @@ - + diff --git a/templates/publication.ptx b/templates/publication.ptx index 35f12844..083b57a5 100644 --- a/templates/publication.ptx +++ b/templates/publication.ptx @@ -2,6 +2,6 @@ - + diff --git a/templates/slideshow/publication/publication.ptx b/templates/slideshow/publication/publication.ptx index 0b7a3b5b..2db8270b 100644 --- a/templates/slideshow/publication/publication.ptx +++ b/templates/slideshow/publication/publication.ptx @@ -19,19 +19,37 @@ along with PreTeXt. If not, see . - + + + + + + + + + - - + + + + + + + + + + + + - + @@ -39,4 +57,12 @@ along with PreTeXt. If not, see . - \ No newline at end of file + + + + + diff --git a/templates/slideshow/source/main.ptx b/templates/slideshow/source/main.ptx index a551d691..e041eace 100644 --- a/templates/slideshow/source/main.ptx +++ b/templates/slideshow/source/main.ptx @@ -21,6 +21,7 @@ along with PreTeXt. If not, see . + \newcommand{\definiteintegral}[4]{\int_{#1}^{#2}\,#3\,d#4} diff --git a/templates/standalone-publication.ptx b/templates/standalone-publication.ptx index b471b7e5..f396543b 100644 --- a/templates/standalone-publication.ptx +++ b/templates/standalone-publication.ptx @@ -5,7 +5,7 @@ This is a publication file for standalone documents. It should only exist in `~ - + From 3b3e460e9c9e25afee4c6cd52e3ceddc17813405 Mon Sep 17 00:00:00 2001 From: Oscar Levin Date: Sat, 1 Aug 2026 12:37:29 -0600 Subject: [PATCH 04/13] update executable model --- CHANGELOG.md | 9 +++ pretext/project/__init__.py | 5 +- pretext/project/xml.py | 60 +++++++++++++++-- pretext/utils.py | 42 +++++++----- .../elaborate/executables.ptx | 2 + tests/test_project.py | 61 +++++++++++++++++ tests/test_utils.py | 66 +++++++++++++++++++ 7 files changed, 220 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9dd15196..e144ea77 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,9 +9,18 @@ Instructions: Add a subsection under `[Unreleased]` for additions, fixes, change ## [Unreleased] +### Added + +- The `jing` executable can now be set in `executables.ptx`, so `pretext validate` can use a jing that isn't on your search path (including a command with options, such as `java -jar /usr/share/java/jing.jar`). + ### Changed - External directory specification should now go in the `` element rather than the publication file. +- The `liblouis`, `pdfsvg`, and `pdfpng` executables are no longer used (braille goes through the `louis` Python bindings, and PDF conversion through pyMuPDF). They are still accepted in `executables.ptx`, with a warning, so existing projects keep working. + +### Fixed + +- Legacy (v1) `project.ptx` manifests now supply the full set of executables to core, instead of omitting `mermaid`, `perl`, `fop`, and `jing`. ## [2.45.0] - 2026-07-26 diff --git a/pretext/project/__init__.py b/pretext/project/__init__.py index 51a0ba7a..d86ae4ed 100644 --- a/pretext/project/__init__.py +++ b/pretext/project/__init__.py @@ -1585,7 +1585,10 @@ class ProjectVersionOnly(pxml.BaseXmlModel, tag="project"): d = legacy_project.model_dump() d["targets"] = new_targets # Rename from `executables` to `_executables` when moving from the old to new project format. - d["_executables"] = legacy_project.executables + # The conversion matters: the legacy element names only a subset of the + # executables core needs, so the rest (`mermaid`, `perl`, `fop`, `jing`) + # must come from the modern model's defaults. + d["_executables"] = Executables.from_legacy(legacy_project.executables) d.pop("executables") p = Project( ptx_version="2", diff --git a/pretext/project/xml.py b/pretext/project/xml.py index 59ff55ac..d179a9c1 100644 --- a/pretext/project/xml.py +++ b/pretext/project/xml.py @@ -1,30 +1,74 @@ +import logging from pathlib import Path import shutil import typing as t from enum import Enum -from pydantic import ConfigDict +from pydantic import ConfigDict, model_validator import pydantic_xml as pxml from pydantic_xml.element.element import SearchMode +log = logging.getLogger("ptxlogger") + +# Executables that the core script once used but no longer does. They are still +# accepted (silently deleting them would make every older `executables.ptx` a +# hard parse error, since the model forbids extra attributes), but they are +# excluded from `model_dump()` and so never reach core. +DEPRECATED_EXECUTABLES = { + "liblouis": "braille is now translated with the `louis` Python bindings", + "pdfsvg": "PDFs are now converted with the pyMuPDF library", + "pdfpng": "PDFs are now converted with the pyMuPDF library", +} + # To prevent circular imports, put this here instead of in `__init__`; however, it's not used in this file. class Executables(pxml.BaseXmlModel, tag="executables"): + """The executables the core script invokes; keep in sync with the + `[executables]` section of the core script's `pretext.cfg`.""" + model_config = ConfigDict(extra="forbid") latex: str = pxml.attr(default="latex") pdflatex: str = pxml.attr(default="pdflatex") xelatex: str = pxml.attr(default="xelatex") - pdfsvg: t.Optional[str] = pxml.attr(default="pdf2svg") # If not specified, use a local executable if it exists; if it doesn't exist, choose `None`, which allows the generation logic to use the server instead. asy: t.Optional[str] = pxml.attr(default=shutil.which("asy")) # No sage server, so we don't do the same for sage. sage: t.Optional[str] = pxml.attr(default="sage") mermaid: str = pxml.attr(default="mmdc") - pdfpng: t.Optional[str] = pxml.attr(default="convert") pdfeps: str = pxml.attr(default="pdftops") node: str = pxml.attr(default="node") - liblouis: str = pxml.attr(default="file2brl") perl: str = pxml.attr(default="perl") fop: str = pxml.attr(default="fop") + # `jing` is a Java program, so this can be the name of an executable (from a + # system package) or a command with options, e.g. `java -jar /usr/share/java/jing.jar`. + jing: str = pxml.attr(default="jing") + + # Deprecated: see `DEPRECATED_EXECUTABLES`. `exclude=True` keeps these out of + # `model_dump()`, which is what gets handed to core. + liblouis: t.Optional[str] = pxml.attr(default=None, exclude=True) + pdfsvg: t.Optional[str] = pxml.attr(default=None, exclude=True) + pdfpng: t.Optional[str] = pxml.attr(default=None, exclude=True) + + @model_validator(mode="after") + def warn_about_deprecated(self) -> "Executables": + for name, reason in DEPRECATED_EXECUTABLES.items(): + if getattr(self, name) is not None: + log.warning( + f'The "{name}" executable is no longer used by PreTeXt ({reason}); ' + "you can remove it from your executables.ptx file." + ) + return self + + @classmethod + def from_legacy(cls, legacy: "LegacyExecutables") -> "Executables": + """Build from a v1 manifest's ``, which names only a subset + of the executables core needs; the rest keep their defaults.""" + return cls( + **{ + key: value + for key, value in legacy.model_dump().items() + if value is not None and key in cls.model_fields + } + ) class LegacyFormat(str, Enum): @@ -86,13 +130,15 @@ class LegacyExecutables( latex: str = pxml.element() pdflatex: str = pxml.element() xelatex: str = pxml.element() - pdfsvg: t.Optional[str] = pxml.element(default=None) asy: str = pxml.element() sage: str = pxml.element() - pdfpng: t.Optional[str] = pxml.element(default=None) pdfeps: str = pxml.element() node: str = pxml.element() - liblouis: str = pxml.element() + # Deprecated (see `DEPRECATED_EXECUTABLES`), but optional rather than absent, + # since legacy manifests in the wild still carry these elements. + liblouis: t.Optional[str] = pxml.element(default=None) + pdfsvg: t.Optional[str] = pxml.element(default=None) + pdfpng: t.Optional[str] = pxml.element(default=None) class LegacyProject(pxml.BaseXmlModel, tag="project", search_mode=SearchMode.UNORDERED): diff --git a/pretext/utils.py b/pretext/utils.py index 43a963d4..520b2f76 100644 --- a/pretext/utils.py +++ b/pretext/utils.py @@ -232,11 +232,31 @@ def xml_validates_against_schema(etree: _Element) -> bool: return False +def _jing_command() -> Optional[List[str]]: + """The `jing` invocation to use, or `None` when jing is unavailable. + + Prefers the project's configured `jing` executable (from `executables.ptx`), + which lets a user point at a jing outside the search path; falls back to a + `jing` on the PATH when no project has been loaded. The configured value may + carry options (e.g. `java -jar /usr/share/java/jing.jar`), so the whole + command line is returned rather than just the executable. + """ + try: + return core.get_executable_cmd("jing") + except (KeyError, TypeError, OSError) as e: + # In turn: core's executables have no `jing` key (a manifest read by an + # older CLI), no project has been loaded so core has no executables at + # all, or the configured command isn't on the PATH. + log.debug(f"Could not use a configured jing command: {e}") + jing_executable = shutil.which("jing") + return [jing_executable] if jing_executable is not None else None + + def _validate_with_jing( etree: _Element, schema_file: Path ) -> Optional[tuple[bool, str]]: - jing_executable = shutil.which("jing") - if jing_executable is None: + jing_command = _jing_command() + if jing_command is None: return None tmp_path: Optional[Path] = None @@ -249,7 +269,7 @@ def _validate_with_jing( tmp_path = Path(tmp.name) result = subprocess.run( - [jing_executable, str(schema_file), str(tmp_path)], + jing_command + [str(schema_file), str(tmp_path)], check=False, capture_output=True, text=True, @@ -442,11 +462,9 @@ def check_asset_execs(element: str, outformats: Optional[List[str]] = None) -> N # Create list of executables needed based on output format required_execs = [] if element == "latex-image": + # svg and png conversions go through the pyMuPDF library, so they need no + # executable beyond the LaTeX engine itself. required_execs = ["xelatex"] - if "svg" in outformats or "all" in outformats: - required_execs.append("pdfsvg") - if "png" in outformats or "all" in outformats: - required_execs.append("pdfpng") if "eps" in outformats or "all" in outformats: required_execs.append("pdfeps") if element == "sageplot": @@ -457,16 +475,6 @@ def check_asset_execs(element: str, outformats: Optional[List[str]] = None) -> N "Darwin": "", "Linux": "", }, - "pdfsvg": { - "Windows": "Follow the instructions at https://pretextbook.org/doc/guide/html/section-installing-pdf2svg.html to install pdf2svg", - "Darwin": "", - "Linux": "You should be able to install pdf2svg with your package manager (e.g., `sudo apt install pdf2svg`. See https://github.com/dawbarton/pdf2svg#pdf2svg.", - }, - "pdfpng": { - "Windows": "See https://pretextbook.org/doc/guide/html/windows-cli-software.html", - "Darwin": "", - "Linux": "", - }, "pdfeps": { "Windows": "See https://pretextbook.org/doc/guide/html/windows-cli-software.html", "Darwin": "", diff --git a/tests/examples/projects/project_refactor/elaborate/executables.ptx b/tests/examples/projects/project_refactor/elaborate/executables.ptx index 3b16edde..d6eb99a6 100644 --- a/tests/examples/projects/project_refactor/elaborate/executables.ptx +++ b/tests/examples/projects/project_refactor/elaborate/executables.ptx @@ -1,4 +1,6 @@ + diff --git a/tests/test_project.py b/tests/test_project.py index 08961ca0..ff84b1d2 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -188,7 +188,9 @@ def test_manifest_elaborate(tmp_path: Path) -> None: assert project.site == Path("my-great-site") assert project.xsl == Path("customizations") assert project._executables.xelatex == "xelatex" + # Deprecated executables still parse, but are never handed to core. assert project._executables.liblouis == "foobar" + assert "liblouis" not in project._executables.model_dump() assert project.asy_method == "local" t_web = project.get_target("web") @@ -255,7 +257,13 @@ def test_manifest_legacy() -> None: assert len(project.targets) == 3 assert project._executables.xelatex == "xelatex" + # Deprecated executables still parse, but are never handed to core. assert project._executables.liblouis == "foobar" + assert "liblouis" not in project._executables.model_dump() + # Executables the legacy format has no element for fall back to their + # modern defaults, so core is never handed a short dict. + assert project._executables.model_dump()["jing"] == "jing" + assert project._executables.model_dump()["mermaid"] == "mmdc" t_html = project.get_target("html") assert t_html is not None @@ -297,7 +305,13 @@ def test_manifest_legacy_wrong() -> None: assert len(project.targets) == 3 assert project._executables.xelatex == "xelatex" + # Deprecated executables still parse, but are never handed to core. assert project._executables.liblouis == "foobar" + assert "liblouis" not in project._executables.model_dump() + # Executables the legacy format has no element for fall back to their + # modern defaults, so core is never handed a short dict. + assert project._executables.model_dump()["jing"] == "jing" + assert project._executables.model_dump()["mermaid"] == "mmdc" t_html = project.get_target("html") assert t_html is not None @@ -329,6 +343,53 @@ def test_manifest_legacy_wrong() -> None: assert project._executables.latex == "latex1" +def test_executables_match_core() -> None: + """The executables handed to core are exactly the keys the core script's + `pretext.cfg` declares -- no missing key (core looks these up in a plain + dict) and no vestigial extras.""" + assert set(pr.Executables().model_dump()) == { + "latex", + "pdflatex", + "xelatex", + "asy", + "mermaid", + "sage", + "pdfeps", + "node", + "perl", + "fop", + "jing", + } + + +def test_executables_jing_from_manifest(tmp_path: Path) -> None: + """A `jing` in executables.ptx reaches core, so a user whose jing is off the + search path (or is a `java -jar ...` command) can point the CLI at it.""" + prj_path = tmp_path / "simple" + shutil.copytree(EXAMPLES_DIR / "projects" / "project_refactor" / "simple", prj_path) + (prj_path / "executables.ptx").write_text( + '' + ) + with utils.working_directory(prj_path): + project = pr.Project.parse() + assert project._executables.jing == "java -jar /opt/jing.jar" + # `model_dump()` is what `init_core()` hands to core, options and all. + assert project._executables.model_dump()["jing"] == "java -jar /opt/jing.jar" + + +def test_executables_deprecated_and_unknown() -> None: + """Executables core no longer uses are accepted (older `executables.ptx` + files must keep parsing) but withheld from core; genuinely unknown ones + are still rejected, so typos don't pass silently.""" + execs = pr.Executables(liblouis="file2brl", pdfsvg="pdf2svg", pdfpng="convert") + assert "liblouis" not in execs.model_dump() + assert "pdfsvg" not in execs.model_dump() + assert "pdfpng" not in execs.model_dump() + + with pytest.raises(pydantic.ValidationError): + pr.Executables(jjing="typo") # type: ignore[call-arg] + + def test_html_build_permissions(tmp_path: Path) -> None: """HTML output is world-readable (0o755+), so it can be served directly from a web root.""" diff --git a/tests/test_utils.py b/tests/test_utils.py index 1662d7b7..3de20d15 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -262,6 +262,72 @@ def _raise(*args: object, **kwargs: object) -> object: ) +def test_jing_command_prefers_configured_executable( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A `jing` configured in executables.ptx wins over one on the PATH, and + keeps its options (jing is a Java program, so it may be a whole command).""" + configured = ["/opt/java", "-jar", "/opt/jing.jar"] + monkeypatch.setattr(utils.core, "get_executable_cmd", lambda name: configured) + monkeypatch.setattr(utils.shutil, "which", lambda name: "/usr/bin/jing") + assert utils._jing_command() == configured + + +@pytest.mark.parametrize("error", [KeyError("jing"), TypeError(), OSError("missing")]) +def test_jing_command_falls_back_to_path( + monkeypatch: pytest.MonkeyPatch, error: Exception +) -> None: + """With no usable configured jing -- no `jing` key, no project loaded, or a + command that isn't installed -- fall back to a jing on the PATH.""" + + def _raise(name: str) -> object: + raise error + + monkeypatch.setattr(utils.core, "get_executable_cmd", _raise) + monkeypatch.setattr(utils.shutil, "which", lambda name: "/usr/bin/jing") + assert utils._jing_command() == ["/usr/bin/jing"] + + +def test_jing_command_none_when_unavailable(monkeypatch: pytest.MonkeyPatch) -> None: + def _raise(name: str) -> object: + raise OSError("missing") + + monkeypatch.setattr(utils.core, "get_executable_cmd", _raise) + monkeypatch.setattr(utils.shutil, "which", lambda name: None) + assert utils._jing_command() is None + # And the engine reports itself unavailable, so the caller can try another. + assert utils._validate_with_jing(ET.fromstring(""), Path("s.rng")) is None + + +def test_validate_with_jing_invokes_configured_command( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """The configured command line, options and all, is what gets run.""" + recorded: list[list[str]] = [] + + class _Result: + returncode = 0 + stdout = "" + stderr = "" + + def _run(cmd: list[str], **kwargs: object) -> _Result: + recorded.append(cmd) + return _Result() + + monkeypatch.setattr( + utils, "_jing_command", lambda: ["java", "-jar", "/opt/jing.jar"] + ) + monkeypatch.setattr(utils.subprocess, "run", _run) + + schema_file = tmp_path / "mini.rng" + assert utils._validate_with_jing(ET.fromstring(""), schema_file) == ( + True, + "", + ) + assert recorded[0][:3] == ["java", "-jar", "/opt/jing.jar"] + assert recorded[0][3] == str(schema_file) + + def test_run_schema_validation_uses_first_available_engine( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: From 93cfdc38bfbd8421916442c30153126f5e55bb70 Mon Sep 17 00:00:00 2001 From: Oscar Levin Date: Sat, 1 Aug 2026 17:14:26 -0600 Subject: [PATCH 05/13] improve validation option --- CHANGELOG.md | 7 +- pretext/cli.py | 83 +++++++--- pretext/project/__init__.py | 117 +++++++++++++- pretext/resources/salve/__init__.py | 2 + pretext/resources/salve/package.json | 10 ++ pretext/resources/salve/ptx-jing-shim.mjs | 121 ++++++++++++++ pretext/utils.py | 188 +++++++++++++--------- tests/test_cli.py | 109 ++++++++++--- tests/test_project.py | 46 ++++++ tests/test_utils.py | 158 +++++++++++------- 10 files changed, 652 insertions(+), 189 deletions(-) create mode 100644 pretext/resources/salve/__init__.py create mode 100644 pretext/resources/salve/package.json create mode 100644 pretext/resources/salve/ptx-jing-shim.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index e144ea77..a77113d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,10 +11,15 @@ Instructions: Add a subsection under `[Unreleased]` for additions, fixes, change ### Added -- The `jing` executable can now be set in `executables.ptx`, so `pretext validate` can use a jing that isn't on your search path (including a command with options, such as `java -jar /usr/share/java/jing.jar`). +- The `jing` executable can now be set in `executables.ptx`, so validation can use a jing that isn't on your search path (including a command with options, such as `java -jar /usr/share/java/jing.jar`). +- `pretext validate --method` chooses how to validate: `local` (an installed jing, the default), `local-dev` (the development schema, same as `--dev`), `server` (jing as a remote service, needing no local install), or `terse` (machine-readable, one tab-separated message per line). +- `pretext validate --engine salve` (EXPERIMENTAL) runs the RelaxNG check with the salve-annos validator from the pretext-tools VS Code extension instead of jing, needing no Java. It installs itself with npm on first use, and needs node. It finds the same problems as jing, but words them differently and does not repeat follow-on messages, so a report from it is not comparable to jing's line for line. Please try it and report differences. ### Changed +- `pretext validate` now produces core's consolidated validation report, written to `logs/`. Alongside the schema messages from jing it includes the "validation-plus" checks that no schema can express, and every message names the source file, path, line, and an excerpt of the offending text -- rather than line numbers of an assembled file you never see. +- Schema validation during a build or generate now warns with a count and writes the messages to `logs/schema-errors.log` (with the assembled source they refer to alongside), instead of putting them on the terminal. It is skipped when jing isn't installed, and never blocks a build. +- Validation now uses jing exclusively; lxml is no longer tried, since it cannot compile the PreTeXt schema (which incorporates PreFigure's). - External directory specification should now go in the `` element rather than the publication file. - The `liblouis`, `pdfsvg`, and `pdfpng` executables are no longer used (braille goes through the `louis` Python bindings, and PDF conversion through pyMuPDF). They are still accepted in `executables.ptx`, with a warning, so existing projects keep working. diff --git a/pretext/cli.py b/pretext/cli.py index 66b06096..f77b8623 100644 --- a/pretext/cli.py +++ b/pretext/cli.py @@ -1,4 +1,3 @@ -from datetime import datetime import importlib import importlib.util import logging @@ -743,48 +742,84 @@ def build( help="Validate against the development schema (pretext-dev.rng) instead of the " "stable schema, allowing experimental elements.", ) +@click.option( + "--method", + type=click.Choice(["local", "local-dev", "server", "terse"]), + default=None, + help='How to validate: "local" (an installed jing, the default), "local-dev" ' + '(as "local", against the development schema), "server" (jing as a remote ' + 'service, needing no local install), or "terse" (machine-readable, one ' + "tab-separated message per line).", +) +@click.option( + "--engine", + type=click.Choice(["jing", "salve"]), + default="jing", + show_default=True, + help='Which RelaxNG engine performs the check. EXPERIMENTAL: "salve" uses ' + "the validator behind the pretext-tools VS Code extension instead of jing, " + "and needs no Java (it installs itself with npm on first use). It finds the " + "same problems but words them differently, so reports are not comparable " + "line for line. Ignored by `--method server`.", +) @click.pass_context @nice_errors -def validate(ctx: click.Context, target_name: Optional[str], dev: bool) -> None: +def validate( + ctx: click.Context, + target_name: Optional[str], + dev: bool, + method: Optional[str], + engine: str, +) -> None: """ Validate the source of TARGET against the PreTeXt RelaxNG schema. - Reports schema errors and exits with a non-zero status when the document is - invalid, so it can gate CI or pre-commit checks. Without TARGET, the first - target in project.ptx is used. Exit codes: 0 = valid, 1 = invalid, 2 = - validation could not be performed (no validator available). + Writes the consolidated validation report: the schema messages from jing + together with those of the "validation-plus" stylesheet, each naming the + source file, path, and line it came from. Without TARGET, the first target + in project.ptx is used. Exit codes: 0 = valid, 1 = invalid, 2 = validation + could not be performed (no validator available). """ project = ctx.obj["project"] target = project.get_target(target_name) # Assemble the source (resolves xinclude); surfaces syntax/xinclude errors. try: - etree = target.source_element() + target.source_element() except Exception as e: log.error(f"Could not assemble source for validation: {e}") raise SystemExit(1) - schema_file = utils.schema_path(dev) - log.info(f"Validating source against schema {schema_file.name}.") - is_valid, error_text = utils.run_schema_validation( - etree, schema_file, order=("jing", "lxml") + if method is None: + method = "local-dev" if dev else "local" + log.info( + f"Validating source of target {target.name} " + f"(method: {method}, engine: {engine})." ) + if engine == "salve": + log.warning( + "The salve engine is experimental. Its messages are worded " + "differently from jing's, so a report from it will not match a " + "report from jing line for line." + ) + result = target.validate_source(method=method, engine=engine) - if is_valid: - log.info(f"PreTeXt source passed schema validation ({schema_file.name}).") - return - if is_valid is None: - log.error(error_text) + if result is None: + log.error("Validation could not be performed.") + log.error( + "Install jing and make sure it is on your PATH (or name it in " + "executables.ptx), or use `pretext validate --method server` or " + "`pretext validate --engine salve`." + ) raise SystemExit(2) - error_log_path = ( - Path("logs") / f"{datetime.now().strftime('%Y%m%d-%H%M%S')}-schema-errors.log" - ) - with open(error_log_path, "w") as error_log_file: - error_log_file.write(error_text) - log.error("PreTeXt source did NOT pass schema validation:") - log.error(error_text) - log.error(f"See {error_log_path} for the full report.") + message_count, report = result + if message_count == 0: + log.info("PreTeXt source passed validation with no messages.") + log.info(f"Report: {report}") + return + log.error(f"PreTeXt source did NOT pass validation ({message_count} messages).") + log.error(f"See {report} for the full report.") raise SystemExit(1) diff --git a/pretext/project/__init__.py b/pretext/project/__init__.py index d86ae4ed..2ee13b9b 100644 --- a/pretext/project/__init__.py +++ b/pretext/project/__init__.py @@ -141,6 +141,9 @@ class Target(pxml.BaseXmlModel, tag="target", search_mode=SearchMode.UNORDERED): source: Path = pxml.attr(default=Path("main.ptx")) # Cache of assembled "version only" source element. _source_element: t.Optional[ET._Element] = None + # Whether the warn-only schema check has already run for this target, so a + # build doesn't repeat it when it generates assets. + _schema_checked: bool = False # Cache of assembled with assembly-ids. _source_element_with_ids: t.Optional[ET._Element] = None # A path to the publication file for this target, relative to the project's `publication` path. This is mostly validated by `post_validate`. @@ -439,6 +442,17 @@ def source_element(self) -> ET._Element: log.debug(f"Using cached source_element for target {self.name}") return self._source_element + def check_schema(self) -> None: + """ + Warn (once per target) if the assembled source has schema errors, pointing + at the log file rather than filling the terminal. Never raises: a missing + jing, or an invalid document, must not stop a build. + """ + if self._schema_checked: + return + self._schema_checked = True + utils.warn_on_schema_errors(self.source_element(), self._project.logs_abspath()) + def source_element_with_ids(self) -> ET._Element: """ Returns the root element for the assembled source, after processing with assembly-id method. Caches the result for future calls. @@ -745,6 +759,87 @@ def build_theme(self) -> None: ) log.info(f"Theme built for target '{self.name}'") + # Not named `validate`: that would shadow pydantic's own (deprecated) + # `BaseModel.validate` classmethod. + def validate_source( + self, + method: str = "local", + dest_dir: t.Optional[Path] = None, + engine: str = "jing", + ) -> t.Optional[t.Tuple[int, Path]]: + """ + Validate the source with core's `validate`, which consolidates the RELAX-NG + messages from jing with those of the "validation-plus" stylesheet into one + report, and deposits the assembled source its line numbers refer to. + + `method` is core's: "local" (the installed jing), "local-dev" (as "local", + against the development schema), "server" (jing as a remote service), or + "terse" (one tab-separated message per line, for a program). + + `engine` selects what performs the RELAX-NG check: "jing", or the + experimental "salve" (the validator behind the pretext-tools VS Code + extension, run through a jing-compatible shim). The engine is irrelevant + to `method="server"`, which validates remotely. + + Returns the number of messages in the report along with its path, or `None` + if validation could not be performed at all. + """ + if dest_dir is None: + dest_dir = self._project.logs_abspath() + dest_dir.mkdir(parents=True, exist_ok=True) + + overrides = None + if engine == "salve": + salve_command = utils.salve_shim_command() + if salve_command is None: + return None + # Core reaches its RELAX-NG engine through the `jing` executable, so + # standing in for jing is all it takes to swap the engine. + overrides = {"jing": " ".join(salve_command)} + self._project.init_core(overrides) + try: + core.validate( + xml_source=self.source_abspath(), + pub_file=self.publication_abspath().as_posix(), + stringparams=self.stringparams.copy(), + out_file=None, + dest_dir=dest_dir.as_posix(), + method=method, + ) + except OSError as e: + # Raised when the configured `jing` can't be found; core has already + # logged which command it looked for. + log.debug(f"Validation could not run: {e}") + return None + finally: + if overrides is not None: + # Put the project's own executables back for anything that follows. + self._project.init_core() + report = dest_dir / (self.source_abspath().stem + "-validation.txt") + if not report.exists(): + # core returns without a report when it cannot reach a validation server. + return None + lines = report.read_text(encoding="utf-8").splitlines() + if method == "terse": + # One message per line, and nothing else in the file. + count = len([line for line in lines if line.strip()]) + else: + # Each message in the consolidated report ends with the check that + # raised it, so those lines count the messages of both sections. The + # report opens with a preamble that describes a message's fields + # (including "check:"), so counting starts at the first section + # banner to avoid mistaking that description for a message. + banner = "=" * 70 + first_section = lines.index(banner) if banner in lines else 0 + count = len( + [ + line + for line in lines[first_section:] + if line.startswith(" check: ") + ] + ) + return count, report + def build( self, clean: bool = False, @@ -767,7 +862,7 @@ def build( if not utils.xml_syntax_is_valid(self.publication_abspath(), "publication"): raise RuntimeError("XML syntax for publication file is invalid") # Validate xml against schema; continue with warning if invalid: - utils.xml_validates_against_schema(self.source_element()) + self.check_schema() # Clean output upon request if clean: @@ -1021,6 +1116,10 @@ def generate_assets( """ log.info("Generating any needed assets.") + # Warn about schema errors here too, since assets are often generated + # without a build. A no-op when a build already ran the check. + self.check_schema() + # To help with debugging, we are temporarily adding a reference generation step here. The only way this will be called is if `pretext generate references` is called explicitly. if requested_asset_types == ("references",): try: @@ -1699,6 +1798,10 @@ def stage_abspath(self) -> Path: def site_abspath(self) -> Path: return self.abspath() / self.site + def logs_abspath(self) -> Path: + # Where the CLI already writes its own logs; `.gitignore`d in a project. + return self.abspath() / "logs" + def deploy_strategy( self, ) -> t.Literal["default_target", "pelican_default", "pelican_custom", "static"]: @@ -1733,10 +1836,18 @@ def server_process( def get_executables(self) -> Executables: return self._executables - def init_core(self) -> None: + def init_core( + self, executable_overrides: t.Optional[t.Dict[str, str]] = None + ) -> None: + # `executable_overrides` swaps in a different command for an executable + # for the duration of one operation (calling `init_core()` again with no + # overrides restores the project's own). Used to run validation through + # an engine other than the configured `jing`. + exec_dict = self._executables.model_dump() + if executable_overrides is not None: + exec_dict.update(executable_overrides) # core does not support None as an executable value, so we must # adjust accordingly - exec_dict = self._executables.model_dump() for k in exec_dict: if exec_dict[k] is None: exec_dict[k] = "None" diff --git a/pretext/resources/salve/__init__.py b/pretext/resources/salve/__init__.py new file mode 100644 index 00000000..6e28970f --- /dev/null +++ b/pretext/resources/salve/__init__.py @@ -0,0 +1,2 @@ +# Marks the shim directory as a package so its files can be located with +# `importlib.resources`. diff --git a/pretext/resources/salve/package.json b/pretext/resources/salve/package.json new file mode 100644 index 00000000..d81dbb45 --- /dev/null +++ b/pretext/resources/salve/package.json @@ -0,0 +1,10 @@ +{ + "name": "ptx-salve-validator", + "version": "1.0.0", + "description": "Runs @pretextbook/schema behind a jing-compatible command-line interface, so core's validate() can use it in place of jing.", + "private": true, + "type": "module", + "dependencies": { + "@pretextbook/schema": "^0.2.0" + } +} diff --git a/pretext/resources/salve/ptx-jing-shim.mjs b/pretext/resources/salve/ptx-jing-shim.mjs new file mode 100644 index 00000000..77b66463 --- /dev/null +++ b/pretext/resources/salve/ptx-jing-shim.mjs @@ -0,0 +1,121 @@ +/** + * A jing-compatible front end for the salve-annos validator in + * `@pretextbook/schema` (the engine behind the pretext-tools VS Code extension). + * + * Core's `validate()` invokes its RELAX-NG engine as + * + * + * + * and reads stdout as lines matching ^.*?:(\d+):(\d+): (.*)$ , using the line + * number and the message body. Exit 0 means valid and 1 means the document has + * messages. Satisfying that contract is all it takes to stand in for jing, and + * core then produces its usual consolidated report. + * + * Runs under Node and under Deno: nothing Node-only is imported at module + * scope, and the package's `require("fs")` lives in a default file reader that + * the `readFile` override below keeps out of the picture. (The grammar + * compiler *is* Node-locked, so it is imported dynamically, only on a cache + * miss.) + */ +import { readFileSync, writeFileSync, mkdirSync, statSync } from "node:fs"; +import { basename } from "node:path"; +import { fileURLToPath } from "node:url"; + +// Kinds that RELAX-NG itself can express. The others the package can report +// (duplicate-id, duplicate-label, dangling-reference) are the ground core +// covers with its "validation-plus" stylesheet, so including them here would +// double-report them in the consolidated report. +const SCHEMA_KINDS = new Set([ + "element-not-allowed", + "attribute-not-allowed", + "attribute-value-invalid", + "choice-not-satisfied", + "text-not-allowed", + "unexpected-end", + "well-formedness", + "other", +]); + +const [schemaPath, sourcePath] = process.argv.slice(2); +if (!schemaPath || !sourcePath) { + console.error("usage: ptx-jing-shim "); + process.exit(2); +} + +/** + * Report a failure of the validator itself as a message *in the report*. + * + * Exiting with a code above 1 would leave core logging a warning and then + * treating the empty output as "no schema errors" -- a silent pass for a + * document nothing actually checked. A message on stdout is counted and + * printed like any other, so the failure cannot be mistaken for success. + */ +function fail(message) { + console.log(`${sourcePath}:1:1: error: the salve validator failed: ${message}`); + process.exit(1); +} + +/** + * A salve grammar for the schema core asked us to use, compiled on first use + * and cached beside this script. + * + * The grammar must come from the schema the CLI actually ships: the copy + * bundled with `@pretextbook/schema` tracks the VS Code extension's schema, + * which drifts from core's and then reports problems with valid documents. + */ +async function loadGrammarJson() { + const cacheDir = new URL("grammars/", import.meta.url); + const cacheFile = new URL(`${basename(schemaPath)}.json`, cacheDir); + try { + if (statSync(cacheFile).mtimeMs >= statSync(schemaPath).mtimeMs) { + return readFileSync(cacheFile, "utf8"); + } + } catch { + // No cache yet (or it is older than the schema): compile below. + } + const { compileRngToJSON } = await import("@pretextbook/schema/compile"); + const { json } = await compileRngToJSON(schemaPath); + try { + mkdirSync(fileURLToPath(cacheDir), { recursive: true }); + writeFileSync(cacheFile, json); + } catch { + // A read-only install just means recompiling (~0.6s) on every run. + } + return json; +} + +let validateDocument, loadGrammarFromJSON, grammarJson, source; +try { + ({ validateDocument, loadGrammarFromJSON } = await import("@pretextbook/schema")); +} catch (e) { + fail(`could not load @pretextbook/schema (${e}). Try \`pretext validate --engine salve\` again to reinstall it.`); +} +try { + grammarJson = await loadGrammarJson(); +} catch (e) { + fail(`could not compile a grammar from ${schemaPath} (${e})`); +} +try { + source = readFileSync(sourcePath, "utf8"); +} catch (e) { + fail(`could not read ${sourcePath} (${e})`); +} + +const { diagnostics } = validateDocument(source, loadGrammarFromJSON(grammarJson), { + uri: sourcePath, + // Core hands over a single, already-assembled file and wants line numbers *of + // that file*; both of these also keep the package from going out to disk. + resolveXIncludes: false, + readFile: () => undefined, +}); + +let count = 0; +for (const d of diagnostics) { + if (d.code !== undefined && !SCHEMA_KINDS.has(String(d.code))) continue; + // LSP positions are 0-based; jing's are 1-based. + const line = d.range.start.line + 1; + const column = d.range.start.character + 1; + console.log(`${sourcePath}:${line}:${column}: error: ${d.message}`); + count += 1; +} +process.exit(count > 0 ? 1 : 0); diff --git a/pretext/utils.py b/pretext/utils.py index 520b2f76..c34f8f11 100644 --- a/pretext/utils.py +++ b/pretext/utils.py @@ -1,6 +1,7 @@ import datetime from hashlib import sha256 import hashlib +import importlib.resources import os from collections.abc import Generator from contextlib import contextmanager @@ -12,7 +13,6 @@ import socketserver import socket import subprocess -import tempfile import time as time import logging import logging.handlers @@ -207,28 +207,56 @@ def xml_syntax_is_valid(xmlfile: Path, root_tag: str = "pretext") -> bool: return True -def xml_validates_against_schema(etree: _Element) -> bool: - schemarngfile = schema_path() - log.debug(f"Validating PreTeXt source against schema {schemarngfile}") - # Build-time validation stays lxml-first (fast) and warn-only. - is_valid, error_text = run_schema_validation( - etree, schemarngfile, order=("lxml", "jing") +def warn_on_schema_errors(etree: _Element, log_dir: Path) -> Optional[bool]: + """A quick, warn-only schema check to accompany a build or generate. + + Runs jing over the assembled source when jing is available, and otherwise + checks nothing (returning `None`) -- a build should not depend on having a + validator installed. Messages are written to a log file rather than the + terminal: they are numerous, they refer to lines of the *assembled* source + (deposited alongside the log so those numbers mean something), and the + consolidated report from `pretext validate` is the better place to read + them. Returns whether the source validated. + """ + schema_file = schema_path() + log.debug(f"Validating PreTeXt source against schema {schema_file}") + log_dir.mkdir(parents=True, exist_ok=True) + assembled_source = log_dir / "schema-assembled-source.xml" + error_log = log_dir / "schema-errors.log" + etree.getroottree().write( + str(assembled_source), encoding="utf-8", xml_declaration=True ) - if is_valid: - log.info("PreTeXt source passed schema validation.") - return True - if is_valid is None: - log.warning(error_text + " Continuing with build.") - else: + + def _discard() -> None: + # Leave nothing behind to be mistaken for the result of this run. + assembled_source.unlink(missing_ok=True) + error_log.unlink(missing_ok=True) + + result = validate_with_jing(assembled_source, schema_file) + if result is None: log.debug( - "PreTeXt document did not pass schema validation; unexpected output " - "may result. See .error_schema.log for hints. Continuing with build." + "No jing executable available, so the source was not checked against " + "the schema. Install jing (or set it in executables.ptx) to have " + "builds report schema errors." ) - log.debug( - "---- Schema validation error details: ----\n" - + error_text - + "\n---- End schema validation error details. ----" + _discard() + return None + + is_valid, output = result + if is_valid: + log.debug("PreTeXt source passed schema validation.") + _discard() + return True + + error_log.write_text(output, encoding="utf-8") + message_count = len([line for line in output.splitlines() if line.strip()]) + log.warning( + f"PreTeXt source did not pass schema validation ({message_count} messages); " + "unexpected output may result. Continuing anyway." ) + log.warning(f" Messages: {error_log}") + log.warning(f" Line numbers refer to the assembled source: {assembled_source}") + log.warning(" Run `pretext validate` for a report that names the source files.") return False @@ -252,78 +280,84 @@ def _jing_command() -> Optional[List[str]]: return [jing_executable] if jing_executable is not None else None -def _validate_with_jing( - etree: _Element, schema_file: Path +def salve_shim_command() -> Optional[List[str]]: + """The command that runs the salve-annos validator as a stand-in for jing. + + An experimental alternative engine, so that it can be compared with jing + without anyone having to install jing at all. The shim presents a + jing-compatible interface, so it drops into core's `validate()` as the + `jing` executable. Returns `None` (with a warning) if it can't be set up. + + Its npm dependencies are installed on first use into the CLI's resource + directory -- the same pattern as the other node packages the CLI needs. + """ + node_cmd = shutil.which("node") + if node_cmd is None: + log.warning("The salve validator needs node, which could not be found.") + return None + + shim_dir = resources.resource_base_path() / "salve" + shim = shim_dir / "ptx-jing-shim.mjs" + + if not (shim_dir / "node_modules").exists(): + npm_cmd = shutil.which("npm") + if npm_cmd is None: + log.warning("The salve validator needs npm to install itself on first use.") + return None + log.info(f"Installing the salve validator into {shim_dir} (first use only).") + shim_dir.mkdir(parents=True, exist_ok=True) + for name in ("ptx-jing-shim.mjs", "package.json"): + with importlib.resources.path("pretext.resources.salve", name) as src: + shutil.copy2(src, shim_dir / name) + try: + result = subprocess.run( + [npm_cmd, "install"], cwd=shim_dir, capture_output=True, text=True + ) + except OSError as e: + log.warning(f"Could not install the salve validator: {e}") + return None + if result.returncode != 0: + log.warning("Could not install the salve validator with npm:") + log.warning(result.stderr.strip()) + return None + + if not shim.exists(): + log.warning(f"The salve validator is missing from {shim_dir}.") + return None + return [node_cmd, str(shim)] + + +def validate_with_jing( + source_file: Path, schema_file: Path ) -> Optional[tuple[bool, str]]: + """Check an XML file against a RELAX-NG schema with jing. + + Returns `(is_valid, messages)`, or `None` when jing isn't available. jing is + the only engine: lxml cannot compile the PreTeXt schema, which incorporates + PreFigure's schema in a way libxml2 won't accept. + """ jing_command = _jing_command() if jing_command is None: return None - tmp_path: Optional[Path] = None try: - xml_payload = ET.tostring( - etree.getroottree(), encoding="utf-8", xml_declaration=True - ) - with tempfile.NamedTemporaryFile(suffix=".xml", delete=False) as tmp: - tmp.write(xml_payload) - tmp_path = Path(tmp.name) - result = subprocess.run( - jing_command + [str(schema_file), str(tmp_path)], + jing_command + [str(schema_file), str(source_file)], check=False, capture_output=True, text=True, ) - output = "\n".join( - chunk for chunk in [result.stdout.strip(), result.stderr.strip()] if chunk - ) - return result.returncode == 0, output except OSError: return None - finally: - if tmp_path is not None: - tmp_path.unlink(missing_ok=True) - - -def _validate_with_lxml( - etree: _Element, schema_file: Path -) -> Optional[tuple[bool, str]]: - # Returns None when lxml cannot compile the schema (the known - # "no define for ref" bug on some libxml2 builds), so the caller can - # fall back to another engine. - try: - relaxng = ET.RelaxNG(file=str(schema_file)) - except ET.RelaxNGParseError: - log.debug( - "lxml could not compile the RelaxNG schema; trying the next validator." - ) - return None - try: - relaxng.assertValid(etree) - return True, "" - except ET.DocumentInvalid as err: - return False, str(err.error_log) - - -def run_schema_validation( - etree: _Element, - schema_file: Path, - order: t.Sequence[str] = ("lxml", "jing"), -) -> tuple[Optional[bool], str]: - # Engines are looked up by name here (not captured at import time) so tests - # can monkeypatch `utils._validate_with_jing` / `utils._validate_with_lxml`. - engines = { - "lxml": _validate_with_lxml, - "jing": _validate_with_jing, - } - for engine_name in order: - result = engines[engine_name](etree, schema_file) - if result is not None: - return result - return None, ( - "Schema validation could not be completed: no validator was available " - "(jing is not installed and lxml could not compile the schema)." + output = "\n".join( + chunk for chunk in [result.stdout.strip(), result.stderr.strip()] if chunk ) + # jing exits 0 when the document is valid and 1 when it has messages; + # anything more means jing itself failed, so nothing was checked. + if result.returncode > 1: + log.debug(f"The jing program failed (code {result.returncode}): {output}") + return None + return result.returncode == 0, output def schema_path(dev: bool = False) -> Path: diff --git a/tests/test_cli.py b/tests/test_cli.py index 033f32e0..c98de093 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -35,8 +35,8 @@ import requests import pretext from lxml import etree as ET # noqa: N812 -from pretext import constants, server, utils -from typing import cast, Generator +from pretext import constants, resources, server, utils +from typing import cast, Generator, List import pytest from pytest_console_scripts import ScriptRunner @@ -734,17 +734,17 @@ def test_custom_webwork_server(tmp_path: Path, script_runner: ScriptRunner) -> N # --------------------------------------------------------------------------- # 5. Validate # +# `pretext validate` runs core's `validate`, which writes a consolidated report +# (jing's schema messages plus the "validation-plus" stylesheet's) into `logs/`. # Exit code contract (see the validate command's docstring in pretext/cli.py): -# 0 = valid, 1 = invalid or malformed, 2 = no validator available. +# 0 = valid, 1 = invalid or malformed, 2 = validation could not be performed. # --------------------------------------------------------------------------- def _validator_available() -> bool: - """True if either lxml or jing can run against the bundled schema.""" - result = utils.run_schema_validation( - ET.fromstring(""), utils.schema_path(), order=("lxml", "jing") - ) - return result[0] is not None + """True if jing can run: it is the only engine, since lxml cannot compile + the PreTeXt schema (it incorporates PreFigure's).""" + return utils._jing_command() is not None def _make_project(tmp_path: Path, script_runner: ScriptRunner) -> Path: @@ -753,9 +753,7 @@ def _make_project(tmp_path: Path, script_runner: ScriptRunner) -> Path: return tmp_path / "new-pretext-project" -@pytest.mark.skipif( - not _validator_available(), reason="no RelaxNG validator (lxml/jing) available" -) +@pytest.mark.skipif(not _validator_available(), reason="jing is not available") def test_validate_invalid_source_is_nonzero( tmp_path: Path, script_runner: ScriptRunner ) -> None: @@ -781,9 +779,7 @@ def test_validate_malformed_xml_is_nonzero( assert ret.returncode == 1 -@pytest.mark.skipif( - not _validator_available(), reason="no RelaxNG validator (lxml/jing) available" -) +@pytest.mark.skipif(not _validator_available(), reason="jing is not available") def test_validate_dev_schema_runs(tmp_path: Path, script_runner: ScriptRunner) -> None: """`pretext validate --dev` validates against the dev schema; a fresh template project passes.""" @@ -795,25 +791,94 @@ def test_validate_dev_schema_runs(tmp_path: Path, script_runner: ScriptRunner) - def test_validate_could_not_validate_exits_2( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, script_runner: ScriptRunner ) -> None: - """When no validation engine is available at all, `pretext validate` - exits 2 to distinguish "could not check" from "invalid".""" + """With no jing to run, `pretext validate` exits 2 to distinguish + "could not check" from "invalid".""" from click.testing import CliRunner from pretext import cli from pretext import utils as putils + from pretext.core import common as core_common project = _make_project(tmp_path, script_runner) monkeypatch.chdir(project) - # Avoid the network update check and force the "no validator available" result. + # Avoid the network update check, and make jing (and only jing) unavailable, + # as core sees it. monkeypatch.setattr(putils, "check_for_updates", lambda *a, **k: None) - monkeypatch.setattr( - putils, - "run_schema_validation", - lambda *a, **k: (None, "no validator available"), - ) + real_get_executable_cmd = core_common.get_executable_cmd + + def _no_jing(exec_name: str) -> List[str]: + if exec_name == "jing": + raise OSError("cannot locate executable with configuration name `jing`") + return real_get_executable_cmd(exec_name) + + monkeypatch.setattr(core_common, "get_executable_cmd", _no_jing) result = CliRunner().invoke(cli.main, ["validate"]) assert result.exit_code == 2 +def test_validate_engine_salve_unavailable_exits_2( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, script_runner: ScriptRunner +) -> None: + """Asking for the salve engine when it can't be set up exits 2, rather than + silently validating with jing and reporting on the wrong engine.""" + from click.testing import CliRunner + from pretext import cli + from pretext import utils as putils + + project = _make_project(tmp_path, script_runner) + monkeypatch.chdir(project) + monkeypatch.setattr(putils, "check_for_updates", lambda *a, **k: None) + monkeypatch.setattr(putils, "salve_shim_command", lambda: None) + result = CliRunner().invoke(cli.main, ["validate", "--engine", "salve"]) + assert result.exit_code == 2 + + +@pytest.mark.skipif( + not (resources.resource_base_path() / "salve" / "node_modules").exists(), + reason="the salve engine has not been installed (run `pretext validate --engine salve` once)", +) +def test_validate_engine_salve_end_to_end( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, script_runner: ScriptRunner +) -> None: + """A template project validates cleanly through the salve engine too. Skipped + until a developer has installed it, since first use needs npm and a network.""" + from click.testing import CliRunner + from pretext import cli + from pretext import utils as putils + + project = _make_project(tmp_path, script_runner) + monkeypatch.chdir(project) + monkeypatch.setattr(putils, "check_for_updates", lambda *a, **k: None) + # `cli.error_flush_handler` is a module global, so errors logged by an + # earlier in-process invocation would make this run exit 1 on their account. + # A real run is a fresh process; here the buffer has to be emptied by hand. + cli.error_flush_handler.buffer.clear() + result = CliRunner().invoke(cli.main, ["validate", "--engine", "salve"]) + assert result.exit_code == 0 + assert (project / "logs" / "main-validation.txt").exists() + + +@pytest.mark.skipif(not _validator_available(), reason="jing is not available") +def test_validate_terse_method_is_machine_readable( + tmp_path: Path, script_runner: ScriptRunner +) -> None: + """`--method terse` writes core's tab-separated report: one message per + line, each naming the source file it came from.""" + project = _make_project(tmp_path, script_runner) + main_src = project / "source" / "main.ptx" + main_src.write_text( + '\n
Text outside of element.
\n' + ) + ret = script_runner.run([PTX_CMD, "validate", "--method", "terse"], cwd=project) + assert ret.returncode == 1 + report = (project / "logs" / "main-validation.txt").read_text() + lines = [line for line in report.splitlines() if line.strip()] + assert len(lines) > 0 + for line in lines: + fields = line.split("\t") + assert len(fields) == 5 + assert fields[0] == "main.ptx" + + # --------------------------------------------------------------------------- # 6. View (preview server) # --------------------------------------------------------------------------- diff --git a/tests/test_project.py b/tests/test_project.py index ff84b1d2..56ce158b 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -377,6 +377,52 @@ def test_executables_jing_from_manifest(tmp_path: Path) -> None: assert project._executables.model_dump()["jing"] == "java -jar /opt/jing.jar" +def test_validate_source_salve_engine_swaps_and_restores_jing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The salve engine reaches core by standing in for the `jing` executable, + and the project's own executables are restored once validation is done (so + a later build isn't left pointing at the shim).""" + prj_path = tmp_path / "simple" + shutil.copytree(EXAMPLES_DIR / "projects" / "project_refactor" / "simple", prj_path) + with utils.working_directory(prj_path): + project = pr.Project.parse() + target = project.get_target() + + handed_to_core = [] + monkeypatch.setattr( + pr.utils, "salve_shim_command", lambda: ["node", "/x/shim.mjs"] + ) + monkeypatch.setattr( + pr.core, "set_executables", lambda d: handed_to_core.append(dict(d)) + ) + monkeypatch.setattr(pr.core, "validate", lambda **kwargs: None) + + target.validate_source(engine="salve") + + assert handed_to_core[0]["jing"] == "node /x/shim.mjs" + assert handed_to_core[-1]["jing"] == "jing" + + +def test_validate_source_salve_unavailable_returns_none( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """With no usable shim, validation reports that it could not run (exit 2 at + the CLI) instead of falling back to jing and quietly testing the wrong thing.""" + prj_path = tmp_path / "simple" + shutil.copytree(EXAMPLES_DIR / "projects" / "project_refactor" / "simple", prj_path) + with utils.working_directory(prj_path): + project = pr.Project.parse() + + monkeypatch.setattr(pr.utils, "salve_shim_command", lambda: None) + + def _unreachable(**kwargs: object) -> None: + raise AssertionError("core.validate should not be called") + + monkeypatch.setattr(pr.core, "validate", _unreachable) + assert project.get_target().validate_source(engine="salve") is None + + def test_executables_deprecated_and_unknown() -> None: """Executables core no longer uses are accepted (older `executables.ptx` files must keep parsing) but withheld from core; genuinely unknown ones diff --git a/tests/test_utils.py b/tests/test_utils.py index 3de20d15..7c10bf21 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -10,6 +10,7 @@ import fnmatch import os import sys +import typing as t import pytest from pathlib import Path from lxml import etree as ET # noqa: N812 @@ -233,35 +234,6 @@ def test_xml_syntax_is_valid(tmp_path: Path) -> None: utils.xml_syntax_is_valid(tmp_path / "nonexistent.ptx") -def test_validate_with_lxml_valid_and_invalid(tmp_path: Path) -> None: - schema_file = tmp_path / "mini.rng" - schema_file.write_text( - '' - '' - ) - - ok = utils._validate_with_lxml(ET.fromstring(""), schema_file) - assert ok == (True, "") - - bad = utils._validate_with_lxml(ET.fromstring(""), schema_file) - assert bad is not None - assert bad[0] is False - assert bad[1] != "" - - -def test_validate_with_lxml_uncompilable_schema_returns_none( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - def _raise(*args: object, **kwargs: object) -> object: - raise ET.RelaxNGParseError("boom") - - monkeypatch.setattr(utils.ET, "RelaxNG", _raise) - assert ( - utils._validate_with_lxml(ET.fromstring(""), tmp_path / "x.rng") - is None - ) - - def test_jing_command_prefers_configured_executable( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -295,8 +267,8 @@ def _raise(name: str) -> object: monkeypatch.setattr(utils.core, "get_executable_cmd", _raise) monkeypatch.setattr(utils.shutil, "which", lambda name: None) assert utils._jing_command() is None - # And the engine reports itself unavailable, so the caller can try another. - assert utils._validate_with_jing(ET.fromstring(""), Path("s.rng")) is None + # And the engine reports itself unavailable, rather than "invalid". + assert utils.validate_with_jing(Path("source.xml"), Path("s.rng")) is None def test_validate_with_jing_invokes_configured_command( @@ -320,52 +292,114 @@ def _run(cmd: list[str], **kwargs: object) -> _Result: monkeypatch.setattr(utils.subprocess, "run", _run) schema_file = tmp_path / "mini.rng" - assert utils._validate_with_jing(ET.fromstring(""), schema_file) == ( - True, - "", - ) - assert recorded[0][:3] == ["java", "-jar", "/opt/jing.jar"] - assert recorded[0][3] == str(schema_file) + source_file = tmp_path / "source.xml" + assert utils.validate_with_jing(source_file, schema_file) == (True, "") + assert recorded[0] == [ + "java", + "-jar", + "/opt/jing.jar", + str(schema_file), + str(source_file), + ] + + +@pytest.mark.parametrize( + "returncode, expected", + # 0 = valid, 1 = the document has messages, anything more = jing itself + # failed, which is "nothing was checked" rather than "invalid". + [(0, (True, "boom")), (1, (False, "boom")), (2, None), (127, None)], +) +def test_validate_with_jing_maps_exit_codes( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + returncode: int, + expected: t.Optional[tuple[bool, str]], +) -> None: + class _Result: + stdout = "boom" + stderr = "" + + _Result.returncode = returncode # type: ignore[attr-defined] + + monkeypatch.setattr(utils, "_jing_command", lambda: ["jing"]) + monkeypatch.setattr(utils.subprocess, "run", lambda *a, **k: _Result()) + assert utils.validate_with_jing(tmp_path / "s.xml", tmp_path / "s.rng") == expected + + +def test_salve_shim_command_needs_node(monkeypatch: pytest.MonkeyPatch) -> None: + """Without node there is no salve engine -- and asking for it must not have + the side effect of installing the CLI's resources.""" + monkeypatch.setattr(utils.shutil, "which", lambda name: None) + + def _no_resources() -> Path: + raise AssertionError("resource_base_path() should not be reached") + + monkeypatch.setattr(utils.resources, "resource_base_path", _no_resources) + assert utils.salve_shim_command() is None -def test_run_schema_validation_uses_first_available_engine( +def test_salve_shim_command_needs_npm_on_first_use( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - monkeypatch.setattr(utils, "_validate_with_jing", lambda *a: (True, "")) + """The shim installs itself with npm the first time; without npm, the engine + is simply unavailable rather than half installed.""" monkeypatch.setattr( - utils, "_validate_with_lxml", lambda *a: (False, "lxml says no") + utils.shutil, "which", lambda name: "/usr/bin/node" if name == "node" else None ) + monkeypatch.setattr(utils.resources, "resource_base_path", lambda: tmp_path) + assert utils.salve_shim_command() is None + assert not (tmp_path / "salve" / "node_modules").exists() - # jing first wins - assert utils.run_schema_validation( - ET.fromstring(""), tmp_path / "s.rng", order=("jing", "lxml") - ) == (True, "") - # lxml first wins - assert utils.run_schema_validation( - ET.fromstring(""), tmp_path / "s.rng", order=("lxml", "jing") - ) == (False, "lxml says no") - -def test_run_schema_validation_skips_unavailable_engine( +def test_salve_shim_command_when_installed( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - monkeypatch.setattr(utils, "_validate_with_jing", lambda *a: None) - monkeypatch.setattr(utils, "_validate_with_lxml", lambda *a: (True, "")) - assert utils.run_schema_validation( - ET.fromstring(""), tmp_path / "s.rng", order=("jing", "lxml") - ) == (True, "") + """Once installed, the command is `node ` -- the shape core needs for + an executable it will call with a schema and a source file.""" + shim_dir = tmp_path / "salve" + (shim_dir / "node_modules").mkdir(parents=True) + (shim_dir / "ptx-jing-shim.mjs").write_text("// shim") + monkeypatch.setattr(utils.shutil, "which", lambda name: f"/usr/bin/{name}") + monkeypatch.setattr(utils.resources, "resource_base_path", lambda: tmp_path) + assert utils.salve_shim_command() == [ + "/usr/bin/node", + str(shim_dir / "ptx-jing-shim.mjs"), + ] -def test_run_schema_validation_all_unavailable_returns_none( +def test_warn_on_schema_errors_writes_log_not_terminal( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: - monkeypatch.setattr(utils, "_validate_with_jing", lambda *a: None) - monkeypatch.setattr(utils, "_validate_with_lxml", lambda *a: None) - result = utils.run_schema_validation( - ET.fromstring(""), tmp_path / "s.rng", order=("jing", "lxml") + """A build's schema check leaves the messages in a log file, alongside the + assembled source their line numbers refer to.""" + monkeypatch.setattr( + utils, "validate_with_jing", lambda source, schema: (False, "line 4: bad") ) - assert result[0] is None - assert "could not be completed" in result[1] + assert utils.warn_on_schema_errors(ET.fromstring(""), tmp_path) is False + assert (tmp_path / "schema-errors.log").read_text() == "line 4: bad" + assert (tmp_path / "schema-assembled-source.xml").exists() + + +def test_warn_on_schema_errors_cleans_up_when_valid( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """A clean run clears the artifacts of an earlier failed one, so no stale + log is left to be read as the result of this build.""" + (tmp_path / "schema-errors.log").write_text("errors from a previous build") + monkeypatch.setattr(utils, "validate_with_jing", lambda source, schema: (True, "")) + assert utils.warn_on_schema_errors(ET.fromstring(""), tmp_path) is True + assert not (tmp_path / "schema-errors.log").exists() + assert not (tmp_path / "schema-assembled-source.xml").exists() + + +def test_warn_on_schema_errors_without_jing( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + """No jing means nothing was checked -- which must not look like success.""" + monkeypatch.setattr(utils, "validate_with_jing", lambda source, schema: None) + assert utils.warn_on_schema_errors(ET.fromstring(""), tmp_path) is None + assert not (tmp_path / "schema-errors.log").exists() + assert not (tmp_path / "schema-assembled-source.xml").exists() def test_schema_path_selects_stable_or_dev() -> None: From b52917803d3f398073dd59d45aceb23284e11699 Mon Sep 17 00:00:00 2001 From: Oscar Levin Date: Sat, 1 Aug 2026 19:21:15 -0600 Subject: [PATCH 06/13] flake --- tests/test_cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index c98de093..98e57526 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -34,7 +34,6 @@ from contextlib import contextmanager import requests import pretext -from lxml import etree as ET # noqa: N812 from pretext import constants, resources, server, utils from typing import cast, Generator, List import pytest From bced150593cb1d7886b720618f7584ef921505cc Mon Sep 17 00:00:00 2001 From: Oscar Levin Date: Sat, 1 Aug 2026 19:25:10 -0600 Subject: [PATCH 07/13] fix types --- pretext/project/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pretext/project/__init__.py b/pretext/project/__init__.py index 2ee13b9b..8284388d 100644 --- a/pretext/project/__init__.py +++ b/pretext/project/__init__.py @@ -532,7 +532,7 @@ def _read_publication_file_subset(self) -> PublicationSubset: p_bytes = ET.tostring(p_et) return PublicationSubset.from_xml(p_bytes) - def _get_managed_directories(self) -> t.Dict[Path, Path]: + def _get_managed_directories(self) -> t.Dict[str, Path]: generated, external = core.get_managed_directories( xml_source=self.source_abspath(), pub_file=self.publication_abspath().as_posix(), From 2384f57e009af46b8482782786843acf23862e2f Mon Sep 17 00:00:00 2001 From: Oscar Levin Date: Sat, 1 Aug 2026 20:22:41 -0600 Subject: [PATCH 08/13] fixes to managed directory detection --- pretext/project/__init__.py | 75 +++++++++++++++---- templates/article/assets/.gitkeep | 0 templates/book/assets/.gitkeep | 0 templates/course/assets/.gitkeep | 0 templates/demo/assets/.gitkeep | 0 templates/slideshow/assets/.gitkeep | 0 .../publication/publication.ptx | 2 +- .../custom-xsl/publication/publication.ptx | 2 +- .../datafile/publication/publication.ptx | 2 +- .../graphics/publication/publication.ptx | 2 +- .../interactive/publication/publication.ptx | 2 +- .../latex-image/publication/publication.ptx | 1 - .../assets/publication/publication.ptx | 3 +- .../legacy/publication/publication.ptx | 2 +- .../legacy_extra/publication/publication.ptx | 2 +- .../projects/xref/publication/publication.ptx | 2 +- tests/test_project.py | 2 +- 17 files changed, 69 insertions(+), 28 deletions(-) create mode 100644 templates/article/assets/.gitkeep create mode 100644 templates/book/assets/.gitkeep create mode 100644 templates/course/assets/.gitkeep create mode 100644 templates/demo/assets/.gitkeep create mode 100644 templates/slideshow/assets/.gitkeep diff --git a/pretext/project/__init__.py b/pretext/project/__init__.py index 8284388d..5f7700dc 100644 --- a/pretext/project/__init__.py +++ b/pretext/project/__init__.py @@ -141,6 +141,10 @@ class Target(pxml.BaseXmlModel, tag="target", search_mode=SearchMode.UNORDERED): source: Path = pxml.attr(default=Path("main.ptx")) # Cache of assembled "version only" source element. _source_element: t.Optional[ET._Element] = None + # Cache of the declared (generated, external) managed directories. Reading + # the external directory means an xinclude-processing parse of the source, + # so it is worth doing only once per target. + _declared_dirs: t.Optional[t.Tuple[Path, t.Optional[Path]]] = None # Whether the warn-only schema check has already run for this target, so a # build doesn't repeat it when it generates assets. _schema_checked: bool = False @@ -532,27 +536,66 @@ def _read_publication_file_subset(self) -> PublicationSubset: p_bytes = ET.tostring(p_et) return PublicationSubset.from_xml(p_bytes) - def _get_managed_directories(self) -> t.Dict[str, Path]: - generated, external = core.get_managed_directories( - xml_source=self.source_abspath(), - pub_file=self.publication_abspath().as_posix(), - ) - return {"generated": generated, "external": external} - - def external_dir(self) -> Path: - return self._get_managed_directories()["external"] - - def external_dir_abspath(self) -> Path: - return (self.source_abspath().parent / self.external_dir()).resolve() + def _declared_external_dir(self) -> t.Optional[str]: + """ + The raw value of the external directory attribute, as declared, or None. + + As of core's 2026-07-30 change the external directory is a fact of the + source, declared as `directories/@external` within `docinfo`. A + publication file `source/directories/@external` is deprecated but still + honored while the docinfo is silent, so we check both, in that order. + Core issues the deprecation warning when it reads the same values, so + this stays quiet. + """ + src_tree = ET.parse(self.source_abspath()) + # docinfo is very often xincluded, so includes must be processed. + src_tree.xinclude() + for directories in src_tree.xpath("/pretext/docinfo/directories"): + if "external" in directories.attrib: + return str(directories.attrib["external"]) + pub_tree = ET.parse(self.publication_abspath()) + pub_tree.xinclude() + for directories in pub_tree.xpath("/publication/source/directories"): + if "external" in directories.attrib: + return str(directories.attrib["external"]) + return None + + def _declared_managed_directories(self) -> t.Tuple[Path, t.Optional[Path]]: + """ + The (generated, external) managed directories as *declared*, in absolute + form, without checking whether they exist. `external` is None when no + external directory is declared at all. + + This deliberately duplicates the reading half of + `core.get_managed_directories`, which the CLI cannot use for this + purpose: that function also verifies the directories exist and raises + if they do not, but creating them is precisely the CLI's job here. + Anything invalid but existence-independent (an absolute path, say) is + still caught by core when it reads the same declarations later, so we + do not re-check it. + """ + if self._declared_dirs is None: + source_dir = self.source_abspath().parent + generated = self._read_publication_file_subset().generated + external = self._declared_external_dir() + self._declared_dirs = ( + (source_dir / generated).resolve(), + None if external is None else (source_dir / external).resolve(), + ) + return self._declared_dirs - def generated_dir(self) -> Path: - return self._get_managed_directories()["generated"] + def external_dir_abspath(self) -> t.Optional[Path]: + return self._declared_managed_directories()[1] def generated_dir_abspath(self) -> Path: - return (self.source_abspath().parent / self.generated_dir()).resolve() + return self._declared_managed_directories()[0] def ensure_asset_directories(self, asset: t.Optional[str] = None) -> None: - self.external_dir_abspath().mkdir(parents=True, exist_ok=True) + # Only the generated directory is ours to create. The external + # directory holds files the author supplies, so core is left to object + # when a declared one is missing: that way a mistyped path is reported + # as the error it is, rather than silently becoming an empty directory. + # An author with no external assets should drop the attribute entirely. self.generated_dir_abspath().mkdir(parents=True, exist_ok=True) if asset is not None: # make directories for each asset type that would be generated from "asset": diff --git a/templates/article/assets/.gitkeep b/templates/article/assets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/templates/book/assets/.gitkeep b/templates/book/assets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/templates/course/assets/.gitkeep b/templates/course/assets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/templates/demo/assets/.gitkeep b/templates/demo/assets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/templates/slideshow/assets/.gitkeep b/templates/slideshow/assets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/examples/projects/custom-wwserver/publication/publication.ptx b/tests/examples/projects/custom-wwserver/publication/publication.ptx index 13fc0bf1..48df6f00 100644 --- a/tests/examples/projects/custom-wwserver/publication/publication.ptx +++ b/tests/examples/projects/custom-wwserver/publication/publication.ptx @@ -2,7 +2,7 @@ - + diff --git a/tests/examples/projects/custom-xsl/publication/publication.ptx b/tests/examples/projects/custom-xsl/publication/publication.ptx index 35f12844..083b57a5 100644 --- a/tests/examples/projects/custom-xsl/publication/publication.ptx +++ b/tests/examples/projects/custom-xsl/publication/publication.ptx @@ -2,6 +2,6 @@ - + diff --git a/tests/examples/projects/datafile/publication/publication.ptx b/tests/examples/projects/datafile/publication/publication.ptx index 35f12844..083b57a5 100644 --- a/tests/examples/projects/datafile/publication/publication.ptx +++ b/tests/examples/projects/datafile/publication/publication.ptx @@ -2,6 +2,6 @@ - + diff --git a/tests/examples/projects/graphics/publication/publication.ptx b/tests/examples/projects/graphics/publication/publication.ptx index 35f12844..083b57a5 100644 --- a/tests/examples/projects/graphics/publication/publication.ptx +++ b/tests/examples/projects/graphics/publication/publication.ptx @@ -2,6 +2,6 @@ - + diff --git a/tests/examples/projects/interactive/publication/publication.ptx b/tests/examples/projects/interactive/publication/publication.ptx index 35f12844..083b57a5 100644 --- a/tests/examples/projects/interactive/publication/publication.ptx +++ b/tests/examples/projects/interactive/publication/publication.ptx @@ -2,6 +2,6 @@ - + diff --git a/tests/examples/projects/latex-image/publication/publication.ptx b/tests/examples/projects/latex-image/publication/publication.ptx index 828f15c4..a8b1b6f0 100644 --- a/tests/examples/projects/latex-image/publication/publication.ptx +++ b/tests/examples/projects/latex-image/publication/publication.ptx @@ -1,7 +1,6 @@ diff --git a/tests/examples/projects/project_refactor/assets/publication/publication.ptx b/tests/examples/projects/project_refactor/assets/publication/publication.ptx index 434166a6..df12a4d4 100644 --- a/tests/examples/projects/project_refactor/assets/publication/publication.ptx +++ b/tests/examples/projects/project_refactor/assets/publication/publication.ptx @@ -1,7 +1,6 @@ - \ No newline at end of file + diff --git a/tests/examples/projects/project_refactor/legacy/publication/publication.ptx b/tests/examples/projects/project_refactor/legacy/publication/publication.ptx index 35f12844..083b57a5 100644 --- a/tests/examples/projects/project_refactor/legacy/publication/publication.ptx +++ b/tests/examples/projects/project_refactor/legacy/publication/publication.ptx @@ -2,6 +2,6 @@ - + diff --git a/tests/examples/projects/project_refactor/legacy_extra/publication/publication.ptx b/tests/examples/projects/project_refactor/legacy_extra/publication/publication.ptx index 35f12844..083b57a5 100644 --- a/tests/examples/projects/project_refactor/legacy_extra/publication/publication.ptx +++ b/tests/examples/projects/project_refactor/legacy_extra/publication/publication.ptx @@ -2,6 +2,6 @@ - + diff --git a/tests/examples/projects/xref/publication/publication.ptx b/tests/examples/projects/xref/publication/publication.ptx index 35f12844..083b57a5 100644 --- a/tests/examples/projects/xref/publication/publication.ptx +++ b/tests/examples/projects/xref/publication/publication.ptx @@ -2,6 +2,6 @@ - + diff --git a/tests/test_project.py b/tests/test_project.py index 56ce158b..e35ff23a 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -507,7 +507,7 @@ def test_xinclude_publication_build(tmp_path: Path) -> None: with utils.working_directory(prj_path): project = pr.Project.parse() target = project.get_target("web") - assert target.external_dir() == Path("../assets") + assert target.external_dir_abspath() == (prj_path / "assets").resolve() target.build() assert (target.output_dir_abspath() / "index.html").exists() From 3fd459ecccd48e6049615048397b9feb805f80bb Mon Sep 17 00:00:00 2001 From: Oscar Levin Date: Sat, 1 Aug 2026 20:30:08 -0600 Subject: [PATCH 09/13] fix types --- pretext/project/__init__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pretext/project/__init__.py b/pretext/project/__init__.py index 5f7700dc..b83bae86 100644 --- a/pretext/project/__init__.py +++ b/pretext/project/__init__.py @@ -550,14 +550,16 @@ def _declared_external_dir(self) -> t.Optional[str]: src_tree = ET.parse(self.source_abspath()) # docinfo is very often xincluded, so includes must be processed. src_tree.xinclude() - for directories in src_tree.xpath("/pretext/docinfo/directories"): - if "external" in directories.attrib: - return str(directories.attrib["external"]) + src_root = src_tree.getroot() + src_external = src_root.xpath("./docinfo/directories/@external") + if src_external: + return str(src_external) pub_tree = ET.parse(self.publication_abspath()) pub_tree.xinclude() - for directories in pub_tree.xpath("/publication/source/directories"): - if "external" in directories.attrib: - return str(directories.attrib["external"]) + pub_root = pub_tree.getroot() + pub_external = pub_root.xpath("./source/directories/@external") + if pub_external: + return str(pub_external) return None def _declared_managed_directories(self) -> t.Tuple[Path, t.Optional[Path]]: From 47662f62ca77b15477180696e8a46903b1df731f Mon Sep 17 00:00:00 2001 From: Oscar Levin Date: Sun, 2 Aug 2026 17:31:08 -0600 Subject: [PATCH 10/13] fix another test --- pretext/__init__.py | 2 +- .../projects/custom-wwserver/assets/.gitkeep | 0 .../webwork/pg/Hello_World/1.pg | 80 ++++++++++++++ .../pg/Hello_World/def/setHello_World.def | 11 ++ .../pg/Hello_World/header/Hello_World.pg | 40 +++++++ .../pg/Hello_World/macros/Hello_World.pl | 4 + .../generated-assets/webwork/root-1-1-3.xml | 100 ++++++++++++++++++ .../projects/custom-wwserver/project.ptx | 33 +----- .../projects/custom-wwserver/source/main.ptx | 3 + .../projects/datafile/source/main.ptx | 3 + 10 files changed, 244 insertions(+), 32 deletions(-) create mode 100644 tests/examples/projects/custom-wwserver/assets/.gitkeep create mode 100644 tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/1.pg create mode 100644 tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/def/setHello_World.def create mode 100644 tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/header/Hello_World.pg create mode 100644 tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/macros/Hello_World.pl create mode 100644 tests/examples/projects/custom-wwserver/generated-assets/webwork/root-1-1-3.xml diff --git a/pretext/__init__.py b/pretext/__init__.py index 8d4ed671..32f6378c 100644 --- a/pretext/__init__.py +++ b/pretext/__init__.py @@ -18,7 +18,7 @@ VERSION = get_version("pretext", Path(__file__).parent.parent) -CORE_COMMIT = "8877b5771ff4e20d369f07378489713a3ac57665" +CORE_COMMIT = "b02c1279be5bcf2d959dce324f2adcc0496c7dc5" def activate() -> None: diff --git a/tests/examples/projects/custom-wwserver/assets/.gitkeep b/tests/examples/projects/custom-wwserver/assets/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/1.pg b/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/1.pg new file mode 100644 index 00000000..cfc0314a --- /dev/null +++ b/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/1.pg @@ -0,0 +1,80 @@ +############################################# +### Generated from PreTeXt source +### +### https://pretextbook.org +### +############################################# +## DBsubject() +## DBchapter() +## DBsection() +## Level() +## KEYWORDS() +## TitleText1() +## EditionText1() +## AuthorText1() +## Section1(not reported) +## Problem1(1) +## Author() +## Institution() +## Language(en-US) + +DOCUMENT(); + +############################################################ +# Load Macros +############################################################ +loadMacros( + "PGstandard.pl", + "PGML.pl", + "AnswerFormatHelp.pl", + "PGcourse.pl", +); +COMMENT('Authored in PreTeXt'); + +############################################################ +# Header +############################################################ + +############################################################ +# PG Setup Code +############################################################ +Context('Numeric'); +$a = Compute(random(1, 9, 1)); +$b = Compute(random(1, 9, 1)); +$c = $a + $b; + +############################################################ +# Body +############################################################ + +BEGIN_PGML +Compute the sum of [`[$a]`] and [`[$b]\text{:}`] + +[`[$a] + [$b] =`] [_]{$c}{5} + +END_PGML + +############################################################ +# Hint +############################################################ +#Set value of $showHint in PGcourse.pl for course-wide attempt threshhold for revealing hints + +BEGIN_PGML_HINT +Add [`[$a]`] and [`[$b]`] together. + +END_PGML_HINT + +############################################################ +# Solution +############################################################ + +BEGIN_PGML_SOLUTION +[`[$a] + [$b] = [$c]\text{.}`] + +END_PGML_SOLUTION + +############################################################ +# End Problem +############################################################ + +ENDDOCUMENT(); diff --git a/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/def/setHello_World.def b/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/def/setHello_World.def new file mode 100644 index 00000000..b9a5a34b --- /dev/null +++ b/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/def/setHello_World.def @@ -0,0 +1,11 @@ +openDate = 01/01/2027 at 12:00am +dueDate = 12/31/2027 at 10:00pm +answerDate = 12/31/2027 at 10:00pm +paperHeaderFile = Hello_World/header/Hello_World.pg +screenHeaderFile = Hello_World/header/Hello_World.pg +description = Hello World! +problemListV2 +problem_start +source_file = Hello_World/1.pg +problem_id = 1 +problem_end diff --git a/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/header/Hello_World.pg b/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/header/Hello_World.pg new file mode 100644 index 00000000..a6caf870 --- /dev/null +++ b/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/header/Hello_World.pg @@ -0,0 +1,40 @@ +# Header file for problem set Hello_World +# This header file can be used for both the screen and hardcopy output + +DOCUMENT(); + +loadMacros( + "PG.pl", + "PGbasicmacros.pl", + "PGML.pl", + "PGcourse.pl", +); + +TEXT($BEGIN_ONE_COLUMN); + +$texTopLine = "\noindent {\large \bf $studentName}\hfill{\large \bf {".protect_underbar($courseName)."}}"; +if (defined($sectionName) and ($sectionName ne '')) {$texTopLine .= " {\large \bf { Section: ".protect_underbar($sectionName)." } }"}; +$texTopLine .= "\par"; + +#################################################### +# +# MODES provides for distinct output for TeX and HTML +# +#################################################### + +TEXT(MODES( + TeX =>"$texTopLine", + HTML=>"", +)); + +TEXT(MODES( + TeX =>"\noindent{\large \sc {Assignment ".protect_underbar($setNumber)." due $formatedDueDate}}\par". + "\noindent \bigskip ", + HTML=>"WeBWorK Assignment ".protect_underbar($setNumber)." is due: $formattedDueDate. $PAR", +)); + +TEXT("This assignment contains exercises from Article of Hello World!."); + +TEXT($END_ONE_COLUMN); + +ENDDOCUMENT(); diff --git a/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/macros/Hello_World.pl b/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/macros/Hello_World.pl new file mode 100644 index 00000000..559e261d --- /dev/null +++ b/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/macros/Hello_World.pl @@ -0,0 +1,4 @@ +############################################################################# +# This macro library supports WeBWorK problems from the PreTeXt project named +# Hello World! +############################################################################# diff --git a/tests/examples/projects/custom-wwserver/generated-assets/webwork/root-1-1-3.xml b/tests/examples/projects/custom-wwserver/generated-assets/webwork/root-1-1-3.xml new file mode 100644 index 00000000..f8b0b724 --- /dev/null +++ b/tests/examples/projects/custom-wwserver/generated-assets/webwork/root-1-1-3.xml @@ -0,0 +1,100 @@ + + + +

Compute the sum of {1} and {7}\text{:}

+

{1} + {7} =

+ +

Add {1} and {7} together.

+ + +

+ 8 +

+ +

{1} + {7} = {8}\text{.}

+ + + + + diff --git a/tests/examples/projects/custom-wwserver/project.ptx b/tests/examples/projects/custom-wwserver/project.ptx index ea237f6a..b2cc0cda 100644 --- a/tests/examples/projects/custom-wwserver/project.ptx +++ b/tests/examples/projects/custom-wwserver/project.ptx @@ -4,37 +4,8 @@ project. To edit the content of your document, open `source/main.ptx` (default location). --> - + - - html - source/main.ptx - publication/publication.ptx - output/html - - - latex - source/main.ptx - publication/publication.ptx - output/latex - - - pdf - source/main.ptx - publication/publication.ptx - output/pdf - + - - latex - pdflatex - xelatex - pdf2svg - asy - sage - convert - pdftops - node - file2brl - diff --git a/tests/examples/projects/custom-wwserver/source/main.ptx b/tests/examples/projects/custom-wwserver/source/main.ptx index 7519cdf8..6572f594 100644 --- a/tests/examples/projects/custom-wwserver/source/main.ptx +++ b/tests/examples/projects/custom-wwserver/source/main.ptx @@ -1,5 +1,8 @@ + + +
Hello World!

This is a PreTeXt document.

diff --git a/tests/examples/projects/datafile/source/main.ptx b/tests/examples/projects/datafile/source/main.ptx index a68f9255..fdaa1a90 100644 --- a/tests/examples/projects/datafile/source/main.ptx +++ b/tests/examples/projects/datafile/source/main.ptx @@ -1,6 +1,9 @@ + + + Datafile Test From 692ecb04fd13c0e8802166fbd3fc54d6cbaae9bb Mon Sep 17 00:00:00 2001 From: Oscar Levin Date: Sun, 2 Aug 2026 18:09:29 -0600 Subject: [PATCH 11/13] fix bug --- pretext/project/__init__.py | 6 ++++-- tests/examples/projects/xi_pub/publication/source.ptx | 2 +- tests/examples/projects/xi_pub/source/main.ptx | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/pretext/project/__init__.py b/pretext/project/__init__.py index b83bae86..a1085147 100644 --- a/pretext/project/__init__.py +++ b/pretext/project/__init__.py @@ -552,14 +552,16 @@ def _declared_external_dir(self) -> t.Optional[str]: src_tree.xinclude() src_root = src_tree.getroot() src_external = src_root.xpath("./docinfo/directories/@external") + assert isinstance(src_external, t.List) if src_external: - return str(src_external) + return str(src_external[0]) pub_tree = ET.parse(self.publication_abspath()) pub_tree.xinclude() pub_root = pub_tree.getroot() pub_external = pub_root.xpath("./source/directories/@external") + assert isinstance(pub_external, t.List) if pub_external: - return str(pub_external) + return str(pub_external[0]) return None def _declared_managed_directories(self) -> t.Tuple[Path, t.Optional[Path]]: diff --git a/tests/examples/projects/xi_pub/publication/source.ptx b/tests/examples/projects/xi_pub/publication/source.ptx index 4cb43edd..52739862 100644 --- a/tests/examples/projects/xi_pub/publication/source.ptx +++ b/tests/examples/projects/xi_pub/publication/source.ptx @@ -1,4 +1,4 @@ - + diff --git a/tests/examples/projects/xi_pub/source/main.ptx b/tests/examples/projects/xi_pub/source/main.ptx index a63fa4eb..44bf7983 100644 --- a/tests/examples/projects/xi_pub/source/main.ptx +++ b/tests/examples/projects/xi_pub/source/main.ptx @@ -2,7 +2,7 @@ - + authorslastnamelowercase-ShortTitleCamelCase \newcommand{\foo}{b^{ar}} From 1d25295665f109615bc4ddb0a5476e4a8a78e8b6 Mon Sep 17 00:00:00 2001 From: Oscar Levin Date: Mon, 3 Aug 2026 11:22:28 -0600 Subject: [PATCH 12/13] clean up managed directories logic --- pretext/cli.py | 12 +-- pretext/project/__init__.py | 98 +++++++---------- .../projects/custom-wwserver/assets/.gitkeep | 0 .../webwork/pg/Hello_World/1.pg | 80 -------------- .../pg/Hello_World/def/setHello_World.def | 11 -- .../pg/Hello_World/header/Hello_World.pg | 40 ------- .../pg/Hello_World/macros/Hello_World.pl | 4 - .../generated-assets/webwork/root-1-1-3.xml | 100 ------------------ .../projects/custom-wwserver/source/main.ptx | 3 - .../projects/graphics/source/main.ptx | 2 +- .../prefigure/publication/publication.ptx | 7 ++ 11 files changed, 54 insertions(+), 303 deletions(-) delete mode 100644 tests/examples/projects/custom-wwserver/assets/.gitkeep delete mode 100644 tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/1.pg delete mode 100644 tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/def/setHello_World.def delete mode 100644 tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/header/Hello_World.pg delete mode 100644 tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/macros/Hello_World.pl delete mode 100644 tests/examples/projects/custom-wwserver/generated-assets/webwork/root-1-1-3.xml create mode 100644 tests/examples/projects/prefigure/publication/publication.ptx diff --git a/pretext/cli.py b/pretext/cli.py index f77b8623..6866d39c 100644 --- a/pretext/cli.py +++ b/pretext/cli.py @@ -711,12 +711,12 @@ def build( # Otherwise, we can report success. log.info("\nSuccess! Built requested target(s) without errors.\n") except ValidationError as e: - # A validation error at this point must be because the publication file is invalid, which only happens if the /source/directories/@generated|@external attributes are missing. + # A validation error at this point must be because the project or publication file failed to validate against the model the CLI builds from. log.critical( - "It appears there is an error with your publication file. Are you missing the required source/directories/@external and @generated attributes?" + "It appears there is an error with your project.ptx or publication file. See the details below." ) + log.critical(e) log.critical("Failed to build without errors. Exiting...") - log.debug(e) log.debug( "\n------------------------\nException info:\n------------------------\n", exc_info=True, @@ -935,12 +935,12 @@ def generate( # Otherwise, we can report success. log.info("Finished generating assets successfully.\n") except ValidationError as e: - # A validation error at this point must be because the publication file is invalid, which only happens if the /source/directories/@generated|@external attributes are missing. + # A validation error at this point must be because the project or publication file failed to validate against the model the CLI builds from. log.critical( - "It appears there is an error with your publication file. Are you missing the required source/directories/@external and @generated attributes?" + "It appears there is an error with your project.ptx or publication file. See the details below." ) + log.critical(e) log.critical("Failed to build. Exiting...") - log.debug(e) log.debug( "\n------------------------\nException info:\n------------------------\n", exc_info=True, diff --git a/pretext/project/__init__.py b/pretext/project/__init__.py index a1085147..e354c5f9 100644 --- a/pretext/project/__init__.py +++ b/pretext/project/__init__.py @@ -72,13 +72,15 @@ class Format(str, Enum): } -# The CLI only needs one value from the publication file. Therefore, this class ignores the vast majority of a publication file's contents, loading and validating only the relevant subset. -# Note that as of 2026-07-30, the external directory has moved to the docinfo element. -# Since we will want to hash the baseurl for generating qr codes, we also load it here. +# The CLI only needs one value from the publication file: the baseurl, which is +# hashed when generating qr codes. Therefore, this class ignores the vast +# majority of a publication file's contents, loading and validating only that. +# The managed directories are read by core (see `Target._managed_directories`): +# as of 2026-07-30 the external directory is declared in `docinfo`, and the +# publication file's `source/directories/@generated` is optional. class PublicationSubset( pxml.BaseXmlModel, tag="publication", search_mode=SearchMode.UNORDERED ): - generated: Path = pxml.wrapped("source/directories", pxml.attr()) baseurl: t.Optional[str] = pxml.wrapped( "html/baseurl", pxml.attr(name="href", default=None) ) @@ -141,10 +143,10 @@ class Target(pxml.BaseXmlModel, tag="target", search_mode=SearchMode.UNORDERED): source: Path = pxml.attr(default=Path("main.ptx")) # Cache of assembled "version only" source element. _source_element: t.Optional[ET._Element] = None - # Cache of the declared (generated, external) managed directories. Reading - # the external directory means an xinclude-processing parse of the source, - # so it is worth doing only once per target. - _declared_dirs: t.Optional[t.Tuple[Path, t.Optional[Path]]] = None + # Cache of the (generated, external) managed directories. Core reads the + # external directory from the source, which means an xinclude-processing + # parse, so it is worth doing only once per target. + _managed_dirs: t.Optional[t.Tuple[Path, t.Optional[Path]]] = None # Whether the warn-only schema check has already run for this target, so a # build doesn't repeat it when it generates assets. _schema_checked: bool = False @@ -536,63 +538,43 @@ def _read_publication_file_subset(self) -> PublicationSubset: p_bytes = ET.tostring(p_et) return PublicationSubset.from_xml(p_bytes) - def _declared_external_dir(self) -> t.Optional[str]: + def _managed_directories(self) -> t.Tuple[Path, t.Optional[Path]]: """ - The raw value of the external directory attribute, as declared, or None. - - As of core's 2026-07-30 change the external directory is a fact of the - source, declared as `directories/@external` within `docinfo`. A - publication file `source/directories/@external` is deprecated but still - honored while the docinfo is silent, so we check both, in that order. - Core issues the deprecation warning when it reads the same values, so - this stays quiet. + The (generated, external) managed directories, in absolute form, as core + reads them. `external` is None when the project declares no external + directory at all. + + Core is the single authority on where these directories are: the + external one is a property of the source (`directories/@external` within + `docinfo`, with the deprecated publication file entry still honored, and + warned about, while the docinfo is silent), while the generated one + comes from the publication file and falls back to `generated` beside the + source. Core also enforces the ownership rule that matters to us here: + the author-owned external directory must exist when declared, whereas + the machine-owned generated directory need not, since creating it is the + CLI's job. A declared external directory that is missing therefore + raises, and since both paths come from the one call, that raise reaches + `generated_dir_abspath` too: a mistyped directory stops the CLI at the + first command that needs either one, which is where the author can still + act on it. """ - src_tree = ET.parse(self.source_abspath()) - # docinfo is very often xincluded, so includes must be processed. - src_tree.xinclude() - src_root = src_tree.getroot() - src_external = src_root.xpath("./docinfo/directories/@external") - assert isinstance(src_external, t.List) - if src_external: - return str(src_external[0]) - pub_tree = ET.parse(self.publication_abspath()) - pub_tree.xinclude() - pub_root = pub_tree.getroot() - pub_external = pub_root.xpath("./source/directories/@external") - assert isinstance(pub_external, t.List) - if pub_external: - return str(pub_external[0]) - return None - - def _declared_managed_directories(self) -> t.Tuple[Path, t.Optional[Path]]: - """ - The (generated, external) managed directories as *declared*, in absolute - form, without checking whether they exist. `external` is None when no - external directory is declared at all. - - This deliberately duplicates the reading half of - `core.get_managed_directories`, which the CLI cannot use for this - purpose: that function also verifies the directories exist and raises - if they do not, but creating them is precisely the CLI's job here. - Anything invalid but existence-independent (an absolute path, say) is - still caught by core when it reads the same declarations later, so we - do not re-check it. - """ - if self._declared_dirs is None: - source_dir = self.source_abspath().parent - generated = self._read_publication_file_subset().generated - external = self._declared_external_dir() - self._declared_dirs = ( - (source_dir / generated).resolve(), - None if external is None else (source_dir / external).resolve(), + if self._managed_dirs is None: + generated, external = core.get_managed_directories( + self.source_abspath(), self.publication_abspath().as_posix() + ) + # Core normalizes but does not resolve; `Project.abspath()` is fully + # resolved, and `clean_assets` compares the two. + self._managed_dirs = ( + Path(generated).resolve(), + None if external is None else Path(external).resolve(), ) - return self._declared_dirs + return self._managed_dirs def external_dir_abspath(self) -> t.Optional[Path]: - return self._declared_managed_directories()[1] + return self._managed_directories()[1] def generated_dir_abspath(self) -> Path: - return self._declared_managed_directories()[0] + return self._managed_directories()[0] def ensure_asset_directories(self, asset: t.Optional[str] = None) -> None: # Only the generated directory is ours to create. The external diff --git a/tests/examples/projects/custom-wwserver/assets/.gitkeep b/tests/examples/projects/custom-wwserver/assets/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/1.pg b/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/1.pg deleted file mode 100644 index cfc0314a..00000000 --- a/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/1.pg +++ /dev/null @@ -1,80 +0,0 @@ -############################################# -### Generated from PreTeXt source -### -### https://pretextbook.org -### -############################################# -## DBsubject() -## DBchapter() -## DBsection() -## Level() -## KEYWORDS() -## TitleText1() -## EditionText1() -## AuthorText1() -## Section1(not reported) -## Problem1(1) -## Author() -## Institution() -## Language(en-US) - -DOCUMENT(); - -############################################################ -# Load Macros -############################################################ -loadMacros( - "PGstandard.pl", - "PGML.pl", - "AnswerFormatHelp.pl", - "PGcourse.pl", -); -COMMENT('Authored in PreTeXt'); - -############################################################ -# Header -############################################################ - -############################################################ -# PG Setup Code -############################################################ -Context('Numeric'); -$a = Compute(random(1, 9, 1)); -$b = Compute(random(1, 9, 1)); -$c = $a + $b; - -############################################################ -# Body -############################################################ - -BEGIN_PGML -Compute the sum of [`[$a]`] and [`[$b]\text{:}`] - -[`[$a] + [$b] =`] [_]{$c}{5} - -END_PGML - -############################################################ -# Hint -############################################################ -#Set value of $showHint in PGcourse.pl for course-wide attempt threshhold for revealing hints - -BEGIN_PGML_HINT -Add [`[$a]`] and [`[$b]`] together. - -END_PGML_HINT - -############################################################ -# Solution -############################################################ - -BEGIN_PGML_SOLUTION -[`[$a] + [$b] = [$c]\text{.}`] - -END_PGML_SOLUTION - -############################################################ -# End Problem -############################################################ - -ENDDOCUMENT(); diff --git a/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/def/setHello_World.def b/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/def/setHello_World.def deleted file mode 100644 index b9a5a34b..00000000 --- a/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/def/setHello_World.def +++ /dev/null @@ -1,11 +0,0 @@ -openDate = 01/01/2027 at 12:00am -dueDate = 12/31/2027 at 10:00pm -answerDate = 12/31/2027 at 10:00pm -paperHeaderFile = Hello_World/header/Hello_World.pg -screenHeaderFile = Hello_World/header/Hello_World.pg -description = Hello World! -problemListV2 -problem_start -source_file = Hello_World/1.pg -problem_id = 1 -problem_end diff --git a/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/header/Hello_World.pg b/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/header/Hello_World.pg deleted file mode 100644 index a6caf870..00000000 --- a/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/header/Hello_World.pg +++ /dev/null @@ -1,40 +0,0 @@ -# Header file for problem set Hello_World -# This header file can be used for both the screen and hardcopy output - -DOCUMENT(); - -loadMacros( - "PG.pl", - "PGbasicmacros.pl", - "PGML.pl", - "PGcourse.pl", -); - -TEXT($BEGIN_ONE_COLUMN); - -$texTopLine = "\noindent {\large \bf $studentName}\hfill{\large \bf {".protect_underbar($courseName)."}}"; -if (defined($sectionName) and ($sectionName ne '')) {$texTopLine .= " {\large \bf { Section: ".protect_underbar($sectionName)." } }"}; -$texTopLine .= "\par"; - -#################################################### -# -# MODES provides for distinct output for TeX and HTML -# -#################################################### - -TEXT(MODES( - TeX =>"$texTopLine", - HTML=>"", -)); - -TEXT(MODES( - TeX =>"\noindent{\large \sc {Assignment ".protect_underbar($setNumber)." due $formatedDueDate}}\par". - "\noindent \bigskip ", - HTML=>"WeBWorK Assignment ".protect_underbar($setNumber)." is due: $formattedDueDate. $PAR", -)); - -TEXT("This assignment contains exercises from Article of Hello World!."); - -TEXT($END_ONE_COLUMN); - -ENDDOCUMENT(); diff --git a/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/macros/Hello_World.pl b/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/macros/Hello_World.pl deleted file mode 100644 index 559e261d..00000000 --- a/tests/examples/projects/custom-wwserver/generated-assets/webwork/pg/Hello_World/macros/Hello_World.pl +++ /dev/null @@ -1,4 +0,0 @@ -############################################################################# -# This macro library supports WeBWorK problems from the PreTeXt project named -# Hello World! -############################################################################# diff --git a/tests/examples/projects/custom-wwserver/generated-assets/webwork/root-1-1-3.xml b/tests/examples/projects/custom-wwserver/generated-assets/webwork/root-1-1-3.xml deleted file mode 100644 index f8b0b724..00000000 --- a/tests/examples/projects/custom-wwserver/generated-assets/webwork/root-1-1-3.xml +++ /dev/null @@ -1,100 +0,0 @@ - - - -

Compute the sum of {1} and {7}\text{:}

-

{1} + {7} =

- -

Add {1} and {7} together.

- - -

- 8 -

- -

{1} + {7} = {8}\text{.}

- - - - - diff --git a/tests/examples/projects/custom-wwserver/source/main.ptx b/tests/examples/projects/custom-wwserver/source/main.ptx index 6572f594..7519cdf8 100644 --- a/tests/examples/projects/custom-wwserver/source/main.ptx +++ b/tests/examples/projects/custom-wwserver/source/main.ptx @@ -1,8 +1,5 @@ - - -
Hello World!

This is a PreTeXt document.

diff --git a/tests/examples/projects/graphics/source/main.ptx b/tests/examples/projects/graphics/source/main.ptx index 32120c1d..7bff6562 100644 --- a/tests/examples/projects/graphics/source/main.ptx +++ b/tests/examples/projects/graphics/source/main.ptx @@ -56,4 +56,4 @@ - \ No newline at end of file + diff --git a/tests/examples/projects/prefigure/publication/publication.ptx b/tests/examples/projects/prefigure/publication/publication.ptx new file mode 100644 index 00000000..083b57a5 --- /dev/null +++ b/tests/examples/projects/prefigure/publication/publication.ptx @@ -0,0 +1,7 @@ + + + + + + + From b694b9c5de605211f0c1556d6a16e245cceb063e Mon Sep 17 00:00:00 2001 From: Oscar Levin Date: Mon, 3 Aug 2026 12:41:18 -0600 Subject: [PATCH 13/13] update core --- pretext/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pretext/__init__.py b/pretext/__init__.py index 32f6378c..ba2affea 100644 --- a/pretext/__init__.py +++ b/pretext/__init__.py @@ -18,7 +18,7 @@ VERSION = get_version("pretext", Path(__file__).parent.parent) -CORE_COMMIT = "b02c1279be5bcf2d959dce324f2adcc0496c7dc5" +CORE_COMMIT = "ea9c025222639b69dd7bf87dd3b9a48b507b8418" def activate() -> None: