From 16b0d6d9c276ca8001595f9343fd0634094bca8d Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Tue, 11 Aug 2026 17:45:23 +0100 Subject: [PATCH 01/24] #102 Emit a JSON model of the wrapped package Add build_python_model (cppwg/utils/python_model.py) and call it from CppWrapperGenerator.generate() via write_python_model(), writing cppwg_model.json to the wrapper root once the info tree is final. The model describes each module's compiled extension name, its classes (base py-name, templated flag, and per-instantiation arg-list -> py_name), enums and free functions - enough for a separate step to generate the Python package layer without re-parsing the source. Co-Authored-By: Claude Opus 4.8 --- cppwg/generators.py | 21 ++++++ cppwg/utils/constants.py | 5 ++ cppwg/utils/python_model.py | 105 ++++++++++++++++++++++++++ tests/test_python_model.py | 146 ++++++++++++++++++++++++++++++++++++ 4 files changed, 277 insertions(+) create mode 100644 cppwg/utils/python_model.py create mode 100644 tests/test_python_model.py diff --git a/cppwg/generators.py b/cppwg/generators.py index 360528c..c78f8dc 100644 --- a/cppwg/generators.py +++ b/cppwg/generators.py @@ -1,5 +1,6 @@ """The main interface for generating Python wrappers.""" +import json import logging import os import re @@ -18,7 +19,9 @@ from cppwg.utils.constants import ( CPPWG_DEFAULT_WRAPPER_DIR, CPPWG_HEADER_COLLECTION_FILENAME, + CPPWG_PYTHON_MODEL_FILENAME, ) +from cppwg.utils.python_model import build_python_model from cppwg.version import __version__ as cppwg_version from cppwg.writers.header_collection_writer import CppHeaderCollectionWriter from cppwg.writers.package_writer import CppPackageWrapperWriter @@ -422,6 +425,20 @@ def write_wrappers(self) -> None: ) package_writer.write() + def write_python_model(self) -> None: + """ + Write the Python-package model (cppwg_model.json) to the wrapper root. + + A small JSON description of the generated modules and their classes / + instantiations / enums / free functions, so a separate step + (tools/cppwg_initgen.py) can generate the Python package layer without + re-parsing the source. Written last, once the info tree is final. + """ + model = build_python_model(self.package_info) + model_path = os.path.join(self.wrapper_root, CPPWG_PYTHON_MODEL_FILENAME) + content = json.dumps(model, indent=2, sort_keys=True) + "\n" + utils.write_file_if_changed(model_path, content, self.overwrite) + def generate(self) -> None: """ Parse yaml configuration and C++ source to generate Python wrappers. @@ -482,3 +499,7 @@ def generate(self) -> None: # Write the wrapper code for the package self.write_wrappers() + + # Write the Python-package model (cppwg_model.json) for the package-layer + # generator (tools/cppwg_initgen.py). + self.write_python_model() diff --git a/cppwg/utils/constants.py b/cppwg/utils/constants.py index c52527b..1b1aa88 100644 --- a/cppwg/utils/constants.py +++ b/cppwg/utils/constants.py @@ -9,6 +9,11 @@ # Default log file name used when --logfile is passed without a value. CPPWG_DEFAULT_LOGFILE = f"{CPPWG_EXT}.log" +# The Python-package model cppwg writes into the wrapper root, describing the +# generated modules/classes so a separate step can build the Python package +# layer (see cppwg.utils.python_model and tools/cppwg_initgen.py). +CPPWG_PYTHON_MODEL_FILENAME = "cppwg_model.json" + CPPWG_TRUE_STRINGS = ["ON", "YES", "Y", "TRUE", "T", "1"] CPPWG_FALSE_STRINGS = ["OFF", "NO", "N", "FALSE", "F", "0", ""] diff --git a/cppwg/utils/python_model.py b/cppwg/utils/python_model.py new file mode 100644 index 0000000..e205d30 --- /dev/null +++ b/cppwg/utils/python_model.py @@ -0,0 +1,105 @@ +"""Build a JSON-serialisable model of the Python package layer. + +cppwg generates the C++/pybind11 wrappers; a separate step (see +``tools/cppwg_initgen.py``) generates the Python package layer - the +``_generated.py`` files that import each compiled extension and define the +``TemplateClass`` subscript stubs. That step needs, per module, the compiled +module name, the wrapped classes with their template instantiations, and the +enum / free-function names. All of this is on the finalized ``PackageInfo`` tree +but not in a form a standalone script can consume, so ``build_python_model`` +distils it into a plain dict that cppwg writes out as ``cppwg_model.json``. +""" + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from cppwg.info.package_info import PackageInfo + + +def compiled_module_name(package_name: str, module_name: str) -> str: + """ + Return the pybind11 module name for a package/module, e.g. ``_pyshapes_geometry``. + + Mirrors CppModuleWrapperWriter.full_module_name so the model and the C++ + wrappers agree on the compiled extension name. + """ + return f"_{package_name}_{module_name}" + + +def build_python_model(package_info: "PackageInfo") -> dict[str, Any]: + """ + Distil a PackageInfo tree into a JSON-serialisable Python-package model. + + Parameters + ---------- + package_info : PackageInfo + The finalized package info (after wrappers are written, so py_names and + template_arg_lists are complete). + + Returns + ------- + dict[str, Any] + ``{"package": name, "modules": [{"name", "compiled_module", "imports", + "classes": [{"base", "templated", "instantiations": [{"args", "py_name"}]}], + "enums": [...], "free_functions": [...]}]}``. Excluded entities are + omitted (they are not wrapped). An untemplated class has ``templated: + false`` and a single instantiation with empty ``args``. + """ + modules = [] + for module in package_info.module_collection: + classes = [] + for class_info in module.class_collection: + if class_info.excluded: + continue + + if class_info.template_arg_lists: + instantiations = [ + {"args": [str(arg) for arg in args], "py_name": py_name} + for args, py_name in zip( + class_info.template_arg_lists, class_info.py_names + ) + ] + templated = True + else: + instantiations = [ + {"args": [], "py_name": py_name} for py_name in class_info.py_names + ] + templated = False + + if not instantiations: + # A class whose every instantiation was pruned contributes nothing. + continue + + classes.append( + { + "base": class_info.py_name_base(), + "templated": templated, + "instantiations": instantiations, + } + ) + + enums = [ + enum_info.name_override or enum_info.name + for enum_info in module.enum_collection + if not enum_info.excluded + ] + free_functions = [ + free_function_info.name + for free_function_info in module.free_function_collection + if not free_function_info.excluded + ] + + modules.append( + { + "name": module.name, + "compiled_module": compiled_module_name( + package_info.name, module.name + ), + "imports": list(module.imports), + "classes": classes, + "enums": enums, + "free_functions": free_functions, + } + ) + + return {"package": package_info.name, "modules": modules} diff --git a/tests/test_python_model.py b/tests/test_python_model.py new file mode 100644 index 0000000..e35e553 --- /dev/null +++ b/tests/test_python_model.py @@ -0,0 +1,146 @@ +"""Unit tests for cppwg.utils.python_model.""" + +from types import SimpleNamespace + +from cppwg.utils.python_model import build_python_model, compiled_module_name + + +def _class(base, py_names, template_arg_lists=(), excluded=False): + return SimpleNamespace( + excluded=excluded, + py_names=list(py_names), + template_arg_lists=[list(a) for a in template_arg_lists], + py_name_base=lambda base=base: base, + ) + + +def _enum(name, name_override="", excluded=False): + return SimpleNamespace(name=name, name_override=name_override, excluded=excluded) + + +def _free_function(name, excluded=False): + return SimpleNamespace(name=name, excluded=excluded) + + +def _module(name, classes=(), enums=(), free_functions=(), imports=()): + return SimpleNamespace( + name=name, + imports=list(imports), + class_collection=list(classes), + enum_collection=list(enums), + free_function_collection=list(free_functions), + ) + + +def _package(name, modules): + return SimpleNamespace(name=name, module_collection=list(modules)) + + +def test_compiled_module_name(): + assert compiled_module_name("pyshapes", "geometry") == "_pyshapes_geometry" + assert compiled_module_name("pycells", "all") == "_pycells_all" + + +def test_build_model_templated_and_untemplated(): + package = _package( + "pyshapes", + [ + _module( + "geometry", + classes=[_class("Point", ["Point_2", "Point_3"], [[2], [3]])], + ), + _module( + "primitives", + classes=[ + _class("Shape", ["Shape_2", "Shape_3"], [[2], [3]]), + _class("UnitSquare", ["UnitSquare"]), # untemplated + ], + enums=[_enum("ShapeKind")], + imports=["pyshapes.geometry._pyshapes_geometry"], + ), + ], + ) + + model = build_python_model(package) + + assert model["package"] == "pyshapes" + geometry, primitives = model["modules"] + + assert geometry["compiled_module"] == "_pyshapes_geometry" + (point,) = geometry["classes"] + assert point == { + "base": "Point", + "templated": True, + "instantiations": [ + {"args": ["2"], "py_name": "Point_2"}, + {"args": ["3"], "py_name": "Point_3"}, + ], + } + + assert primitives["imports"] == ["pyshapes.geometry._pyshapes_geometry"] + shape, unit_square = primitives["classes"] + assert shape["templated"] is True + assert unit_square == { + "base": "UnitSquare", + "templated": False, + "instantiations": [{"args": [], "py_name": "UnitSquare"}], + } + assert primitives["enums"] == ["ShapeKind"] + + +def test_build_model_omits_excluded_and_pruned(): + package = _package( + "pkg", + [ + _module( + "mod", + classes=[ + _class("Kept", ["Kept"]), + _class("Gone", ["Gone"], excluded=True), # excluded + _class("Pruned", [], [[2]]), # all instantiations pruned away + ], + enums=[_enum("KeptEnum"), _enum("GoneEnum", excluded=True)], + free_functions=[ + _free_function("kept_fn"), + _free_function("gone_fn", excluded=True), + ], + ) + ], + ) + + (module,) = build_python_model(package)["modules"] + + assert [c["base"] for c in module["classes"]] == ["Kept"] + assert module["enums"] == ["KeptEnum"] + assert module["free_functions"] == ["kept_fn"] + + +def test_build_model_multi_arg_and_class_arg_keys(): + """Multi-arg and class-name-arg instantiations stringify each argument.""" + package = _package( + "pycells", + [ + _module( + "all", + classes=[ + _class("MacroMesh", ["MacroMesh_2_2"], [[2, 2]]), + _class("CellFactory", ["CellFactory_Cell_2"], [["Cell", 2]]), + ], + ) + ], + ) + + (module,) = build_python_model(package)["modules"] + macro, factory = module["classes"] + assert macro["instantiations"] == [{"args": ["2", "2"], "py_name": "MacroMesh_2_2"}] + assert factory["instantiations"] == [ + {"args": ["Cell", "2"], "py_name": "CellFactory_Cell_2"} + ] + + +def test_enum_name_override_used(): + package = _package( + "pkg", [_module("mod", enums=[_enum("RawName", name_override="PyName")])] + ) + (module,) = build_python_model(package)["modules"] + assert module["enums"] == ["PyName"] From bc8f56c3cb1a90a93d183a2081c45a86087214c8 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Tue, 11 Aug 2026 17:45:35 +0100 Subject: [PATCH 02/24] #102 Add cppwg_initgen.py to generate the Python package layer A standalone script that reads cppwg_model.json plus a small YAML layout manifest and writes a per-subpackage _generated.py: the compiled-extension import and the TemplateClass subscript stubs (Point[2] -> Point_2). It never touches the hand-written __init__.py, which does `from ._generated import *` and keeps the bespoke pieces (TemplateMethod attachments, package init, curation). Output is black-formatted so it is stable under a git-diff / black --check reproducibility gate. Two layouts are supported: module-per-subpackage (each cppwg module owns its own extension, `from . import *`) and shared-module split (one extension divided into subpackages by an explicit membership manifest, used where a single compiled module backs several Python subpackages). An opt-in diagonal_shorthand flag adds a single-arg alias for all-equal instantiations (Element[2] == Element[2, 2]). Co-Authored-By: Claude Opus 4.8 --- tests/test_initgen.py | 213 +++++++++++++++++++++++++++++++ tools/cppwg_initgen.py | 284 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 497 insertions(+) create mode 100644 tests/test_initgen.py create mode 100644 tools/cppwg_initgen.py diff --git a/tests/test_initgen.py b/tests/test_initgen.py new file mode 100644 index 0000000..dec1609 --- /dev/null +++ b/tests/test_initgen.py @@ -0,0 +1,213 @@ +"""Unit tests for tools/cppwg_initgen.py.""" + +import importlib.util +import os + +_INITGEN_PATH = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "tools", "cppwg_initgen.py" +) +_spec = importlib.util.spec_from_file_location("cppwg_initgen", _INITGEN_PATH) +initgen = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(initgen) + + +def _class(base, instantiations, templated=True): + return {"base": base, "templated": templated, "instantiations": instantiations} + + +def _inst(args, py_name): + return {"args": list(args), "py_name": py_name} + + +def test_key_repr_singleton_and_multi(): + assert initgen._key_repr(["2"]) == '("2",)' + assert initgen._key_repr(["2", "2"]) == '("2", "2")' + assert initgen._key_repr(["Cell", "2"]) == '("Cell", "2")' + + +def test_stub_source(): + stub = initgen._stub_source( + "Point", [_inst(["2"], "Point_2"), _inst(["3"], "Point_3")] + ) + assert stub == ( + "class Point(TemplateClass):\n" + " _instantiations = {\n" + ' ("2",): Point_2,\n' + ' ("3",): Point_3,\n' + " }" + ) + + +def test_stub_source_diagonal_shorthand(): + """A multi-arg diagonal instantiation gains a single-arg alias when opted in.""" + stub = initgen._stub_source( + "Element", + [_inst(["2", "2"], "Element_2_2"), _inst(["1", "2"], "Element_1_2")], + diagonal_shorthand=True, + ) + # <2, 2> is diagonal -> add the Element[2] alias; <1, 2> is not -> no alias. + assert '("2", "2"): Element_2_2,' in stub + assert '("2",): Element_2_2,' in stub + assert '("1", "2"): Element_1_2,' in stub + assert '("1",):' not in stub + # Off by default: no aliases. + plain = initgen._stub_source("Element", [_inst(["2", "2"], "Element_2_2")]) + assert '("2",):' not in plain + + +def test_render_generated_module_with_stub_imports_syntax(): + point = _class("Point", [_inst(["2"], "Point_2"), _inst(["3"], "Point_3")]) + content = initgen.render_generated_module( + "pyshapes", "from ._pyshapes_geometry import *", [point], [point] + ) + assert content.startswith('"""Generated by tools/cppwg_initgen.py') + assert "from ._pyshapes_geometry import *" in content + assert "from pyshapes._syntax import TemplateClass" in content + assert "class Point(TemplateClass):" in content + assert content.endswith("\n") + + +def test_render_generated_module_no_templates_omits_syntax(): + plain = _class("Square", [_inst([], "Square")], templated=False) + content = initgen.render_generated_module( + "pyshapes", "from ._pyshapes_composites import *", [plain], [] + ) + assert "from ._pyshapes_composites import *" in content + assert "import TemplateClass" not in content # no templated -> no syntax import + + +def test_generate_module_per_subpackage(tmp_path): + model = { + "package": "pyshapes", + "modules": [ + { + "name": "geometry", + "compiled_module": "_pyshapes_geometry", + "imports": [], + "classes": [ + _class("Point", [_inst(["2"], "Point_2"), _inst(["3"], "Point_3")]) + ], + "enums": [], + "free_functions": [], + }, + { + "name": "math_funcs", + "compiled_module": "_pyshapes_math_funcs", + "imports": [], + "classes": [], + "enums": [], + "free_functions": ["add"], + }, + ], + } + manifest = {"package": "pyshapes", "package_root": str(tmp_path)} + + initgen.generate_module_per_subpackage(model, manifest, overwrite=False) + + geometry = (tmp_path / "geometry" / "_generated.py").read_text() + assert "from ._pyshapes_geometry import *" in geometry + assert "class Point(TemplateClass):" in geometry + + math_funcs = (tmp_path / "math_funcs" / "_generated.py").read_text() + assert "from ._pyshapes_math_funcs import *" in math_funcs + assert "import TemplateClass" not in math_funcs # free functions only + + +def test_module_dirs_override_places_single_module_at_root(tmp_path): + """A `module_dirs` entry redirects a module's _generated.py (e.g. cells' root).""" + model = { + "package": "pycells", + "modules": [ + { + "name": "all", + "compiled_module": "_pycells_all", + "imports": [], + "classes": [_class("Node", [_inst(["2"], "Node_2")])], + "enums": [], + "free_functions": [], + } + ], + } + manifest = {"package_root": str(tmp_path), "module_dirs": {"all": "."}} + + initgen.generate_module_per_subpackage(model, manifest, overwrite=False) + + generated = (tmp_path / "_generated.py").read_text() # at the root, not all/ + assert "from ._pycells_all import *" in generated + assert "class Node(TemplateClass):" in generated + + +def test_generate_shared_module_split(tmp_path, capsys): + """Split one shared extension into subpackages by an explicit membership list.""" + model = { + "package": "pychaste", + "modules": [ + { + "name": "all", + "compiled_module": "_pychaste_all", + "imports": [], + "classes": [ + _class("Node", [_inst(["2"], "Node_2"), _inst(["3"], "Node_3")]), + _class("FileFinder", [_inst([], "FileFinder")], templated=False), + _class("PottsMesh", [_inst(["2"], "PottsMesh_2")]), + ], + "enums": ["RelativeTo"], + "free_functions": [], + } + ], + } + manifest = { + "package": "chaste", + "package_root": str(tmp_path), + "compiled_module": "_pychaste_all", + "subpackages": { + "core": ["FileFinder", "RelativeTo"], + "mesh": ["Node", "PottsMesh"], + }, + } + + initgen.generate_shared_module_split(model, manifest, overwrite=False) + + core = (tmp_path / "core" / "_generated.py").read_text() + assert "from chaste._pychaste_all import (" in core + assert "FileFinder," in core and "RelativeTo," in core + assert "import TemplateClass" not in core # core has no templated class + + mesh = (tmp_path / "mesh" / "_generated.py").read_text() + assert "from chaste._pychaste_all import (" in mesh + # concrete names imported, sorted + assert "Node_2," in mesh and "Node_3," in mesh and "PottsMesh_2," in mesh + assert "from chaste._syntax import TemplateClass" in mesh + assert "class Node(TemplateClass):" in mesh + assert "class PottsMesh(TemplateClass):" in mesh + + +def test_shared_split_warns_on_unknown_and_unassigned(tmp_path, capsys): + model = { + "package": "pkg", + "modules": [ + { + "name": "all", + "compiled_module": "_pkg_all", + "imports": [], + "classes": [ + _class("Kept", [_inst([], "Kept")], templated=False), + _class("Orphan", [_inst([], "Orphan")], templated=False), + ], + "enums": [], + "free_functions": [], + } + ], + } + manifest = { + "package": "pkg", + "package_root": str(tmp_path), + "compiled_module": "_pkg_all", + "subpackages": {"sub": ["Kept", "DoesNotExist"]}, + } + + initgen.generate_shared_module_split(model, manifest, overwrite=False) + + err = capsys.readouterr().err + assert "'DoesNotExist'" in err # listed but not in model + assert "'Orphan'" in err # in model but not assigned to a subpackage diff --git a/tools/cppwg_initgen.py b/tools/cppwg_initgen.py new file mode 100644 index 0000000..9b278fb --- /dev/null +++ b/tools/cppwg_initgen.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +"""Generate the Python package layer (``_generated.py``) for a cppwg project. + +cppwg generates the C++/pybind11 wrappers and, alongside them, a JSON model of +what it produced (``cppwg_model.json`` in the wrapper root - see +``cppwg.utils.python_model``). This standalone script turns that model, plus a +small layout manifest, into a ``_generated.py`` per Python subpackage: the +compiled-extension import and the ``TemplateClass`` subscript stubs +(``Point[2] -> Point_2``). It never touches the hand-written ``__init__.py``, +which does ``from ._generated import *`` and adds the bespoke pieces +(``TemplateMethod`` attachments, package ``init()``, curation, comments). + +Two layouts: + +* module-per-subpackage (default): each cppwg module becomes a subpackage that + owns its own compiled extension, imported with ``from . import *``. + Used by the shapes/cells examples. +* shared-module split: one compiled extension (e.g. ``_pychaste_all``) is split + into several subpackages by a manifest that lists which names each owns; + imported explicitly with ``from . import (...)``. Used by + pychaste. + +Usage:: + + cppwg_initgen.py --model wrapper/cppwg_model.json --manifest py_layout.yaml +""" + +import argparse +import json +import os +import sys + +# Reuse cppwg's idempotent writer when importable; fall back to a local copy so +# the script also runs standalone (e.g. from a build tree without cppwg on the +# path). +try: + from cppwg.utils.utils import write_file_if_changed +except ImportError: # pragma: no cover - exercised only without cppwg installed + + def write_file_if_changed(filepath, content, overwrite=False): + """Write content unless an identical file already exists.""" + if not overwrite and os.path.isfile(filepath): + with open(filepath) as in_file: + if in_file.read() == content: + return False + with open(filepath, "w") as out_file: + out_file.write(content) + return True + + +GENERATED_HEADER = ( + '"""Generated by tools/cppwg_initgen.py - do not edit.\n' + "\n" + "Compiled-extension imports and TemplateClass subscript stubs for this\n" + "subpackage. Hand-written code (TemplateMethod attachments, package init,\n" + 'curation) belongs in the sibling __init__.py, which does `from ._generated\n' + 'import *`.\n' + '"""\n' +) + + +def _key_repr(args: list[str]) -> str: + """Render a template-argument list as a Python tuple literal of strings. + + e.g. ``["2"]`` -> ``("2",)`` and ``["Cell", "2"]`` -> ``("Cell", "2")``. + Matches _syntax._normalize_key, which normalizes keys to string tuples. + """ + inner = ", ".join(f'"{arg}"' for arg in args) + if len(args) == 1: + inner += "," + return f"({inner})" + + +def _stub_source( + base: str, instantiations: list[dict], diagonal_shorthand: bool = False +) -> str: + """Render a ``class (TemplateClass)`` stub for a templated class. + + When ``diagonal_shorthand`` is set, a multi-argument instantiation whose + arguments are all equal (a "diagonal", e.g. ``Element<2, 2>``) also gets a + single-argument alias key (``Element[2] -> Element_2_2``). This is an + opt-in Python-layer convenience some packages (pychaste) adopt for their + ````-style classes; others (the cells example) keep + the explicit multi-argument form only. + """ + lines = [f"class {base}(TemplateClass):", " _instantiations = {"] + for inst in instantiations: + args = inst["args"] + lines.append(f' {_key_repr(args)}: {inst["py_name"]},') + if diagonal_shorthand and len(args) > 1 and len(set(args)) == 1: + lines.append(f' {_key_repr(args[:1])}: {inst["py_name"]},') + lines.append(" }") + return "\n".join(lines) + + +def render_generated_module( + package: str, + compiled_import: str, + classes: list[dict], + templated_classes: list[dict], + diagonal_shorthand: bool = False, +) -> str: + """ + Render a subpackage's ``_generated.py`` content. + + Parameters + ---------- + package : str + The Python import root (e.g. ``pyshapes``, ``chaste``), used to import + the shared ``_syntax`` helper. + compiled_import : str + The import statement pulling in the compiled extension's names. + classes : list[dict] + The subpackage's classes (from the model); used to know whether any + TemplateClass import is needed. + templated_classes : list[dict] + The subset that is templated, each rendered as a stub. + + Returns + ------- + str + The file content (ending with a newline). + """ + # Build black-clean output: docstring, one blank line, the imports, then each + # stub separated by two blank lines. + lines = [GENERATED_HEADER.rstrip("\n"), "", compiled_import] + if templated_classes: + lines.append(f"from {package}._syntax import TemplateClass") + for class_info in templated_classes: + lines.extend(["", ""]) # two blank lines before each top-level class + lines.append( + _stub_source( + class_info["base"], + class_info["instantiations"], + diagonal_shorthand, + ) + ) + return "\n".join(lines) + "\n" + + +def _concrete_names(class_info: dict) -> list[str]: + """Return the concrete py_names of a class's instantiations.""" + return [inst["py_name"] for inst in class_info["instantiations"]] + + +def _explicit_import(package: str, compiled_module: str, names: list[str]) -> str: + """Render an explicit ``from . import (...)`` statement.""" + body = "".join(f" {name},\n" for name in sorted(names)) + return f"from {package}.{compiled_module} import (\n{body})" + + +def _format(content: str) -> str: + """Format with black when available, so output is stable and idempotent. + + Long instantiation keys (e.g. pychaste's CellsGenerator) exceed black's line + length and must be wrapped exactly as black would, or a project's + ``black --check`` / ``git diff`` reproducibility gate fails. When black is + not installed the content is written as-is (still valid Python). + """ + try: + import black + except ImportError: # pragma: no cover - only when black is absent + return content + return black.format_str(content, mode=black.Mode()) + + +def _write_generated(path: str, content: str, overwrite: bool) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + content = _format(content) + if write_file_if_changed(path, content, overwrite): + print(f"wrote {path}") + else: + print(f"unchanged {path}") + + +def generate_module_per_subpackage(model: dict, manifest: dict, overwrite: bool): + """Emit one ``_generated.py`` per cppwg module (each owns its own extension).""" + package = manifest.get("package", model["package"]) + package_root = manifest["package_root"] + module_dirs = manifest.get("module_dirs", {}) + diagonal_shorthand = manifest.get("diagonal_shorthand", False) + + for module in model["modules"]: + # The subpackage directory (relative to package_root); default = module + # name, overridable (e.g. a single "all" module living at the root). + subdir = module_dirs.get(module["name"], module["name"]) + compiled_import = f"from .{module['compiled_module']} import *" + templated = [c for c in module["classes"] if c["templated"]] + content = render_generated_module( + package, compiled_import, module["classes"], templated, diagonal_shorthand + ) + path = os.path.join(package_root, subdir, "_generated.py") + _write_generated(path, content, overwrite) + + +def generate_shared_module_split(model: dict, manifest: dict, overwrite: bool): + """Emit a ``_generated.py`` per subpackage, splitting one shared extension.""" + package = manifest.get("package", model["package"]) + package_root = manifest["package_root"] + compiled_module = manifest["compiled_module"] + diagonal_shorthand = manifest.get("diagonal_shorthand", False) + + # Index every wrapped entity across the model by name so a manifest entry can + # be matched to a class (with its instantiations), an enum or a free function. + classes_by_base = {} + other_names = set() + for module in model["modules"]: + for class_info in module["classes"]: + classes_by_base[class_info["base"]] = class_info + for name in module["enums"] + module["free_functions"]: + other_names.add(name) + + assigned = set() + for subpkg, names in manifest["subpackages"].items(): + classes = [] + import_names = [] + for name in names: + assigned.add(name) + if name in classes_by_base: + class_info = classes_by_base[name] + classes.append(class_info) + import_names.extend(_concrete_names(class_info)) + elif name in other_names: + import_names.append(name) + else: + print(f"warning: '{name}' ({subpkg}) not found in model", file=sys.stderr) + + templated = [c for c in classes if c["templated"]] + compiled_import = _explicit_import(package, compiled_module, import_names) + content = render_generated_module( + package, compiled_import, classes, templated, diagonal_shorthand + ) + path = os.path.join(package_root, subpkg, "_generated.py") + _write_generated(path, content, overwrite) + + # Flag any wrapped class not placed in a subpackage, so nothing is silently + # dropped when the config gains a class. + unplaced = sorted(set(classes_by_base) | other_names) + for name in unplaced: + if name not in assigned: + print(f"warning: '{name}' is wrapped but not assigned to a subpackage", + file=sys.stderr) + + +def main(argv=None) -> int: + """Entry point: read the model + manifest and write the _generated.py files.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--model", required=True, help="Path to cppwg_model.json (from cppwg)." + ) + parser.add_argument( + "--manifest", required=True, help="Path to the Python layout manifest (YAML)." + ) + parser.add_argument( + "--overwrite", + action="store_true", + help="Rewrite files even if unchanged.", + ) + args = parser.parse_args(argv) + + import yaml + + with open(args.model) as model_file: + model = json.load(model_file) + with open(args.manifest) as manifest_file: + manifest = yaml.safe_load(manifest_file) + + # Resolve a relative package_root against the manifest's own directory, so a + # manifest is portable regardless of the caller's working directory. + if not os.path.isabs(manifest["package_root"]): + manifest_dir = os.path.dirname(os.path.abspath(args.manifest)) + manifest["package_root"] = os.path.normpath( + os.path.join(manifest_dir, manifest["package_root"]) + ) + + if manifest.get("subpackages"): + generate_shared_module_split(model, manifest, args.overwrite) + else: + generate_module_per_subpackage(model, manifest, args.overwrite) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 4544bf033519643436a009e28e3bbb164de60956 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Tue, 11 Aug 2026 17:45:56 +0100 Subject: [PATCH 03/24] #102 Generate the shapes and cells package layers Apply cppwg_initgen.py to both examples. Each subpackage's mechanical __init__.py body is replaced with `from ._generated import *`, and the generated _generated.py (compiled import + TemplateClass stubs) is added alongside; the shapes primitives __init__ keeps its bespoke UnitSquare.GetAreaIn TemplateMethod attachment. A py_layout.yaml manifest and the emitted cppwg_model.json are committed for each (cells uses a module_dirs override to place its single `all` module at the package root). Co-Authored-By: Claude Opus 4.8 --- examples/cells/dynamic/py_layout.yaml | 11 + .../cells/dynamic/wrappers/cppwg_model.json | 230 ++++++++++++++++++ examples/cells/src/py/pycells/__init__.py | 118 +-------- examples/cells/src/py/pycells/_generated.py | 85 +++++++ .../src/py/pyshapes/composites/__init__.py | 5 +- .../src/py/pyshapes/composites/_generated.py | 9 + .../src/py/pyshapes/geometry/__init__.py | 13 +- .../src/py/pyshapes/geometry/_generated.py | 17 ++ .../src/py/pyshapes/math_funcs/__init__.py | 5 +- .../src/py/pyshapes/math_funcs/_generated.py | 9 + .../src/py/pyshapes/primitives/__init__.py | 16 +- .../src/py/pyshapes/primitives/_generated.py | 38 +++ examples/shapes/wrapper/cppwg_model.json | 221 +++++++++++++++++ examples/shapes/wrapper/py_layout.yaml | 9 + 14 files changed, 650 insertions(+), 136 deletions(-) create mode 100644 examples/cells/dynamic/py_layout.yaml create mode 100644 examples/cells/dynamic/wrappers/cppwg_model.json create mode 100644 examples/cells/src/py/pycells/_generated.py create mode 100644 examples/shapes/src/py/pyshapes/composites/_generated.py create mode 100644 examples/shapes/src/py/pyshapes/geometry/_generated.py create mode 100644 examples/shapes/src/py/pyshapes/math_funcs/_generated.py create mode 100644 examples/shapes/src/py/pyshapes/primitives/_generated.py create mode 100644 examples/shapes/wrapper/cppwg_model.json create mode 100644 examples/shapes/wrapper/py_layout.yaml diff --git a/examples/cells/dynamic/py_layout.yaml b/examples/cells/dynamic/py_layout.yaml new file mode 100644 index 0000000..8451dd1 --- /dev/null +++ b/examples/cells/dynamic/py_layout.yaml @@ -0,0 +1,11 @@ +# Python-package layout for tools/cppwg_initgen.py (cells example). +# +# A single cppwg module `all` -> the flat pycells package (its compiled +# extension _pycells_all sits at the package root, not in an `all/` subdir), so +# module_dirs maps `all` to ".". +# +# package_root is resolved relative to this manifest's directory. +package: pycells +package_root: ../src/py/pycells +module_dirs: + all: "." diff --git a/examples/cells/dynamic/wrappers/cppwg_model.json b/examples/cells/dynamic/wrappers/cppwg_model.json new file mode 100644 index 0000000..b12b07a --- /dev/null +++ b/examples/cells/dynamic/wrappers/cppwg_model.json @@ -0,0 +1,230 @@ +{ + "modules": [ + { + "classes": [ + { + "base": "Cell", + "instantiations": [ + { + "args": [], + "py_name": "Cell" + } + ], + "templated": false + }, + { + "base": "CellFactory", + "instantiations": [ + { + "args": [ + "Cell", + "2" + ], + "py_name": "CellFactory_Cell_2" + }, + { + "args": [ + "Cell", + "3" + ], + "py_name": "CellFactory_Cell_3" + } + ], + "templated": true + }, + { + "base": "Corner", + "instantiations": [ + { + "args": [ + "2" + ], + "py_name": "Corner_2" + } + ], + "templated": true + }, + { + "base": "Facet", + "instantiations": [ + { + "args": [ + "2" + ], + "py_name": "Facet_2" + } + ], + "templated": true + }, + { + "base": "MacroMesh", + "instantiations": [ + { + "args": [ + "2", + "2" + ], + "py_name": "MacroMesh_2_2" + }, + { + "args": [ + "3", + "3" + ], + "py_name": "MacroMesh_3_3" + } + ], + "templated": true + }, + { + "base": "Node", + "instantiations": [ + { + "args": [ + "2" + ], + "py_name": "Node_2" + }, + { + "args": [ + "3" + ], + "py_name": "Node_3" + } + ], + "templated": true + }, + { + "base": "AbstractMesh", + "instantiations": [ + { + "args": [ + "2", + "2" + ], + "py_name": "AbstractMesh_2_2" + }, + { + "args": [ + "3", + "3" + ], + "py_name": "AbstractMesh_3_3" + } + ], + "templated": true + }, + { + "base": "AbstractSphericalMesh", + "instantiations": [ + { + "args": [ + "2", + "2" + ], + "py_name": "AbstractSphericalMesh_2_2" + }, + { + "args": [ + "3", + "3" + ], + "py_name": "AbstractSphericalMesh_3_3" + } + ], + "templated": true + }, + { + "base": "PetscUtils", + "instantiations": [ + { + "args": [], + "py_name": "PetscUtils" + } + ], + "templated": false + }, + { + "base": "PottsMesh", + "instantiations": [ + { + "args": [ + "2" + ], + "py_name": "PottsMesh_2" + }, + { + "args": [ + "3" + ], + "py_name": "PottsMesh_3" + } + ], + "templated": true + }, + { + "base": "MeshFactory", + "instantiations": [ + { + "args": [ + "PottsMesh<2>" + ], + "py_name": "MeshFactory_PottsMesh_2" + }, + { + "args": [ + "PottsMesh<3>" + ], + "py_name": "MeshFactory_PottsMesh_3" + } + ], + "templated": true + }, + { + "base": "Scene", + "instantiations": [ + { + "args": [ + "2" + ], + "py_name": "Scene_2" + }, + { + "args": [ + "3" + ], + "py_name": "Scene_3" + } + ], + "templated": true + }, + { + "base": "SphericalMesh", + "instantiations": [ + { + "args": [ + "2", + "2" + ], + "py_name": "SphericalMesh_2_2" + }, + { + "args": [ + "3", + "3" + ], + "py_name": "SphericalMesh_3_3" + } + ], + "templated": true + } + ], + "compiled_module": "_pycells_all", + "enums": [], + "free_functions": [], + "imports": [], + "name": "all" + } + ], + "package": "pycells" +} diff --git a/examples/cells/src/py/pycells/__init__.py b/examples/cells/src/py/pycells/__init__.py index 17464d7..e456c54 100644 --- a/examples/cells/src/py/pycells/__init__.py +++ b/examples/cells/src/py/pycells/__init__.py @@ -1,113 +1,9 @@ -"""Main pycells module.""" +"""Main pycells module. -from ._pycells_all import ( - Cell, - CellFactory_Cell_2, - CellFactory_Cell_3, - Corner_2, - Facet_2, - MacroMesh_2_2, - MacroMesh_3_3, - MeshFactory_PottsMesh_2, - MeshFactory_PottsMesh_3, - Node_2, - Node_3, - PetscUtils, - PottsMesh_2, - PottsMesh_3, - Scene_2, - Scene_3, - SphericalMesh_2_2, - SphericalMesh_3_3, -) -from ._syntax import TemplateClass +The compiled-extension imports and TemplateClass subscript stubs are generated +by tools/cppwg_initgen.py into _generated.py; add any hand-written code here. +Curation rationale (why Facet<1>/Corner<1>/AbstractMesh are or aren't wrapped) +lives with the C++ sources and examples/cells/dynamic/config.yaml. +""" - -# CellFactory names CELL_TYPE only as a template argument, so -# cppwg's auto_includes resolved Cell.hpp from the instantiation arguments (not -# a signature) - see CellFactory.hpp and examples/cells/dynamic/config.yaml. -class CellFactory(TemplateClass): - _instantiations = { - ("Cell", "2"): CellFactory_Cell_2, - ("Cell", "3"): CellFactory_Cell_3, - } - - -# Only Facet<2> is wrapped (curated). Facet<1> is deliberately left unwrapped; -# wrapping it would reference the never-instantiated Facet<0> and fail to import -# (see Facet.hpp and examples/cells/dynamic/config.yaml). -class Facet(TemplateClass): - _instantiations = { - ("2",): Facet_2, - } - - -# Non-curated counterpart to Facet: discovery finds Corner<1> and Corner<2>, and -# cppwg auto-drops Corner<1> (its GetSub returns the never-instantiated -# Corner<0>), so only Corner<2> is wrapped (see Corner.hpp). -class Corner(TemplateClass): - _instantiations = { - ("2",): Corner_2, - } - - -class MacroMesh(TemplateClass): - _instantiations = { - ("2", "2"): MacroMesh_2_2, - ("3", "3"): MacroMesh_3_3, - } - - -class MeshFactory(TemplateClass): - _instantiations = { - ("PottsMesh", "2"): MeshFactory_PottsMesh_2, - ("PottsMesh", "3"): MeshFactory_PottsMesh_3, - } - - -class Node(TemplateClass): - _instantiations = { - ("2",): Node_2, - ("3",): Node_3, - } - - -class PottsMesh(TemplateClass): - _instantiations = { - ("2",): PottsMesh_2, - ("3",): PottsMesh_3, - } - - -# SphericalMesh is the concrete leaf of AbstractMesh -> AbstractSphericalMesh -# -> SphericalMesh. Its inherited overrides (Scale, GetNumElements) are bound -# on the abstract bases and remain callable through inheritance despite being -# skipped on the leaf by exclude_inherited_overrides. The abstract bases are not -# constructible, so - like AbstractMesh - they are not surfaced here. -class SphericalMesh(TemplateClass): - _instantiations = { - ("2", "2"): SphericalMesh_2_2, - ("3", "3"): SphericalMesh_3_3, - } - - -class Scene(TemplateClass): - _instantiations = { - ("2",): Scene_2, - ("3",): Scene_3, - } - - -__all__ = [ - "Cell", - "CellFactory", - "Corner", - "Facet", - "MacroMesh", - "MeshFactory", - "Node", - "PetscUtils", - "PottsMesh", - "Scene", - "SphericalMesh", -] +from ._generated import * # noqa: F401,F403 diff --git a/examples/cells/src/py/pycells/_generated.py b/examples/cells/src/py/pycells/_generated.py new file mode 100644 index 0000000..201753f --- /dev/null +++ b/examples/cells/src/py/pycells/_generated.py @@ -0,0 +1,85 @@ +"""Generated by tools/cppwg_initgen.py - do not edit. + +Compiled-extension imports and TemplateClass subscript stubs for this +subpackage. Hand-written code (TemplateMethod attachments, package init, +curation) belongs in the sibling __init__.py, which does `from ._generated +import *`. +""" + +from ._pycells_all import * +from pycells._syntax import TemplateClass + + +class CellFactory(TemplateClass): + _instantiations = { + ("Cell", "2"): CellFactory_Cell_2, + ("Cell", "3"): CellFactory_Cell_3, + } + + +class Corner(TemplateClass): + _instantiations = { + ("2",): Corner_2, + } + + +class Facet(TemplateClass): + _instantiations = { + ("2",): Facet_2, + } + + +class MacroMesh(TemplateClass): + _instantiations = { + ("2", "2"): MacroMesh_2_2, + ("3", "3"): MacroMesh_3_3, + } + + +class Node(TemplateClass): + _instantiations = { + ("2",): Node_2, + ("3",): Node_3, + } + + +class AbstractMesh(TemplateClass): + _instantiations = { + ("2", "2"): AbstractMesh_2_2, + ("3", "3"): AbstractMesh_3_3, + } + + +class AbstractSphericalMesh(TemplateClass): + _instantiations = { + ("2", "2"): AbstractSphericalMesh_2_2, + ("3", "3"): AbstractSphericalMesh_3_3, + } + + +class PottsMesh(TemplateClass): + _instantiations = { + ("2",): PottsMesh_2, + ("3",): PottsMesh_3, + } + + +class MeshFactory(TemplateClass): + _instantiations = { + ("PottsMesh<2>",): MeshFactory_PottsMesh_2, + ("PottsMesh<3>",): MeshFactory_PottsMesh_3, + } + + +class Scene(TemplateClass): + _instantiations = { + ("2",): Scene_2, + ("3",): Scene_3, + } + + +class SphericalMesh(TemplateClass): + _instantiations = { + ("2", "2"): SphericalMesh_2_2, + ("3", "3"): SphericalMesh_3_3, + } diff --git a/examples/shapes/src/py/pyshapes/composites/__init__.py b/examples/shapes/src/py/pyshapes/composites/__init__.py index 35aa80f..fe9c6c5 100644 --- a/examples/shapes/src/py/pyshapes/composites/__init__.py +++ b/examples/shapes/src/py/pyshapes/composites/__init__.py @@ -1,2 +1,3 @@ -# Bring in everything from the shared module -from pyshapes.composites._pyshapes_composites import * +# The compiled-extension imports and TemplateClass stubs are generated by +# tools/cppwg_initgen.py into _generated.py; add any hand-written code here. +from ._generated import * # noqa: F401,F403 diff --git a/examples/shapes/src/py/pyshapes/composites/_generated.py b/examples/shapes/src/py/pyshapes/composites/_generated.py new file mode 100644 index 0000000..6f4d9e2 --- /dev/null +++ b/examples/shapes/src/py/pyshapes/composites/_generated.py @@ -0,0 +1,9 @@ +"""Generated by tools/cppwg_initgen.py - do not edit. + +Compiled-extension imports and TemplateClass subscript stubs for this +subpackage. Hand-written code (TemplateMethod attachments, package init, +curation) belongs in the sibling __init__.py, which does `from ._generated +import *`. +""" + +from ._pyshapes_composites import * diff --git a/examples/shapes/src/py/pyshapes/geometry/__init__.py b/examples/shapes/src/py/pyshapes/geometry/__init__.py index 9cd02bb..fe9c6c5 100644 --- a/examples/shapes/src/py/pyshapes/geometry/__init__.py +++ b/examples/shapes/src/py/pyshapes/geometry/__init__.py @@ -1,10 +1,3 @@ -# Bring in everything from the shared module -from pyshapes._syntax import TemplateClass -from pyshapes.geometry._pyshapes_geometry import * - - -class Point(TemplateClass): - _instantiations = { - 2: Point_2, - 3: Point_3, - } +# The compiled-extension imports and TemplateClass stubs are generated by +# tools/cppwg_initgen.py into _generated.py; add any hand-written code here. +from ._generated import * # noqa: F401,F403 diff --git a/examples/shapes/src/py/pyshapes/geometry/_generated.py b/examples/shapes/src/py/pyshapes/geometry/_generated.py new file mode 100644 index 0000000..f836db4 --- /dev/null +++ b/examples/shapes/src/py/pyshapes/geometry/_generated.py @@ -0,0 +1,17 @@ +"""Generated by tools/cppwg_initgen.py - do not edit. + +Compiled-extension imports and TemplateClass subscript stubs for this +subpackage. Hand-written code (TemplateMethod attachments, package init, +curation) belongs in the sibling __init__.py, which does `from ._generated +import *`. +""" + +from ._pyshapes_geometry import * +from pyshapes._syntax import TemplateClass + + +class Point(TemplateClass): + _instantiations = { + ("2",): Point_2, + ("3",): Point_3, + } diff --git a/examples/shapes/src/py/pyshapes/math_funcs/__init__.py b/examples/shapes/src/py/pyshapes/math_funcs/__init__.py index 53f3fe0..fe9c6c5 100644 --- a/examples/shapes/src/py/pyshapes/math_funcs/__init__.py +++ b/examples/shapes/src/py/pyshapes/math_funcs/__init__.py @@ -1,2 +1,3 @@ -# Bring in everything from the shared module -from pyshapes.math_funcs._pyshapes_math_funcs import * +# The compiled-extension imports and TemplateClass stubs are generated by +# tools/cppwg_initgen.py into _generated.py; add any hand-written code here. +from ._generated import * # noqa: F401,F403 diff --git a/examples/shapes/src/py/pyshapes/math_funcs/_generated.py b/examples/shapes/src/py/pyshapes/math_funcs/_generated.py new file mode 100644 index 0000000..46924ad --- /dev/null +++ b/examples/shapes/src/py/pyshapes/math_funcs/_generated.py @@ -0,0 +1,9 @@ +"""Generated by tools/cppwg_initgen.py - do not edit. + +Compiled-extension imports and TemplateClass subscript stubs for this +subpackage. Hand-written code (TemplateMethod attachments, package init, +curation) belongs in the sibling __init__.py, which does `from ._generated +import *`. +""" + +from ._pyshapes_math_funcs import * diff --git a/examples/shapes/src/py/pyshapes/primitives/__init__.py b/examples/shapes/src/py/pyshapes/primitives/__init__.py index 872ae91..1ebac1a 100644 --- a/examples/shapes/src/py/pyshapes/primitives/__init__.py +++ b/examples/shapes/src/py/pyshapes/primitives/__init__.py @@ -1,17 +1,11 @@ -# Bring in everything from the shared module -from pyshapes._syntax import TemplateClass, TemplateMethod -from pyshapes.primitives._pyshapes_primitives import * - - -class Shape(TemplateClass): - _instantiations = { - 2: Shape_2, - 3: Shape_3, - } - +# The compiled-extension imports and TemplateClass stubs are generated by +# tools/cppwg_initgen.py into _generated.py; add any hand-written code here. +from ._generated import * # noqa: F401,F403 +from pyshapes._syntax import TemplateMethod # UnitSquare::GetAreaIn() is a templated method (see GetAreaInCustomTemplate.py); # expose its per-unit bindings as GetAreaIn[Unit](). GetAreaIn is also a plain # overload (GetAreaIn(perSquareMetre)); pass it as the fallback so the descriptor # does not shadow it and UnitSquare.GetAreaIn(factor) keeps working. +# (TemplateMethod attachments cannot be auto-generated, so they live here.) UnitSquare.GetAreaIn = TemplateMethod("GetAreaIn", UnitSquare.GetAreaIn) diff --git a/examples/shapes/src/py/pyshapes/primitives/_generated.py b/examples/shapes/src/py/pyshapes/primitives/_generated.py new file mode 100644 index 0000000..db972a0 --- /dev/null +++ b/examples/shapes/src/py/pyshapes/primitives/_generated.py @@ -0,0 +1,38 @@ +"""Generated by tools/cppwg_initgen.py - do not edit. + +Compiled-extension imports and TemplateClass subscript stubs for this +subpackage. Hand-written code (TemplateMethod attachments, package init, +curation) belongs in the sibling __init__.py, which does `from ._generated +import *`. +""" + +from ._pyshapes_primitives import * +from pyshapes._syntax import TemplateClass + + +class AbstractShape(TemplateClass): + _instantiations = { + ("2",): AbstractShape_2, + ("3",): AbstractShape_3, + } + + +class AbstractPolygon(TemplateClass): + _instantiations = { + ("2",): AbstractPolygon_2, + ("3",): AbstractPolygon_3, + } + + +class RegularPolygon(TemplateClass): + _instantiations = { + ("2",): RegularPolygon_2, + ("3",): RegularPolygon_3, + } + + +class Shape(TemplateClass): + _instantiations = { + ("2",): Shape_2, + ("3",): Shape_3, + } diff --git a/examples/shapes/wrapper/cppwg_model.json b/examples/shapes/wrapper/cppwg_model.json new file mode 100644 index 0000000..4499147 --- /dev/null +++ b/examples/shapes/wrapper/cppwg_model.json @@ -0,0 +1,221 @@ +{ + "modules": [ + { + "classes": [], + "compiled_module": "_pyshapes_math_funcs", + "enums": [], + "free_functions": [ + "add", + "throw_exception", + "throw_unwrapped_exception" + ], + "imports": [], + "name": "math_funcs" + }, + { + "classes": [ + { + "base": "Point", + "instantiations": [ + { + "args": [ + "2" + ], + "py_name": "Point_2" + }, + { + "args": [ + "3" + ], + "py_name": "Point_3" + } + ], + "templated": true + } + ], + "compiled_module": "_pyshapes_geometry", + "enums": [], + "free_functions": [], + "imports": [], + "name": "geometry" + }, + { + "classes": [ + { + "base": "AbstractShape", + "instantiations": [ + { + "args": [ + "2" + ], + "py_name": "AbstractShape_2" + }, + { + "args": [ + "3" + ], + "py_name": "AbstractShape_3" + } + ], + "templated": true + }, + { + "base": "AbstractPolygon", + "instantiations": [ + { + "args": [ + "2" + ], + "py_name": "AbstractPolygon_2" + }, + { + "args": [ + "3" + ], + "py_name": "AbstractPolygon_3" + } + ], + "templated": true + }, + { + "base": "RegularPolygon", + "instantiations": [ + { + "args": [ + "2" + ], + "py_name": "RegularPolygon_2" + }, + { + "args": [ + "3" + ], + "py_name": "RegularPolygon_3" + } + ], + "templated": true + }, + { + "base": "Shape", + "instantiations": [ + { + "args": [ + "2" + ], + "py_name": "Shape_2" + }, + { + "args": [ + "3" + ], + "py_name": "Shape_3" + } + ], + "templated": true + }, + { + "base": "Cuboid", + "instantiations": [ + { + "args": [], + "py_name": "Cuboid" + } + ], + "templated": false + }, + { + "base": "Rectangle", + "instantiations": [ + { + "args": [], + "py_name": "Rectangle" + } + ], + "templated": false + }, + { + "base": "ShapeClassifier", + "instantiations": [ + { + "args": [], + "py_name": "ShapeClassifier" + } + ], + "templated": false + }, + { + "base": "ShapeMetrics", + "instantiations": [ + { + "args": [], + "py_name": "ShapeMetrics" + } + ], + "templated": false + }, + { + "base": "SquareFeet", + "instantiations": [ + { + "args": [], + "py_name": "SquareFeet" + } + ], + "templated": false + }, + { + "base": "SquareMetres", + "instantiations": [ + { + "args": [], + "py_name": "SquareMetres" + } + ], + "templated": false + }, + { + "base": "UnitSquare", + "instantiations": [ + { + "args": [], + "py_name": "UnitSquare" + } + ], + "templated": false + } + ], + "compiled_module": "_pyshapes_primitives", + "enums": [ + "Handedness", + "ShapeKind" + ], + "free_functions": [], + "imports": [ + "pyshapes.geometry._pyshapes_geometry" + ], + "name": "primitives" + }, + { + "classes": [ + { + "base": "Square", + "instantiations": [ + { + "args": [], + "py_name": "Square" + } + ], + "templated": false + } + ], + "compiled_module": "_pyshapes_composites", + "enums": [], + "free_functions": [], + "imports": [ + "pyshapes.primitives._pyshapes_primitives" + ], + "name": "composites" + } + ], + "package": "pyshapes" +} diff --git a/examples/shapes/wrapper/py_layout.yaml b/examples/shapes/wrapper/py_layout.yaml new file mode 100644 index 0000000..83c2a7d --- /dev/null +++ b/examples/shapes/wrapper/py_layout.yaml @@ -0,0 +1,9 @@ +# Python-package layout for tools/cppwg_initgen.py (shapes example). +# +# Module-per-subpackage: each cppwg module (geometry/primitives/composites/ +# math_funcs) is a subpackage that owns its own compiled extension, so each +# _generated.py does `from .<_pyshapes_module> import *`. +# +# package_root is resolved relative to this manifest's directory. +package: pyshapes +package_root: ../src/py/pyshapes From 38d22993299050999f497627bb49d231e2dd4503 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Tue, 11 Aug 2026 18:15:37 +0100 Subject: [PATCH 04/24] #102 Emit the model as YAML instead of JSON Switch cppwg_model from JSON to YAML: it is human-readable and reuses the config format cppwg already depends on, so no new dependency and one fewer format in the tree. write_python_model() now uses yaml.safe_dump and cppwg_initgen.py reads it with yaml.safe_load; the model dict and the generated _generated.py are unchanged. Co-Authored-By: Claude Opus 4.8 --- cppwg/generators.py | 10 +- cppwg/utils/constants.py | 2 +- cppwg/utils/python_model.py | 2 +- .../cells/dynamic/wrappers/cppwg_model.json | 230 ------------------ .../cells/dynamic/wrappers/cppwg_model.yaml | 121 +++++++++ examples/shapes/wrapper/cppwg_model.json | 221 ----------------- examples/shapes/wrapper/cppwg_model.yaml | 118 +++++++++ tools/cppwg_initgen.py | 11 +- 8 files changed, 251 insertions(+), 464 deletions(-) delete mode 100644 examples/cells/dynamic/wrappers/cppwg_model.json create mode 100644 examples/cells/dynamic/wrappers/cppwg_model.yaml delete mode 100644 examples/shapes/wrapper/cppwg_model.json create mode 100644 examples/shapes/wrapper/cppwg_model.yaml diff --git a/cppwg/generators.py b/cppwg/generators.py index c78f8dc..180974d 100644 --- a/cppwg/generators.py +++ b/cppwg/generators.py @@ -1,6 +1,5 @@ """The main interface for generating Python wrappers.""" -import json import logging import os import re @@ -10,6 +9,7 @@ from pathlib import Path import pygccxml +import yaml from cppwg.info.package_info import PackageInfo from cppwg.parsers.package_info_parser import PackageInfoParser @@ -427,16 +427,16 @@ def write_wrappers(self) -> None: def write_python_model(self) -> None: """ - Write the Python-package model (cppwg_model.json) to the wrapper root. + Write the Python-package model (cppwg_model.yaml) to the wrapper root. - A small JSON description of the generated modules and their classes / + A small YAML description of the generated modules and their classes / instantiations / enums / free functions, so a separate step (tools/cppwg_initgen.py) can generate the Python package layer without re-parsing the source. Written last, once the info tree is final. """ model = build_python_model(self.package_info) model_path = os.path.join(self.wrapper_root, CPPWG_PYTHON_MODEL_FILENAME) - content = json.dumps(model, indent=2, sort_keys=True) + "\n" + content = yaml.safe_dump(model, default_flow_style=False, sort_keys=True) utils.write_file_if_changed(model_path, content, self.overwrite) def generate(self) -> None: @@ -500,6 +500,6 @@ def generate(self) -> None: # Write the wrapper code for the package self.write_wrappers() - # Write the Python-package model (cppwg_model.json) for the package-layer + # Write the Python-package model (cppwg_model.yaml) for the package-layer # generator (tools/cppwg_initgen.py). self.write_python_model() diff --git a/cppwg/utils/constants.py b/cppwg/utils/constants.py index 1b1aa88..709761b 100644 --- a/cppwg/utils/constants.py +++ b/cppwg/utils/constants.py @@ -12,7 +12,7 @@ # The Python-package model cppwg writes into the wrapper root, describing the # generated modules/classes so a separate step can build the Python package # layer (see cppwg.utils.python_model and tools/cppwg_initgen.py). -CPPWG_PYTHON_MODEL_FILENAME = "cppwg_model.json" +CPPWG_PYTHON_MODEL_FILENAME = "cppwg_model.yaml" CPPWG_TRUE_STRINGS = ["ON", "YES", "Y", "TRUE", "T", "1"] CPPWG_FALSE_STRINGS = ["OFF", "NO", "N", "FALSE", "F", "0", ""] diff --git a/cppwg/utils/python_model.py b/cppwg/utils/python_model.py index e205d30..9dfbf15 100644 --- a/cppwg/utils/python_model.py +++ b/cppwg/utils/python_model.py @@ -7,7 +7,7 @@ module name, the wrapped classes with their template instantiations, and the enum / free-function names. All of this is on the finalized ``PackageInfo`` tree but not in a form a standalone script can consume, so ``build_python_model`` -distils it into a plain dict that cppwg writes out as ``cppwg_model.json``. +distils it into a plain dict that cppwg writes out as ``cppwg_model.yaml``. """ from typing import TYPE_CHECKING, Any diff --git a/examples/cells/dynamic/wrappers/cppwg_model.json b/examples/cells/dynamic/wrappers/cppwg_model.json deleted file mode 100644 index b12b07a..0000000 --- a/examples/cells/dynamic/wrappers/cppwg_model.json +++ /dev/null @@ -1,230 +0,0 @@ -{ - "modules": [ - { - "classes": [ - { - "base": "Cell", - "instantiations": [ - { - "args": [], - "py_name": "Cell" - } - ], - "templated": false - }, - { - "base": "CellFactory", - "instantiations": [ - { - "args": [ - "Cell", - "2" - ], - "py_name": "CellFactory_Cell_2" - }, - { - "args": [ - "Cell", - "3" - ], - "py_name": "CellFactory_Cell_3" - } - ], - "templated": true - }, - { - "base": "Corner", - "instantiations": [ - { - "args": [ - "2" - ], - "py_name": "Corner_2" - } - ], - "templated": true - }, - { - "base": "Facet", - "instantiations": [ - { - "args": [ - "2" - ], - "py_name": "Facet_2" - } - ], - "templated": true - }, - { - "base": "MacroMesh", - "instantiations": [ - { - "args": [ - "2", - "2" - ], - "py_name": "MacroMesh_2_2" - }, - { - "args": [ - "3", - "3" - ], - "py_name": "MacroMesh_3_3" - } - ], - "templated": true - }, - { - "base": "Node", - "instantiations": [ - { - "args": [ - "2" - ], - "py_name": "Node_2" - }, - { - "args": [ - "3" - ], - "py_name": "Node_3" - } - ], - "templated": true - }, - { - "base": "AbstractMesh", - "instantiations": [ - { - "args": [ - "2", - "2" - ], - "py_name": "AbstractMesh_2_2" - }, - { - "args": [ - "3", - "3" - ], - "py_name": "AbstractMesh_3_3" - } - ], - "templated": true - }, - { - "base": "AbstractSphericalMesh", - "instantiations": [ - { - "args": [ - "2", - "2" - ], - "py_name": "AbstractSphericalMesh_2_2" - }, - { - "args": [ - "3", - "3" - ], - "py_name": "AbstractSphericalMesh_3_3" - } - ], - "templated": true - }, - { - "base": "PetscUtils", - "instantiations": [ - { - "args": [], - "py_name": "PetscUtils" - } - ], - "templated": false - }, - { - "base": "PottsMesh", - "instantiations": [ - { - "args": [ - "2" - ], - "py_name": "PottsMesh_2" - }, - { - "args": [ - "3" - ], - "py_name": "PottsMesh_3" - } - ], - "templated": true - }, - { - "base": "MeshFactory", - "instantiations": [ - { - "args": [ - "PottsMesh<2>" - ], - "py_name": "MeshFactory_PottsMesh_2" - }, - { - "args": [ - "PottsMesh<3>" - ], - "py_name": "MeshFactory_PottsMesh_3" - } - ], - "templated": true - }, - { - "base": "Scene", - "instantiations": [ - { - "args": [ - "2" - ], - "py_name": "Scene_2" - }, - { - "args": [ - "3" - ], - "py_name": "Scene_3" - } - ], - "templated": true - }, - { - "base": "SphericalMesh", - "instantiations": [ - { - "args": [ - "2", - "2" - ], - "py_name": "SphericalMesh_2_2" - }, - { - "args": [ - "3", - "3" - ], - "py_name": "SphericalMesh_3_3" - } - ], - "templated": true - } - ], - "compiled_module": "_pycells_all", - "enums": [], - "free_functions": [], - "imports": [], - "name": "all" - } - ], - "package": "pycells" -} diff --git a/examples/cells/dynamic/wrappers/cppwg_model.yaml b/examples/cells/dynamic/wrappers/cppwg_model.yaml new file mode 100644 index 0000000..41341d8 --- /dev/null +++ b/examples/cells/dynamic/wrappers/cppwg_model.yaml @@ -0,0 +1,121 @@ +modules: +- classes: + - base: Cell + instantiations: + - args: [] + py_name: Cell + templated: false + - base: CellFactory + instantiations: + - args: + - Cell + - '2' + py_name: CellFactory_Cell_2 + - args: + - Cell + - '3' + py_name: CellFactory_Cell_3 + templated: true + - base: Corner + instantiations: + - args: + - '2' + py_name: Corner_2 + templated: true + - base: Facet + instantiations: + - args: + - '2' + py_name: Facet_2 + templated: true + - base: MacroMesh + instantiations: + - args: + - '2' + - '2' + py_name: MacroMesh_2_2 + - args: + - '3' + - '3' + py_name: MacroMesh_3_3 + templated: true + - base: Node + instantiations: + - args: + - '2' + py_name: Node_2 + - args: + - '3' + py_name: Node_3 + templated: true + - base: AbstractMesh + instantiations: + - args: + - '2' + - '2' + py_name: AbstractMesh_2_2 + - args: + - '3' + - '3' + py_name: AbstractMesh_3_3 + templated: true + - base: AbstractSphericalMesh + instantiations: + - args: + - '2' + - '2' + py_name: AbstractSphericalMesh_2_2 + - args: + - '3' + - '3' + py_name: AbstractSphericalMesh_3_3 + templated: true + - base: PetscUtils + instantiations: + - args: [] + py_name: PetscUtils + templated: false + - base: PottsMesh + instantiations: + - args: + - '2' + py_name: PottsMesh_2 + - args: + - '3' + py_name: PottsMesh_3 + templated: true + - base: MeshFactory + instantiations: + - args: + - PottsMesh<2> + py_name: MeshFactory_PottsMesh_2 + - args: + - PottsMesh<3> + py_name: MeshFactory_PottsMesh_3 + templated: true + - base: Scene + instantiations: + - args: + - '2' + py_name: Scene_2 + - args: + - '3' + py_name: Scene_3 + templated: true + - base: SphericalMesh + instantiations: + - args: + - '2' + - '2' + py_name: SphericalMesh_2_2 + - args: + - '3' + - '3' + py_name: SphericalMesh_3_3 + templated: true + compiled_module: _pycells_all + enums: [] + free_functions: [] + imports: [] + name: all +package: pycells diff --git a/examples/shapes/wrapper/cppwg_model.json b/examples/shapes/wrapper/cppwg_model.json deleted file mode 100644 index 4499147..0000000 --- a/examples/shapes/wrapper/cppwg_model.json +++ /dev/null @@ -1,221 +0,0 @@ -{ - "modules": [ - { - "classes": [], - "compiled_module": "_pyshapes_math_funcs", - "enums": [], - "free_functions": [ - "add", - "throw_exception", - "throw_unwrapped_exception" - ], - "imports": [], - "name": "math_funcs" - }, - { - "classes": [ - { - "base": "Point", - "instantiations": [ - { - "args": [ - "2" - ], - "py_name": "Point_2" - }, - { - "args": [ - "3" - ], - "py_name": "Point_3" - } - ], - "templated": true - } - ], - "compiled_module": "_pyshapes_geometry", - "enums": [], - "free_functions": [], - "imports": [], - "name": "geometry" - }, - { - "classes": [ - { - "base": "AbstractShape", - "instantiations": [ - { - "args": [ - "2" - ], - "py_name": "AbstractShape_2" - }, - { - "args": [ - "3" - ], - "py_name": "AbstractShape_3" - } - ], - "templated": true - }, - { - "base": "AbstractPolygon", - "instantiations": [ - { - "args": [ - "2" - ], - "py_name": "AbstractPolygon_2" - }, - { - "args": [ - "3" - ], - "py_name": "AbstractPolygon_3" - } - ], - "templated": true - }, - { - "base": "RegularPolygon", - "instantiations": [ - { - "args": [ - "2" - ], - "py_name": "RegularPolygon_2" - }, - { - "args": [ - "3" - ], - "py_name": "RegularPolygon_3" - } - ], - "templated": true - }, - { - "base": "Shape", - "instantiations": [ - { - "args": [ - "2" - ], - "py_name": "Shape_2" - }, - { - "args": [ - "3" - ], - "py_name": "Shape_3" - } - ], - "templated": true - }, - { - "base": "Cuboid", - "instantiations": [ - { - "args": [], - "py_name": "Cuboid" - } - ], - "templated": false - }, - { - "base": "Rectangle", - "instantiations": [ - { - "args": [], - "py_name": "Rectangle" - } - ], - "templated": false - }, - { - "base": "ShapeClassifier", - "instantiations": [ - { - "args": [], - "py_name": "ShapeClassifier" - } - ], - "templated": false - }, - { - "base": "ShapeMetrics", - "instantiations": [ - { - "args": [], - "py_name": "ShapeMetrics" - } - ], - "templated": false - }, - { - "base": "SquareFeet", - "instantiations": [ - { - "args": [], - "py_name": "SquareFeet" - } - ], - "templated": false - }, - { - "base": "SquareMetres", - "instantiations": [ - { - "args": [], - "py_name": "SquareMetres" - } - ], - "templated": false - }, - { - "base": "UnitSquare", - "instantiations": [ - { - "args": [], - "py_name": "UnitSquare" - } - ], - "templated": false - } - ], - "compiled_module": "_pyshapes_primitives", - "enums": [ - "Handedness", - "ShapeKind" - ], - "free_functions": [], - "imports": [ - "pyshapes.geometry._pyshapes_geometry" - ], - "name": "primitives" - }, - { - "classes": [ - { - "base": "Square", - "instantiations": [ - { - "args": [], - "py_name": "Square" - } - ], - "templated": false - } - ], - "compiled_module": "_pyshapes_composites", - "enums": [], - "free_functions": [], - "imports": [ - "pyshapes.primitives._pyshapes_primitives" - ], - "name": "composites" - } - ], - "package": "pyshapes" -} diff --git a/examples/shapes/wrapper/cppwg_model.yaml b/examples/shapes/wrapper/cppwg_model.yaml new file mode 100644 index 0000000..2550420 --- /dev/null +++ b/examples/shapes/wrapper/cppwg_model.yaml @@ -0,0 +1,118 @@ +modules: +- classes: [] + compiled_module: _pyshapes_math_funcs + enums: [] + free_functions: + - add + - throw_exception + - throw_unwrapped_exception + imports: [] + name: math_funcs +- classes: + - base: Point + instantiations: + - args: + - '2' + py_name: Point_2 + - args: + - '3' + py_name: Point_3 + templated: true + compiled_module: _pyshapes_geometry + enums: [] + free_functions: [] + imports: [] + name: geometry +- classes: + - base: AbstractShape + instantiations: + - args: + - '2' + py_name: AbstractShape_2 + - args: + - '3' + py_name: AbstractShape_3 + templated: true + - base: AbstractPolygon + instantiations: + - args: + - '2' + py_name: AbstractPolygon_2 + - args: + - '3' + py_name: AbstractPolygon_3 + templated: true + - base: RegularPolygon + instantiations: + - args: + - '2' + py_name: RegularPolygon_2 + - args: + - '3' + py_name: RegularPolygon_3 + templated: true + - base: Shape + instantiations: + - args: + - '2' + py_name: Shape_2 + - args: + - '3' + py_name: Shape_3 + templated: true + - base: Cuboid + instantiations: + - args: [] + py_name: Cuboid + templated: false + - base: Rectangle + instantiations: + - args: [] + py_name: Rectangle + templated: false + - base: ShapeClassifier + instantiations: + - args: [] + py_name: ShapeClassifier + templated: false + - base: ShapeMetrics + instantiations: + - args: [] + py_name: ShapeMetrics + templated: false + - base: SquareFeet + instantiations: + - args: [] + py_name: SquareFeet + templated: false + - base: SquareMetres + instantiations: + - args: [] + py_name: SquareMetres + templated: false + - base: UnitSquare + instantiations: + - args: [] + py_name: UnitSquare + templated: false + compiled_module: _pyshapes_primitives + enums: + - Handedness + - ShapeKind + free_functions: [] + imports: + - pyshapes.geometry._pyshapes_geometry + name: primitives +- classes: + - base: Square + instantiations: + - args: [] + py_name: Square + templated: false + compiled_module: _pyshapes_composites + enums: [] + free_functions: [] + imports: + - pyshapes.primitives._pyshapes_primitives + name: composites +package: pyshapes diff --git a/tools/cppwg_initgen.py b/tools/cppwg_initgen.py index 9b278fb..bd65a20 100644 --- a/tools/cppwg_initgen.py +++ b/tools/cppwg_initgen.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """Generate the Python package layer (``_generated.py``) for a cppwg project. -cppwg generates the C++/pybind11 wrappers and, alongside them, a JSON model of -what it produced (``cppwg_model.json`` in the wrapper root - see +cppwg generates the C++/pybind11 wrappers and, alongside them, a YAML model of +what it produced (``cppwg_model.yaml`` in the wrapper root - see ``cppwg.utils.python_model``). This standalone script turns that model, plus a small layout manifest, into a ``_generated.py`` per Python subpackage: the compiled-extension import and the ``TemplateClass`` subscript stubs @@ -22,11 +22,10 @@ Usage:: - cppwg_initgen.py --model wrapper/cppwg_model.json --manifest py_layout.yaml + cppwg_initgen.py --model wrapper/cppwg_model.yaml --manifest py_layout.yaml """ import argparse -import json import os import sys @@ -246,7 +245,7 @@ def main(argv=None) -> int: """Entry point: read the model + manifest and write the _generated.py files.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - "--model", required=True, help="Path to cppwg_model.json (from cppwg)." + "--model", required=True, help="Path to cppwg_model.yaml (from cppwg)." ) parser.add_argument( "--manifest", required=True, help="Path to the Python layout manifest (YAML)." @@ -261,7 +260,7 @@ def main(argv=None) -> int: import yaml with open(args.model) as model_file: - model = json.load(model_file) + model = yaml.safe_load(model_file) with open(args.manifest) as manifest_file: manifest = yaml.safe_load(manifest_file) From 5ccde676dfdefc1f175bac7c0955a2e0ddb77d4a Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Tue, 11 Aug 2026 19:12:56 +0100 Subject: [PATCH 05/24] #102 Add a flatten-to-root option for top-level class access Opt-in manifest flag `flatten_to_root: true` makes the generator also write a top-level _generated.py that re-exports every subpackage's class, enum and free function into the package root, so `package.ClassName` works in addition to `package.subpackage.ClassName` (PyChaste issue #73). It re-exports the base (stub) names, not the concrete instantiations, and warns if a name is exported by more than one subpackage. Only the shared-module-split layout uses it; shapes/cells leave it off. Co-Authored-By: Claude Opus 4.8 --- tests/test_initgen.py | 65 ++++++++++++++++++++++++++++++++++++++++++ tools/cppwg_initgen.py | 55 +++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) diff --git a/tests/test_initgen.py b/tests/test_initgen.py index dec1609..8233ea3 100644 --- a/tests/test_initgen.py +++ b/tests/test_initgen.py @@ -182,6 +182,71 @@ def test_generate_shared_module_split(tmp_path, capsys): assert "class PottsMesh(TemplateClass):" in mesh +def test_flatten_to_root(tmp_path): + """flatten_to_root emits a top-level _generated.py re-exporting every name.""" + model = { + "package": "pychaste", + "modules": [ + { + "name": "all", + "compiled_module": "_pychaste_all", + "imports": [], + "classes": [ + _class("Node", [_inst(["2"], "Node_2")]), + _class("FileFinder", [_inst([], "FileFinder")], templated=False), + ], + "enums": ["RelativeTo"], + "free_functions": [], + } + ], + } + manifest = { + "package": "chaste", + "package_root": str(tmp_path), + "compiled_module": "_pychaste_all", + "flatten_to_root": True, + "subpackages": {"core": ["FileFinder", "RelativeTo"], "mesh": ["Node"]}, + } + + initgen.generate_shared_module_split(model, manifest, overwrite=False) + + root = (tmp_path / "_generated.py").read_text() + # Re-exports the base/enum names (not the concrete Node_2) from each subpackage. + assert "from chaste.core import (" in root + assert "from chaste.mesh import (" in root + assert "FileFinder," in root and "RelativeTo," in root and "Node," in root + assert "Node_2" not in root # the stub name is flattened, not the concrete + assert '"FileFinder",' in root and '"Node",' in root # __all__ + assert "__all__ = [" in root + + +def test_flatten_to_root_warns_on_name_clash(tmp_path, capsys): + """A base name exported by two subpackages is flagged as ambiguous.""" + model = { + "package": "pkg", + "modules": [ + { + "name": "all", + "compiled_module": "_pkg_all", + "imports": [], + "classes": [_class("Dup", [_inst([], "Dup")], templated=False)], + "enums": [], + "free_functions": [], + } + ], + } + manifest = { + "package": "pkg", + "package_root": str(tmp_path), + "compiled_module": "_pkg_all", + "flatten_to_root": True, + "subpackages": {"a": ["Dup"], "b": ["Dup"]}, + } + + initgen.generate_shared_module_split(model, manifest, overwrite=False) + assert "'Dup'" in capsys.readouterr().err # ambiguous top-level export + + def test_shared_split_warns_on_unknown_and_unassigned(tmp_path, capsys): model = { "package": "pkg", diff --git a/tools/cppwg_initgen.py b/tools/cppwg_initgen.py index bd65a20..869bec0 100644 --- a/tools/cppwg_initgen.py +++ b/tools/cppwg_initgen.py @@ -58,6 +58,18 @@ def write_file_if_changed(filepath, content, overwrite=False): ) +FLATTEN_HEADER = ( + '"""Generated by tools/cppwg_initgen.py - do not edit.\n' + "\n" + "Re-exports every wrapped class, enum and free function from the subpackages\n" + "into the top-level package namespace, so `package.ClassName` works as well\n" + "as `package.subpackage.ClassName`. Importing this imports the subpackages.\n" + "The hand-written __init__.py does `from ._generated import *` and adds the\n" + "bespoke top-level pieces (package init, etc.).\n" + '"""\n' +) + + def _key_repr(args: list[str]) -> str: """Render a template-argument list as a Python tuple literal of strings. @@ -210,17 +222,21 @@ def generate_shared_module_split(model: dict, manifest: dict, overwrite: bool): other_names.add(name) assigned = set() + flatten_names = {} # subpackage -> the top-level names it exposes (issue #73) for subpkg, names in manifest["subpackages"].items(): classes = [] import_names = [] + exported = [] for name in names: assigned.add(name) if name in classes_by_base: class_info = classes_by_base[name] classes.append(class_info) import_names.extend(_concrete_names(class_info)) + exported.append(name) elif name in other_names: import_names.append(name) + exported.append(name) else: print(f"warning: '{name}' ({subpkg}) not found in model", file=sys.stderr) @@ -231,6 +247,7 @@ def generate_shared_module_split(model: dict, manifest: dict, overwrite: bool): ) path = os.path.join(package_root, subpkg, "_generated.py") _write_generated(path, content, overwrite) + flatten_names[subpkg] = exported # Flag any wrapped class not placed in a subpackage, so nothing is silently # dropped when the config gains a class. @@ -240,6 +257,44 @@ def generate_shared_module_split(model: dict, manifest: dict, overwrite: bool): print(f"warning: '{name}' is wrapped but not assigned to a subpackage", file=sys.stderr) + if manifest.get("flatten_to_root"): + generate_root_flatten(package, package_root, flatten_names, overwrite) + + +def generate_root_flatten( + package: str, package_root: str, flatten_names: dict, overwrite: bool +) -> None: + """Emit a top-level ``_generated.py`` re-exporting every subpackage's names. + + Gives ``package.ClassName`` in addition to ``package.subpackage.ClassName`` + (issue #73). Names are expected to be unique across subpackages; a clash is + warned about (the last subpackage's binding would win at import time). + """ + owner = {} # name -> subpackage that first exported it, for a clash check + for subpkg in sorted(flatten_names): + for name in flatten_names[subpkg]: + if name in owner: + print( + f"warning: '{name}' is exported by both '{owner[name]}' and " + f"'{subpkg}'; top-level {package}.{name} would be ambiguous", + file=sys.stderr, + ) + else: + owner[name] = subpkg + + lines = [FLATTEN_HEADER.rstrip("\n"), ""] + for subpkg in sorted(flatten_names): + names = flatten_names[subpkg] + if names: + lines.append(_explicit_import(package, subpkg, names)) + lines.append("") + lines.append("__all__ = [") + for name in sorted(owner): + lines.append(f' "{name}",') + lines.append("]") + content = "\n".join(lines) + "\n" + _write_generated(os.path.join(package_root, "_generated.py"), content, overwrite) + def main(argv=None) -> int: """Entry point: read the model + manifest and write the _generated.py files.""" From 9b427c86aa5e7487f2ee0bf5ccb60a88c51b4382 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Tue, 11 Aug 2026 21:07:42 +0100 Subject: [PATCH 06/24] #102 Key a nested templated template-argument by its Python class name A template argument that is itself a wrapped templated type - cells' MeshFactory> - is now keyed in the generated stub by the Python concrete class name (PottsMesh_2) instead of the raw C++ type string (PottsMesh<2>). So the natural MeshFactory[PottsMesh[2]] resolves: PottsMesh[2] is the PottsMesh_2 class and _normalize_key keys it by __name__. The split-argument MeshFactory[PottsMesh, 2] form is dropped. _build_cxx_index maps each wrapped instantiation's C++ type string to its py_name; only cells' MeshFactory has such an argument, so shapes and pychaste are unchanged. Adds a cells test exercising MeshFactory[PottsMesh[2]]. Co-Authored-By: Claude Opus 4.8 --- examples/cells/src/py/pycells/_generated.py | 4 +- examples/cells/tests/test_cells.py | 15 +++++++ tests/test_initgen.py | 33 ++++++++++++++ tools/cppwg_initgen.py | 48 +++++++++++++++++++-- 4 files changed, 94 insertions(+), 6 deletions(-) diff --git a/examples/cells/src/py/pycells/_generated.py b/examples/cells/src/py/pycells/_generated.py index 201753f..dd294c0 100644 --- a/examples/cells/src/py/pycells/_generated.py +++ b/examples/cells/src/py/pycells/_generated.py @@ -66,8 +66,8 @@ class PottsMesh(TemplateClass): class MeshFactory(TemplateClass): _instantiations = { - ("PottsMesh<2>",): MeshFactory_PottsMesh_2, - ("PottsMesh<3>",): MeshFactory_PottsMesh_3, + ("PottsMesh_2",): MeshFactory_PottsMesh_2, + ("PottsMesh_3",): MeshFactory_PottsMesh_3, } diff --git a/examples/cells/tests/test_cells.py b/examples/cells/tests/test_cells.py index 5fe46c0..b44e70b 100644 --- a/examples/cells/tests/test_cells.py +++ b/examples/cells/tests/test_cells.py @@ -9,8 +9,10 @@ Corner, Facet, MacroMesh, + MeshFactory, Node, PetscUtils, + PottsMesh, Scene, SphericalMesh, ) @@ -55,6 +57,19 @@ def testMacroInstantiationFallback(self): self.assertEqual(MacroMesh[2, 2]().GetDimension(), 2) self.assertEqual(MacroMesh[3, 3]().GetDimension(), 3) + def testNestedTemplateArgSubscript(self): + # MeshFactory has a single template argument that is itself a + # templated type - MeshFactory>. It is subscripted with the + # Python mesh class, MeshFactory[PottsMesh[2]] (PottsMesh[2] is the + # PottsMesh_2 class), rather than splitting the argument as + # MeshFactory[PottsMesh, 2]. + factory = MeshFactory[PottsMesh[2]]() + self.assertIsInstance(factory.generateMesh(), PottsMesh[2]) + self.assertIsInstance(MeshFactory[PottsMesh[3]]().generateMesh(), PottsMesh[3]) + # The split-argument form is not a valid key. + with self.assertRaises(KeyError): + _ = MeshFactory[PottsMesh, 2] + def testInheritedOverrideStillCallable(self): # SphericalMesh is the concrete leaf of an abstract chain # (AbstractMesh -> AbstractSphericalMesh -> SphericalMesh). Its diff --git a/tests/test_initgen.py b/tests/test_initgen.py index 8233ea3..40fd714 100644 --- a/tests/test_initgen.py +++ b/tests/test_initgen.py @@ -55,6 +55,39 @@ def test_stub_source_diagonal_shorthand(): assert '("2",):' not in plain +def test_stub_source_nested_templated_arg(): + """A templated-type argument is keyed by its Python concrete class name.""" + # MeshFactory> -> key ("PottsMesh_2",), not ("PottsMesh<2>",), + # so MeshFactory[PottsMesh[2]] resolves (PottsMesh[2] is the PottsMesh_2 class). + cxx_to_pyname = {"PottsMesh<2>": "PottsMesh_2", "PottsMesh<3>": "PottsMesh_3"} + stub = initgen._stub_source( + "MeshFactory", + [ + _inst(["PottsMesh<2>"], "MeshFactory_PottsMesh_2"), + _inst(["PottsMesh<3>"], "MeshFactory_PottsMesh_3"), + ], + cxx_to_pyname=cxx_to_pyname, + ) + assert '("PottsMesh_2",): MeshFactory_PottsMesh_2,' in stub + assert '("PottsMesh_3",): MeshFactory_PottsMesh_3,' in stub + assert "<" not in stub # the raw C++ type string is gone + + +def test_build_cxx_index(): + model = { + "modules": [ + { + "classes": [ + _class("PottsMesh", [_inst(["2"], "PottsMesh_2")]), + _class("Cell", [_inst([], "Cell")], templated=False), + ] + } + ] + } + index = initgen._build_cxx_index(model) + assert index == {"PottsMesh<2>": "PottsMesh_2"} # untemplated Cell has no <...> + + def test_render_generated_module_with_stub_imports_syntax(): point = _class("Point", [_inst(["2"], "Point_2"), _inst(["3"], "Point_3")]) content = initgen.render_generated_module( diff --git a/tools/cppwg_initgen.py b/tools/cppwg_initgen.py index 869bec0..2b2057b 100644 --- a/tools/cppwg_initgen.py +++ b/tools/cppwg_initgen.py @@ -82,11 +82,36 @@ def _key_repr(args: list[str]) -> str: return f"({inner})" +def _build_cxx_index(model: dict) -> dict: + """Map each wrapped instantiation's C++ type string to its Python class name. + + e.g. ``PottsMesh<2> -> PottsMesh_2``. Used to key a template argument that is + itself a wrapped templated type (``MeshFactory>``) by the Python + concrete class name, so ``MeshFactory[PottsMesh[2]]`` resolves: ``PottsMesh[2]`` + is the ``PottsMesh_2`` class and ``_normalize_key`` keys it by its ``__name__``. + """ + index = {} + for module in model["modules"]: + for class_info in module["classes"]: + for inst in class_info["instantiations"]: + if inst["args"]: + cxx = f'{class_info["base"]}<{",".join(inst["args"])}>' + index[cxx] = inst["py_name"] + return index + + def _stub_source( - base: str, instantiations: list[dict], diagonal_shorthand: bool = False + base: str, + instantiations: list[dict], + diagonal_shorthand: bool = False, + cxx_to_pyname: dict = None, ) -> str: """Render a ``class (TemplateClass)`` stub for a templated class. + A template argument that is itself a wrapped templated type is keyed by its + Python concrete class name via ``cxx_to_pyname`` (``PottsMesh<2>`` -> + ``PottsMesh_2``), so ``MeshFactory[PottsMesh[2]]`` resolves. + When ``diagonal_shorthand`` is set, a multi-argument instantiation whose arguments are all equal (a "diagonal", e.g. ``Element<2, 2>``) also gets a single-argument alias key (``Element[2] -> Element_2_2``). This is an @@ -94,9 +119,10 @@ def _stub_source( ````-style classes; others (the cells example) keep the explicit multi-argument form only. """ + cxx_to_pyname = cxx_to_pyname or {} lines = [f"class {base}(TemplateClass):", " _instantiations = {"] for inst in instantiations: - args = inst["args"] + args = [cxx_to_pyname.get(arg, arg) for arg in inst["args"]] lines.append(f' {_key_repr(args)}: {inst["py_name"]},') if diagonal_shorthand and len(args) > 1 and len(set(args)) == 1: lines.append(f' {_key_repr(args[:1])}: {inst["py_name"]},') @@ -110,6 +136,7 @@ def render_generated_module( classes: list[dict], templated_classes: list[dict], diagonal_shorthand: bool = False, + cxx_to_pyname: dict = None, ) -> str: """ Render a subpackage's ``_generated.py`` content. @@ -144,6 +171,7 @@ def render_generated_module( class_info["base"], class_info["instantiations"], diagonal_shorthand, + cxx_to_pyname, ) ) return "\n".join(lines) + "\n" @@ -190,6 +218,7 @@ def generate_module_per_subpackage(model: dict, manifest: dict, overwrite: bool) package_root = manifest["package_root"] module_dirs = manifest.get("module_dirs", {}) diagonal_shorthand = manifest.get("diagonal_shorthand", False) + cxx_to_pyname = _build_cxx_index(model) for module in model["modules"]: # The subpackage directory (relative to package_root); default = module @@ -198,7 +227,12 @@ def generate_module_per_subpackage(model: dict, manifest: dict, overwrite: bool) compiled_import = f"from .{module['compiled_module']} import *" templated = [c for c in module["classes"] if c["templated"]] content = render_generated_module( - package, compiled_import, module["classes"], templated, diagonal_shorthand + package, + compiled_import, + module["classes"], + templated, + diagonal_shorthand, + cxx_to_pyname, ) path = os.path.join(package_root, subdir, "_generated.py") _write_generated(path, content, overwrite) @@ -210,6 +244,7 @@ def generate_shared_module_split(model: dict, manifest: dict, overwrite: bool): package_root = manifest["package_root"] compiled_module = manifest["compiled_module"] diagonal_shorthand = manifest.get("diagonal_shorthand", False) + cxx_to_pyname = _build_cxx_index(model) # Index every wrapped entity across the model by name so a manifest entry can # be matched to a class (with its instantiations), an enum or a free function. @@ -243,7 +278,12 @@ def generate_shared_module_split(model: dict, manifest: dict, overwrite: bool): templated = [c for c in classes if c["templated"]] compiled_import = _explicit_import(package, compiled_module, import_names) content = render_generated_module( - package, compiled_import, classes, templated, diagonal_shorthand + package, + compiled_import, + classes, + templated, + diagonal_shorthand, + cxx_to_pyname, ) path = os.path.join(package_root, subpkg, "_generated.py") _write_generated(path, content, overwrite) From b721f315e1ffd23488b8d6ad16414be9ac47cff1 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 11:41:13 +0100 Subject: [PATCH 07/24] #102 Make the template-argument name separator configurable The "_" joining a templated class's base name to its template arguments (and successive/nested args) was hardcoded in class_info.update_py_names. Hoist it into a single CPPWG_TEMPLATE_ARG_SEPARATOR constant so a project can widen it (e.g. "__") to avoid Python-name clashes. Defaults to "_", so generated names are unchanged. Co-Authored-By: Claude Opus 4.8 --- cppwg/info/class_info.py | 27 +++++++++++++++------------ cppwg/utils/constants.py | 7 +++++++ 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/cppwg/info/class_info.py b/cppwg/info/class_info.py index a00dbbe..9c6ce40 100644 --- a/cppwg/info/class_info.py +++ b/cppwg/info/class_info.py @@ -9,7 +9,7 @@ from cppwg.info.cpp_entity_info import CppEntityInfo from cppwg.utils import utils -from cppwg.utils.constants import CPPWG_EXT +from cppwg.utils.constants import CPPWG_EXT, CPPWG_TEMPLATE_ARG_SEPARATOR if TYPE_CHECKING: from pygccxml.declarations import declaration_t @@ -589,10 +589,11 @@ def update_py_names(self) -> None: Set the Python names for the class, accounting for template args. Set the name(s) of the class as it should appear in Python. This - collapses template arguments, separates them by underscores, and removes - special characters. There can be multiple names, one for each template - class instantiation. For example, class "Foo" with template arguments - [[2, 2], [3, 3]] will have a Python name list ["Foo_2_2", "Foo_3_3"]. + collapses template arguments, separates them by CPPWG_TEMPLATE_ARG_SEPARATOR + (default "_"), and removes special characters. There can be multiple names, + one for each template class instantiation. For example, class "Foo" with + template arguments [[2, 2], [3, 3]] will have a Python name list + ["Foo_2_2", "Foo_3_3"]. """ class_name = self.py_name_base() @@ -601,22 +602,24 @@ class instantiation. For example, class "Foo" with template arguments self.py_names.append(class_name) return - # Create a string of template args separated by "_" e.g. 2_2 + separator = CPPWG_TEMPLATE_ARG_SEPARATOR + + # Create a string of template args separated by `separator`, e.g. 2_2 for template_arg_list in self.template_arg_lists: # Example template_arg_list : [2, 2] template_string = "" for idx, arg in enumerate(template_arg_list): - # A nested template arg keeps its structure via "_" separators, - # e.g. PottsMesh<2> -> PottsMesh_2 (separator="_"). - arg_str = self._mangle_py_token(str(arg), separator="_") + # A nested template arg keeps its structure via `separator`, + # e.g. PottsMesh<2> -> PottsMesh_2. + arg_str = self._mangle_py_token(str(arg), separator=separator) - # Add "_" between template arguments + # Add the separator between template arguments template_string += arg_str if idx < len(template_arg_list) - 1: - template_string += "_" + template_string += separator - self.py_names.append(class_name + "_" + template_string) + self.py_names.append(class_name + separator + template_string) def update_cpp_names(self) -> None: """ diff --git a/cppwg/utils/constants.py b/cppwg/utils/constants.py index 709761b..043d318 100644 --- a/cppwg/utils/constants.py +++ b/cppwg/utils/constants.py @@ -20,3 +20,10 @@ CPPWG_DEFAULT_WRAPPER_DIR = "cppwg_wrappers" CPPWG_CLASS_OVERRIDE_SUFFIX = "_Overrides" + +# Separator inserted between a templated class's base name and its template +# arguments, between successive arguments, and within a nested template argument +# when building the class's Python name (e.g. Foo<2, 2> -> "Foo_2_2", +# MeshFactory> -> "MeshFactory_PottsMesh_2"). A project can widen +# this (e.g. "__") to avoid Python-name clashes with similarly-named types. +CPPWG_TEMPLATE_ARG_SEPARATOR = "_" From 9ebac3a0c1c592260f92d192b4a0d1b598532798 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 11:41:28 +0100 Subject: [PATCH 08/24] #102 Tidy stale docstrings and comments The package-model docstring still called the model "JSON-serialisable" even though cppwg now writes it as cppwg_model.yaml; "serialisable (plain-dict)" describes the shape without implying a format. Also reword the BASE_INFO_OPTIONS comment about options becoming unreachable from the YAML. Co-Authored-By: Claude Opus 4.8 --- cppwg/info/base_info.py | 10 +++++----- cppwg/utils/python_model.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cppwg/info/base_info.py b/cppwg/info/base_info.py index dcc26fa..8321b51 100644 --- a/cppwg/info/base_info.py +++ b/cppwg/info/base_info.py @@ -19,11 +19,11 @@ # defaults and copies any the config overrides, and the parser # (cppwg.parsers.package_info_parser) builds its config dicts from the same # schema. An option added here is therefore understood everywhere - defined in -# one place instead of being restated in BaseInfo and the parser (which is how -# options such as name_replacements previously became unreachable from the YAML). -# Mutable defaults are deep-copied per use so no two objects share a list/dict. -# See the class Attributes docstring for what each option means; tri-state -# options default to None, meaning "inherit from further up the info tree". +# one place instead of being restated in BaseInfo and the parser (which can +# cause options to became unreachable from the YAML). Mutable defaults are +# deep-copied per use so no two objects share a list/dict. See the class +# Attributes docstring for what each option means; tri-state options default to +# None, meaning "inherit from further up the info tree". BASE_INFO_OPTIONS: dict[str, Any] = { "arg_type_excludes": [], "auto_includes": None, diff --git a/cppwg/utils/python_model.py b/cppwg/utils/python_model.py index 9dfbf15..1e37a19 100644 --- a/cppwg/utils/python_model.py +++ b/cppwg/utils/python_model.py @@ -1,4 +1,4 @@ -"""Build a JSON-serialisable model of the Python package layer. +"""Build a serialisable (plain-dict) model of the Python package layer. cppwg generates the C++/pybind11 wrappers; a separate step (see ``tools/cppwg_initgen.py``) generates the Python package layer - the @@ -28,7 +28,7 @@ def compiled_module_name(package_name: str, module_name: str) -> str: def build_python_model(package_info: "PackageInfo") -> dict[str, Any]: """ - Distil a PackageInfo tree into a JSON-serialisable Python-package model. + Distil a PackageInfo tree into a serialisable (plain-dict) Python-package model. Parameters ---------- From 2d40f0d8bffc77c32146a29406248f02a47605dd Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 12:08:15 +0100 Subject: [PATCH 09/24] #102 Rename package-layer artifacts for clarity Unify the package-layer generation vocabulary and fix a misnomer: - python_model -> package_model: the module (cppwg/utils/package_model.py), build_python_model/write_python_model -> build_package_model/write_package_model, CPPWG_PYTHON_MODEL_FILENAME -> CPPWG_PACKAGE_MODEL_FILENAME, and the emitted file cppwg_model.yaml -> cppwg_package_model.yaml. "python" said nothing; "package" names what is modelled. - py_layout.yaml -> package_layout.yaml, pairing with the existing package_info.yaml config; the initgen CLI flag --manifest -> --layout and the `manifest` variable -> `layout`. - cppwg_initgen.py -> cppwg_genpackage.py: the script generates _generated.py and never touches the hand-written __init__.py, so "initgen" was misleading. Behaviour is unchanged; re-running the generator on the shapes/cells examples reproduces identical output (only the "Generated by ..." header names the new script). Co-Authored-By: Claude Opus 4.8 --- cppwg/generators.py | 20 +++--- cppwg/utils/constants.py | 8 +-- .../{python_model.py => package_model.py} | 8 +-- .../{py_layout.yaml => package_layout.yaml} | 4 +- ...wg_model.yaml => cppwg_package_model.yaml} | 0 examples/cells/src/py/pycells/__init__.py | 2 +- examples/cells/src/py/pycells/_generated.py | 2 +- .../src/py/pyshapes/composites/__init__.py | 2 +- .../src/py/pyshapes/composites/_generated.py | 2 +- .../src/py/pyshapes/geometry/__init__.py | 2 +- .../src/py/pyshapes/geometry/_generated.py | 2 +- .../src/py/pyshapes/math_funcs/__init__.py | 2 +- .../src/py/pyshapes/math_funcs/_generated.py | 2 +- .../src/py/pyshapes/primitives/__init__.py | 2 +- .../src/py/pyshapes/primitives/_generated.py | 2 +- ...wg_model.yaml => cppwg_package_model.yaml} | 0 .../{py_layout.yaml => package_layout.yaml} | 4 +- tests/{test_initgen.py => test_genpackage.py} | 58 +++++++-------- ..._python_model.py => test_package_model.py} | 12 ++-- .../{cppwg_initgen.py => cppwg_genpackage.py} | 70 +++++++++---------- 20 files changed, 102 insertions(+), 102 deletions(-) rename cppwg/utils/{python_model.py => package_model.py} (92%) rename examples/cells/dynamic/{py_layout.yaml => package_layout.yaml} (65%) rename examples/cells/dynamic/wrappers/{cppwg_model.yaml => cppwg_package_model.yaml} (100%) rename examples/shapes/wrapper/{cppwg_model.yaml => cppwg_package_model.yaml} (100%) rename examples/shapes/wrapper/{py_layout.yaml => package_layout.yaml} (65%) rename tests/{test_initgen.py => test_genpackage.py} (86%) rename tests/{test_python_model.py => test_package_model.py} (92%) rename tools/{cppwg_initgen.py => cppwg_genpackage.py} (85%) diff --git a/cppwg/generators.py b/cppwg/generators.py index 180974d..7fd8500 100644 --- a/cppwg/generators.py +++ b/cppwg/generators.py @@ -19,9 +19,9 @@ from cppwg.utils.constants import ( CPPWG_DEFAULT_WRAPPER_DIR, CPPWG_HEADER_COLLECTION_FILENAME, - CPPWG_PYTHON_MODEL_FILENAME, + CPPWG_PACKAGE_MODEL_FILENAME, ) -from cppwg.utils.python_model import build_python_model +from cppwg.utils.package_model import build_package_model from cppwg.version import __version__ as cppwg_version from cppwg.writers.header_collection_writer import CppHeaderCollectionWriter from cppwg.writers.package_writer import CppPackageWrapperWriter @@ -425,17 +425,17 @@ def write_wrappers(self) -> None: ) package_writer.write() - def write_python_model(self) -> None: + def write_package_model(self) -> None: """ - Write the Python-package model (cppwg_model.yaml) to the wrapper root. + Write the package model (cppwg_package_model.yaml) to the wrapper root. A small YAML description of the generated modules and their classes / instantiations / enums / free functions, so a separate step - (tools/cppwg_initgen.py) can generate the Python package layer without + (tools/cppwg_genpackage.py) can generate the Python package layer without re-parsing the source. Written last, once the info tree is final. """ - model = build_python_model(self.package_info) - model_path = os.path.join(self.wrapper_root, CPPWG_PYTHON_MODEL_FILENAME) + model = build_package_model(self.package_info) + model_path = os.path.join(self.wrapper_root, CPPWG_PACKAGE_MODEL_FILENAME) content = yaml.safe_dump(model, default_flow_style=False, sort_keys=True) utils.write_file_if_changed(model_path, content, self.overwrite) @@ -500,6 +500,6 @@ def generate(self) -> None: # Write the wrapper code for the package self.write_wrappers() - # Write the Python-package model (cppwg_model.yaml) for the package-layer - # generator (tools/cppwg_initgen.py). - self.write_python_model() + # Write the package model (cppwg_package_model.yaml) for the package-layer + # generator (tools/cppwg_genpackage.py). + self.write_package_model() diff --git a/cppwg/utils/constants.py b/cppwg/utils/constants.py index 043d318..a122974 100644 --- a/cppwg/utils/constants.py +++ b/cppwg/utils/constants.py @@ -9,10 +9,10 @@ # Default log file name used when --logfile is passed without a value. CPPWG_DEFAULT_LOGFILE = f"{CPPWG_EXT}.log" -# The Python-package model cppwg writes into the wrapper root, describing the -# generated modules/classes so a separate step can build the Python package -# layer (see cppwg.utils.python_model and tools/cppwg_initgen.py). -CPPWG_PYTHON_MODEL_FILENAME = "cppwg_model.yaml" +# The package model cppwg writes into the wrapper root, describing the generated +# modules/classes so a separate step can build the Python package layer (see +# cppwg.utils.package_model and tools/cppwg_genpackage.py). +CPPWG_PACKAGE_MODEL_FILENAME = "cppwg_package_model.yaml" CPPWG_TRUE_STRINGS = ["ON", "YES", "Y", "TRUE", "T", "1"] CPPWG_FALSE_STRINGS = ["OFF", "NO", "N", "FALSE", "F", "0", ""] diff --git a/cppwg/utils/python_model.py b/cppwg/utils/package_model.py similarity index 92% rename from cppwg/utils/python_model.py rename to cppwg/utils/package_model.py index 1e37a19..e54bb86 100644 --- a/cppwg/utils/python_model.py +++ b/cppwg/utils/package_model.py @@ -1,13 +1,13 @@ """Build a serialisable (plain-dict) model of the Python package layer. cppwg generates the C++/pybind11 wrappers; a separate step (see -``tools/cppwg_initgen.py``) generates the Python package layer - the +``tools/cppwg_genpackage.py``) generates the Python package layer - the ``_generated.py`` files that import each compiled extension and define the ``TemplateClass`` subscript stubs. That step needs, per module, the compiled module name, the wrapped classes with their template instantiations, and the enum / free-function names. All of this is on the finalized ``PackageInfo`` tree -but not in a form a standalone script can consume, so ``build_python_model`` -distils it into a plain dict that cppwg writes out as ``cppwg_model.yaml``. +but not in a form a standalone script can consume, so ``build_package_model`` +distils it into a plain dict that cppwg writes out as ``cppwg_package_model.yaml``. """ from typing import TYPE_CHECKING, Any @@ -26,7 +26,7 @@ def compiled_module_name(package_name: str, module_name: str) -> str: return f"_{package_name}_{module_name}" -def build_python_model(package_info: "PackageInfo") -> dict[str, Any]: +def build_package_model(package_info: "PackageInfo") -> dict[str, Any]: """ Distil a PackageInfo tree into a serialisable (plain-dict) Python-package model. diff --git a/examples/cells/dynamic/py_layout.yaml b/examples/cells/dynamic/package_layout.yaml similarity index 65% rename from examples/cells/dynamic/py_layout.yaml rename to examples/cells/dynamic/package_layout.yaml index 8451dd1..7382e49 100644 --- a/examples/cells/dynamic/py_layout.yaml +++ b/examples/cells/dynamic/package_layout.yaml @@ -1,10 +1,10 @@ -# Python-package layout for tools/cppwg_initgen.py (cells example). +# Python-package layout for tools/cppwg_genpackage.py (cells example). # # A single cppwg module `all` -> the flat pycells package (its compiled # extension _pycells_all sits at the package root, not in an `all/` subdir), so # module_dirs maps `all` to ".". # -# package_root is resolved relative to this manifest's directory. +# package_root is resolved relative to this layout file's directory. package: pycells package_root: ../src/py/pycells module_dirs: diff --git a/examples/cells/dynamic/wrappers/cppwg_model.yaml b/examples/cells/dynamic/wrappers/cppwg_package_model.yaml similarity index 100% rename from examples/cells/dynamic/wrappers/cppwg_model.yaml rename to examples/cells/dynamic/wrappers/cppwg_package_model.yaml diff --git a/examples/cells/src/py/pycells/__init__.py b/examples/cells/src/py/pycells/__init__.py index e456c54..7249079 100644 --- a/examples/cells/src/py/pycells/__init__.py +++ b/examples/cells/src/py/pycells/__init__.py @@ -1,7 +1,7 @@ """Main pycells module. The compiled-extension imports and TemplateClass subscript stubs are generated -by tools/cppwg_initgen.py into _generated.py; add any hand-written code here. +by tools/cppwg_genpackage.py into _generated.py; add any hand-written code here. Curation rationale (why Facet<1>/Corner<1>/AbstractMesh are or aren't wrapped) lives with the C++ sources and examples/cells/dynamic/config.yaml. """ diff --git a/examples/cells/src/py/pycells/_generated.py b/examples/cells/src/py/pycells/_generated.py index dd294c0..ff9cd90 100644 --- a/examples/cells/src/py/pycells/_generated.py +++ b/examples/cells/src/py/pycells/_generated.py @@ -1,4 +1,4 @@ -"""Generated by tools/cppwg_initgen.py - do not edit. +"""Generated by tools/cppwg_genpackage.py - do not edit. Compiled-extension imports and TemplateClass subscript stubs for this subpackage. Hand-written code (TemplateMethod attachments, package init, diff --git a/examples/shapes/src/py/pyshapes/composites/__init__.py b/examples/shapes/src/py/pyshapes/composites/__init__.py index fe9c6c5..7397b2f 100644 --- a/examples/shapes/src/py/pyshapes/composites/__init__.py +++ b/examples/shapes/src/py/pyshapes/composites/__init__.py @@ -1,3 +1,3 @@ # The compiled-extension imports and TemplateClass stubs are generated by -# tools/cppwg_initgen.py into _generated.py; add any hand-written code here. +# tools/cppwg_genpackage.py into _generated.py; add any hand-written code here. from ._generated import * # noqa: F401,F403 diff --git a/examples/shapes/src/py/pyshapes/composites/_generated.py b/examples/shapes/src/py/pyshapes/composites/_generated.py index 6f4d9e2..669681d 100644 --- a/examples/shapes/src/py/pyshapes/composites/_generated.py +++ b/examples/shapes/src/py/pyshapes/composites/_generated.py @@ -1,4 +1,4 @@ -"""Generated by tools/cppwg_initgen.py - do not edit. +"""Generated by tools/cppwg_genpackage.py - do not edit. Compiled-extension imports and TemplateClass subscript stubs for this subpackage. Hand-written code (TemplateMethod attachments, package init, diff --git a/examples/shapes/src/py/pyshapes/geometry/__init__.py b/examples/shapes/src/py/pyshapes/geometry/__init__.py index fe9c6c5..7397b2f 100644 --- a/examples/shapes/src/py/pyshapes/geometry/__init__.py +++ b/examples/shapes/src/py/pyshapes/geometry/__init__.py @@ -1,3 +1,3 @@ # The compiled-extension imports and TemplateClass stubs are generated by -# tools/cppwg_initgen.py into _generated.py; add any hand-written code here. +# tools/cppwg_genpackage.py into _generated.py; add any hand-written code here. from ._generated import * # noqa: F401,F403 diff --git a/examples/shapes/src/py/pyshapes/geometry/_generated.py b/examples/shapes/src/py/pyshapes/geometry/_generated.py index f836db4..5c6a3ab 100644 --- a/examples/shapes/src/py/pyshapes/geometry/_generated.py +++ b/examples/shapes/src/py/pyshapes/geometry/_generated.py @@ -1,4 +1,4 @@ -"""Generated by tools/cppwg_initgen.py - do not edit. +"""Generated by tools/cppwg_genpackage.py - do not edit. Compiled-extension imports and TemplateClass subscript stubs for this subpackage. Hand-written code (TemplateMethod attachments, package init, diff --git a/examples/shapes/src/py/pyshapes/math_funcs/__init__.py b/examples/shapes/src/py/pyshapes/math_funcs/__init__.py index fe9c6c5..7397b2f 100644 --- a/examples/shapes/src/py/pyshapes/math_funcs/__init__.py +++ b/examples/shapes/src/py/pyshapes/math_funcs/__init__.py @@ -1,3 +1,3 @@ # The compiled-extension imports and TemplateClass stubs are generated by -# tools/cppwg_initgen.py into _generated.py; add any hand-written code here. +# tools/cppwg_genpackage.py into _generated.py; add any hand-written code here. from ._generated import * # noqa: F401,F403 diff --git a/examples/shapes/src/py/pyshapes/math_funcs/_generated.py b/examples/shapes/src/py/pyshapes/math_funcs/_generated.py index 46924ad..6530fc3 100644 --- a/examples/shapes/src/py/pyshapes/math_funcs/_generated.py +++ b/examples/shapes/src/py/pyshapes/math_funcs/_generated.py @@ -1,4 +1,4 @@ -"""Generated by tools/cppwg_initgen.py - do not edit. +"""Generated by tools/cppwg_genpackage.py - do not edit. Compiled-extension imports and TemplateClass subscript stubs for this subpackage. Hand-written code (TemplateMethod attachments, package init, diff --git a/examples/shapes/src/py/pyshapes/primitives/__init__.py b/examples/shapes/src/py/pyshapes/primitives/__init__.py index 1ebac1a..2a8dc92 100644 --- a/examples/shapes/src/py/pyshapes/primitives/__init__.py +++ b/examples/shapes/src/py/pyshapes/primitives/__init__.py @@ -1,5 +1,5 @@ # The compiled-extension imports and TemplateClass stubs are generated by -# tools/cppwg_initgen.py into _generated.py; add any hand-written code here. +# tools/cppwg_genpackage.py into _generated.py; add any hand-written code here. from ._generated import * # noqa: F401,F403 from pyshapes._syntax import TemplateMethod diff --git a/examples/shapes/src/py/pyshapes/primitives/_generated.py b/examples/shapes/src/py/pyshapes/primitives/_generated.py index db972a0..0b74e9b 100644 --- a/examples/shapes/src/py/pyshapes/primitives/_generated.py +++ b/examples/shapes/src/py/pyshapes/primitives/_generated.py @@ -1,4 +1,4 @@ -"""Generated by tools/cppwg_initgen.py - do not edit. +"""Generated by tools/cppwg_genpackage.py - do not edit. Compiled-extension imports and TemplateClass subscript stubs for this subpackage. Hand-written code (TemplateMethod attachments, package init, diff --git a/examples/shapes/wrapper/cppwg_model.yaml b/examples/shapes/wrapper/cppwg_package_model.yaml similarity index 100% rename from examples/shapes/wrapper/cppwg_model.yaml rename to examples/shapes/wrapper/cppwg_package_model.yaml diff --git a/examples/shapes/wrapper/py_layout.yaml b/examples/shapes/wrapper/package_layout.yaml similarity index 65% rename from examples/shapes/wrapper/py_layout.yaml rename to examples/shapes/wrapper/package_layout.yaml index 83c2a7d..560a3b1 100644 --- a/examples/shapes/wrapper/py_layout.yaml +++ b/examples/shapes/wrapper/package_layout.yaml @@ -1,9 +1,9 @@ -# Python-package layout for tools/cppwg_initgen.py (shapes example). +# Python-package layout for tools/cppwg_genpackage.py (shapes example). # # Module-per-subpackage: each cppwg module (geometry/primitives/composites/ # math_funcs) is a subpackage that owns its own compiled extension, so each # _generated.py does `from .<_pyshapes_module> import *`. # -# package_root is resolved relative to this manifest's directory. +# package_root is resolved relative to this layout file's directory. package: pyshapes package_root: ../src/py/pyshapes diff --git a/tests/test_initgen.py b/tests/test_genpackage.py similarity index 86% rename from tests/test_initgen.py rename to tests/test_genpackage.py index 40fd714..05866fb 100644 --- a/tests/test_initgen.py +++ b/tests/test_genpackage.py @@ -1,14 +1,14 @@ -"""Unit tests for tools/cppwg_initgen.py.""" +"""Unit tests for tools/cppwg_genpackage.py.""" import importlib.util import os -_INITGEN_PATH = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "tools", "cppwg_initgen.py" +_GENPACKAGE_PATH = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "tools", "cppwg_genpackage.py" ) -_spec = importlib.util.spec_from_file_location("cppwg_initgen", _INITGEN_PATH) -initgen = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(initgen) +_spec = importlib.util.spec_from_file_location("cppwg_genpackage", _GENPACKAGE_PATH) +genpackage = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(genpackage) def _class(base, instantiations, templated=True): @@ -20,13 +20,13 @@ def _inst(args, py_name): def test_key_repr_singleton_and_multi(): - assert initgen._key_repr(["2"]) == '("2",)' - assert initgen._key_repr(["2", "2"]) == '("2", "2")' - assert initgen._key_repr(["Cell", "2"]) == '("Cell", "2")' + assert genpackage._key_repr(["2"]) == '("2",)' + assert genpackage._key_repr(["2", "2"]) == '("2", "2")' + assert genpackage._key_repr(["Cell", "2"]) == '("Cell", "2")' def test_stub_source(): - stub = initgen._stub_source( + stub = genpackage._stub_source( "Point", [_inst(["2"], "Point_2"), _inst(["3"], "Point_3")] ) assert stub == ( @@ -40,7 +40,7 @@ def test_stub_source(): def test_stub_source_diagonal_shorthand(): """A multi-arg diagonal instantiation gains a single-arg alias when opted in.""" - stub = initgen._stub_source( + stub = genpackage._stub_source( "Element", [_inst(["2", "2"], "Element_2_2"), _inst(["1", "2"], "Element_1_2")], diagonal_shorthand=True, @@ -51,7 +51,7 @@ def test_stub_source_diagonal_shorthand(): assert '("1", "2"): Element_1_2,' in stub assert '("1",):' not in stub # Off by default: no aliases. - plain = initgen._stub_source("Element", [_inst(["2", "2"], "Element_2_2")]) + plain = genpackage._stub_source("Element", [_inst(["2", "2"], "Element_2_2")]) assert '("2",):' not in plain @@ -60,7 +60,7 @@ def test_stub_source_nested_templated_arg(): # MeshFactory> -> key ("PottsMesh_2",), not ("PottsMesh<2>",), # so MeshFactory[PottsMesh[2]] resolves (PottsMesh[2] is the PottsMesh_2 class). cxx_to_pyname = {"PottsMesh<2>": "PottsMesh_2", "PottsMesh<3>": "PottsMesh_3"} - stub = initgen._stub_source( + stub = genpackage._stub_source( "MeshFactory", [ _inst(["PottsMesh<2>"], "MeshFactory_PottsMesh_2"), @@ -84,16 +84,16 @@ def test_build_cxx_index(): } ] } - index = initgen._build_cxx_index(model) + index = genpackage._build_cxx_index(model) assert index == {"PottsMesh<2>": "PottsMesh_2"} # untemplated Cell has no <...> def test_render_generated_module_with_stub_imports_syntax(): point = _class("Point", [_inst(["2"], "Point_2"), _inst(["3"], "Point_3")]) - content = initgen.render_generated_module( + content = genpackage.render_generated_module( "pyshapes", "from ._pyshapes_geometry import *", [point], [point] ) - assert content.startswith('"""Generated by tools/cppwg_initgen.py') + assert content.startswith('"""Generated by tools/cppwg_genpackage.py') assert "from ._pyshapes_geometry import *" in content assert "from pyshapes._syntax import TemplateClass" in content assert "class Point(TemplateClass):" in content @@ -102,7 +102,7 @@ def test_render_generated_module_with_stub_imports_syntax(): def test_render_generated_module_no_templates_omits_syntax(): plain = _class("Square", [_inst([], "Square")], templated=False) - content = initgen.render_generated_module( + content = genpackage.render_generated_module( "pyshapes", "from ._pyshapes_composites import *", [plain], [] ) assert "from ._pyshapes_composites import *" in content @@ -133,9 +133,9 @@ def test_generate_module_per_subpackage(tmp_path): }, ], } - manifest = {"package": "pyshapes", "package_root": str(tmp_path)} + layout = {"package": "pyshapes", "package_root": str(tmp_path)} - initgen.generate_module_per_subpackage(model, manifest, overwrite=False) + genpackage.generate_module_per_subpackage(model, layout, overwrite=False) geometry = (tmp_path / "geometry" / "_generated.py").read_text() assert "from ._pyshapes_geometry import *" in geometry @@ -161,9 +161,9 @@ def test_module_dirs_override_places_single_module_at_root(tmp_path): } ], } - manifest = {"package_root": str(tmp_path), "module_dirs": {"all": "."}} + layout = {"package_root": str(tmp_path), "module_dirs": {"all": "."}} - initgen.generate_module_per_subpackage(model, manifest, overwrite=False) + genpackage.generate_module_per_subpackage(model, layout, overwrite=False) generated = (tmp_path / "_generated.py").read_text() # at the root, not all/ assert "from ._pycells_all import *" in generated @@ -189,7 +189,7 @@ def test_generate_shared_module_split(tmp_path, capsys): } ], } - manifest = { + layout = { "package": "chaste", "package_root": str(tmp_path), "compiled_module": "_pychaste_all", @@ -199,7 +199,7 @@ def test_generate_shared_module_split(tmp_path, capsys): }, } - initgen.generate_shared_module_split(model, manifest, overwrite=False) + genpackage.generate_shared_module_split(model, layout, overwrite=False) core = (tmp_path / "core" / "_generated.py").read_text() assert "from chaste._pychaste_all import (" in core @@ -233,7 +233,7 @@ def test_flatten_to_root(tmp_path): } ], } - manifest = { + layout = { "package": "chaste", "package_root": str(tmp_path), "compiled_module": "_pychaste_all", @@ -241,7 +241,7 @@ def test_flatten_to_root(tmp_path): "subpackages": {"core": ["FileFinder", "RelativeTo"], "mesh": ["Node"]}, } - initgen.generate_shared_module_split(model, manifest, overwrite=False) + genpackage.generate_shared_module_split(model, layout, overwrite=False) root = (tmp_path / "_generated.py").read_text() # Re-exports the base/enum names (not the concrete Node_2) from each subpackage. @@ -268,7 +268,7 @@ def test_flatten_to_root_warns_on_name_clash(tmp_path, capsys): } ], } - manifest = { + layout = { "package": "pkg", "package_root": str(tmp_path), "compiled_module": "_pkg_all", @@ -276,7 +276,7 @@ def test_flatten_to_root_warns_on_name_clash(tmp_path, capsys): "subpackages": {"a": ["Dup"], "b": ["Dup"]}, } - initgen.generate_shared_module_split(model, manifest, overwrite=False) + genpackage.generate_shared_module_split(model, layout, overwrite=False) assert "'Dup'" in capsys.readouterr().err # ambiguous top-level export @@ -297,14 +297,14 @@ def test_shared_split_warns_on_unknown_and_unassigned(tmp_path, capsys): } ], } - manifest = { + layout = { "package": "pkg", "package_root": str(tmp_path), "compiled_module": "_pkg_all", "subpackages": {"sub": ["Kept", "DoesNotExist"]}, } - initgen.generate_shared_module_split(model, manifest, overwrite=False) + genpackage.generate_shared_module_split(model, layout, overwrite=False) err = capsys.readouterr().err assert "'DoesNotExist'" in err # listed but not in model diff --git a/tests/test_python_model.py b/tests/test_package_model.py similarity index 92% rename from tests/test_python_model.py rename to tests/test_package_model.py index e35e553..ce702a1 100644 --- a/tests/test_python_model.py +++ b/tests/test_package_model.py @@ -1,8 +1,8 @@ -"""Unit tests for cppwg.utils.python_model.""" +"""Unit tests for cppwg.utils.package_model.""" from types import SimpleNamespace -from cppwg.utils.python_model import build_python_model, compiled_module_name +from cppwg.utils.package_model import build_package_model, compiled_module_name def _class(base, py_names, template_arg_lists=(), excluded=False): @@ -61,7 +61,7 @@ def test_build_model_templated_and_untemplated(): ], ) - model = build_python_model(package) + model = build_package_model(package) assert model["package"] == "pyshapes" geometry, primitives = model["modules"] @@ -108,7 +108,7 @@ def test_build_model_omits_excluded_and_pruned(): ], ) - (module,) = build_python_model(package)["modules"] + (module,) = build_package_model(package)["modules"] assert [c["base"] for c in module["classes"]] == ["Kept"] assert module["enums"] == ["KeptEnum"] @@ -130,7 +130,7 @@ def test_build_model_multi_arg_and_class_arg_keys(): ], ) - (module,) = build_python_model(package)["modules"] + (module,) = build_package_model(package)["modules"] macro, factory = module["classes"] assert macro["instantiations"] == [{"args": ["2", "2"], "py_name": "MacroMesh_2_2"}] assert factory["instantiations"] == [ @@ -142,5 +142,5 @@ def test_enum_name_override_used(): package = _package( "pkg", [_module("mod", enums=[_enum("RawName", name_override="PyName")])] ) - (module,) = build_python_model(package)["modules"] + (module,) = build_package_model(package)["modules"] assert module["enums"] == ["PyName"] diff --git a/tools/cppwg_initgen.py b/tools/cppwg_genpackage.py similarity index 85% rename from tools/cppwg_initgen.py rename to tools/cppwg_genpackage.py index 2b2057b..b881462 100644 --- a/tools/cppwg_initgen.py +++ b/tools/cppwg_genpackage.py @@ -2,9 +2,9 @@ """Generate the Python package layer (``_generated.py``) for a cppwg project. cppwg generates the C++/pybind11 wrappers and, alongside them, a YAML model of -what it produced (``cppwg_model.yaml`` in the wrapper root - see -``cppwg.utils.python_model``). This standalone script turns that model, plus a -small layout manifest, into a ``_generated.py`` per Python subpackage: the +what it produced (``cppwg_package_model.yaml`` in the wrapper root - see +``cppwg.utils.package_model``). This standalone script turns that model, plus a +small package-layout file, into a ``_generated.py`` per Python subpackage: the compiled-extension import and the ``TemplateClass`` subscript stubs (``Point[2] -> Point_2``). It never touches the hand-written ``__init__.py``, which does ``from ._generated import *`` and adds the bespoke pieces @@ -16,13 +16,13 @@ owns its own compiled extension, imported with ``from . import *``. Used by the shapes/cells examples. * shared-module split: one compiled extension (e.g. ``_pychaste_all``) is split - into several subpackages by a manifest that lists which names each owns; + into several subpackages by a layout that lists which names each owns; imported explicitly with ``from . import (...)``. Used by pychaste. Usage:: - cppwg_initgen.py --model wrapper/cppwg_model.yaml --manifest py_layout.yaml + cppwg_genpackage.py --model wrapper/cppwg_package_model.yaml --layout package_layout.yaml """ import argparse @@ -48,7 +48,7 @@ def write_file_if_changed(filepath, content, overwrite=False): GENERATED_HEADER = ( - '"""Generated by tools/cppwg_initgen.py - do not edit.\n' + '"""Generated by tools/cppwg_genpackage.py - do not edit.\n' "\n" "Compiled-extension imports and TemplateClass subscript stubs for this\n" "subpackage. Hand-written code (TemplateMethod attachments, package init,\n" @@ -59,7 +59,7 @@ def write_file_if_changed(filepath, content, overwrite=False): FLATTEN_HEADER = ( - '"""Generated by tools/cppwg_initgen.py - do not edit.\n' + '"""Generated by tools/cppwg_genpackage.py - do not edit.\n' "\n" "Re-exports every wrapped class, enum and free function from the subpackages\n" "into the top-level package namespace, so `package.ClassName` works as well\n" @@ -212,12 +212,12 @@ def _write_generated(path: str, content: str, overwrite: bool) -> None: print(f"unchanged {path}") -def generate_module_per_subpackage(model: dict, manifest: dict, overwrite: bool): +def generate_module_per_subpackage(model: dict, layout: dict, overwrite: bool): """Emit one ``_generated.py`` per cppwg module (each owns its own extension).""" - package = manifest.get("package", model["package"]) - package_root = manifest["package_root"] - module_dirs = manifest.get("module_dirs", {}) - diagonal_shorthand = manifest.get("diagonal_shorthand", False) + package = layout.get("package", model["package"]) + package_root = layout["package_root"] + module_dirs = layout.get("module_dirs", {}) + diagonal_shorthand = layout.get("diagonal_shorthand", False) cxx_to_pyname = _build_cxx_index(model) for module in model["modules"]: @@ -238,15 +238,15 @@ def generate_module_per_subpackage(model: dict, manifest: dict, overwrite: bool) _write_generated(path, content, overwrite) -def generate_shared_module_split(model: dict, manifest: dict, overwrite: bool): +def generate_shared_module_split(model: dict, layout: dict, overwrite: bool): """Emit a ``_generated.py`` per subpackage, splitting one shared extension.""" - package = manifest.get("package", model["package"]) - package_root = manifest["package_root"] - compiled_module = manifest["compiled_module"] - diagonal_shorthand = manifest.get("diagonal_shorthand", False) + package = layout.get("package", model["package"]) + package_root = layout["package_root"] + compiled_module = layout["compiled_module"] + diagonal_shorthand = layout.get("diagonal_shorthand", False) cxx_to_pyname = _build_cxx_index(model) - # Index every wrapped entity across the model by name so a manifest entry can + # Index every wrapped entity across the model by name so a layout entry can # be matched to a class (with its instantiations), an enum or a free function. classes_by_base = {} other_names = set() @@ -258,7 +258,7 @@ def generate_shared_module_split(model: dict, manifest: dict, overwrite: bool): assigned = set() flatten_names = {} # subpackage -> the top-level names it exposes (issue #73) - for subpkg, names in manifest["subpackages"].items(): + for subpkg, names in layout["subpackages"].items(): classes = [] import_names = [] exported = [] @@ -297,7 +297,7 @@ def generate_shared_module_split(model: dict, manifest: dict, overwrite: bool): print(f"warning: '{name}' is wrapped but not assigned to a subpackage", file=sys.stderr) - if manifest.get("flatten_to_root"): + if layout.get("flatten_to_root"): generate_root_flatten(package, package_root, flatten_names, overwrite) @@ -337,13 +337,13 @@ def generate_root_flatten( def main(argv=None) -> int: - """Entry point: read the model + manifest and write the _generated.py files.""" + """Entry point: read the model + layout and write the _generated.py files.""" parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( - "--model", required=True, help="Path to cppwg_model.yaml (from cppwg)." + "--model", required=True, help="Path to cppwg_package_model.yaml (from cppwg)." ) parser.add_argument( - "--manifest", required=True, help="Path to the Python layout manifest (YAML)." + "--layout", required=True, help="Path to the Python package-layout file (YAML)." ) parser.add_argument( "--overwrite", @@ -356,21 +356,21 @@ def main(argv=None) -> int: with open(args.model) as model_file: model = yaml.safe_load(model_file) - with open(args.manifest) as manifest_file: - manifest = yaml.safe_load(manifest_file) - - # Resolve a relative package_root against the manifest's own directory, so a - # manifest is portable regardless of the caller's working directory. - if not os.path.isabs(manifest["package_root"]): - manifest_dir = os.path.dirname(os.path.abspath(args.manifest)) - manifest["package_root"] = os.path.normpath( - os.path.join(manifest_dir, manifest["package_root"]) + with open(args.layout) as layout_file: + layout = yaml.safe_load(layout_file) + + # Resolve a relative package_root against the layout's own directory, so a + # layout is portable regardless of the caller's working directory. + if not os.path.isabs(layout["package_root"]): + layout_dir = os.path.dirname(os.path.abspath(args.layout)) + layout["package_root"] = os.path.normpath( + os.path.join(layout_dir, layout["package_root"]) ) - if manifest.get("subpackages"): - generate_shared_module_split(model, manifest, args.overwrite) + if layout.get("subpackages"): + generate_shared_module_split(model, layout, args.overwrite) else: - generate_module_per_subpackage(model, manifest, args.overwrite) + generate_module_per_subpackage(model, layout, args.overwrite) return 0 From 7a55af0490787133dbd4f4e6f3888cefacbc6669 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 12:19:05 +0100 Subject: [PATCH 10/24] #102 Add a 'cppwg genpackage' subcommand for the package-layer generator Move the package-layer generator into the installed package as cppwg/genpackage.py and dispatch to it from cppwg.__main__ when the first argument is "genpackage". The subcommand is intercepted before argument parsing, so a normal `cppwg SOURCE_ROOT ...` run is entirely unaffected. Because tools/ is not part of the installed package, the generator was previously unavailable to pip-installed users; it now ships and runs wherever cppwg is installed (`cppwg genpackage --model ... --layout ...`). tools/cppwg_genpackage.py remains as a thin launcher for the old direct-path invocation. Now that the code always lives in-package, the standalone write_file_if_changed fallback (and its no-cover pragma) is dropped. Co-Authored-By: Claude Opus 4.8 --- cppwg/__main__.py | 15 +- cppwg/genpackage.py | 364 ++++++++++++++++++++++++++++++++++++ tests/test_genpackage.py | 36 +++- tools/cppwg_genpackage.py | 375 +------------------------------------- 4 files changed, 411 insertions(+), 379 deletions(-) create mode 100644 cppwg/genpackage.py diff --git a/cppwg/__main__.py b/cppwg/__main__.py index 0c7293c..b9daf05 100644 --- a/cppwg/__main__.py +++ b/cppwg/__main__.py @@ -2,6 +2,7 @@ import argparse import logging +import sys from datetime import datetime from pathlib import Path @@ -54,6 +55,7 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( prog="cppwg", description="Generate Python Wrappers for C++ code", + epilog="Run 'cppwg genpackage --help' for the Python package-layer generator.", ) parser.add_argument( @@ -193,7 +195,18 @@ def generate(args: argparse.Namespace) -> None: def main() -> None: - """Generate wrappers from command line arguments.""" + """Generate wrappers from command line arguments. + + ``cppwg genpackage ...`` is dispatched to the package-layer generator + (:mod:`cppwg.genpackage`); any other invocation runs wrapper generation as + usual. The subcommand is intercepted before argument parsing so the normal + CLI (a leading ``SOURCE_ROOT`` positional) is entirely unaffected. + """ + if len(sys.argv) > 1 and sys.argv[1] == "genpackage": + from cppwg.genpackage import main as genpackage_main + + raise SystemExit(genpackage_main(sys.argv[2:])) + args = parse_args() log_handlers = [] diff --git a/cppwg/genpackage.py b/cppwg/genpackage.py new file mode 100644 index 0000000..5e5867e --- /dev/null +++ b/cppwg/genpackage.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +"""Generate the Python package layer (``_generated.py``) for a cppwg project. + +cppwg generates the C++/pybind11 wrappers and, alongside them, a YAML model of +what it produced (``cppwg_package_model.yaml`` in the wrapper root - see +``cppwg.utils.package_model``). This module (run via ``cppwg genpackage``) turns +that model, plus a small package-layout file, into a ``_generated.py`` per +Python subpackage: the +compiled-extension import and the ``TemplateClass`` subscript stubs +(``Point[2] -> Point_2``). It never touches the hand-written ``__init__.py``, +which does ``from ._generated import *`` and adds the bespoke pieces +(``TemplateMethod`` attachments, package ``init()``, curation, comments). + +Two layouts: + +* module-per-subpackage (default): each cppwg module becomes a subpackage that + owns its own compiled extension, imported with ``from . import *``. + Used by the shapes/cells examples. +* shared-module split: one compiled extension (e.g. ``_pychaste_all``) is split + into several subpackages by a layout that lists which names each owns; + imported explicitly with ``from . import (...)``. Used by + pychaste. + +Usage:: + + cppwg genpackage --model wrapper/cppwg_package_model.yaml --layout package_layout.yaml +""" + +import argparse +import os +import sys + +from cppwg.utils.utils import write_file_if_changed + + +GENERATED_HEADER = ( + '"""Generated by tools/cppwg_genpackage.py - do not edit.\n' + "\n" + "Compiled-extension imports and TemplateClass subscript stubs for this\n" + "subpackage. Hand-written code (TemplateMethod attachments, package init,\n" + 'curation) belongs in the sibling __init__.py, which does `from ._generated\n' + 'import *`.\n' + '"""\n' +) + + +FLATTEN_HEADER = ( + '"""Generated by tools/cppwg_genpackage.py - do not edit.\n' + "\n" + "Re-exports every wrapped class, enum and free function from the subpackages\n" + "into the top-level package namespace, so `package.ClassName` works as well\n" + "as `package.subpackage.ClassName`. Importing this imports the subpackages.\n" + "The hand-written __init__.py does `from ._generated import *` and adds the\n" + "bespoke top-level pieces (package init, etc.).\n" + '"""\n' +) + + +def _key_repr(args: list[str]) -> str: + """Render a template-argument list as a Python tuple literal of strings. + + e.g. ``["2"]`` -> ``("2",)`` and ``["Cell", "2"]`` -> ``("Cell", "2")``. + Matches _syntax._normalize_key, which normalizes keys to string tuples. + """ + inner = ", ".join(f'"{arg}"' for arg in args) + if len(args) == 1: + inner += "," + return f"({inner})" + + +def _build_cxx_index(model: dict) -> dict: + """Map each wrapped instantiation's C++ type string to its Python class name. + + e.g. ``PottsMesh<2> -> PottsMesh_2``. Used to key a template argument that is + itself a wrapped templated type (``MeshFactory>``) by the Python + concrete class name, so ``MeshFactory[PottsMesh[2]]`` resolves: ``PottsMesh[2]`` + is the ``PottsMesh_2`` class and ``_normalize_key`` keys it by its ``__name__``. + """ + index = {} + for module in model["modules"]: + for class_info in module["classes"]: + for inst in class_info["instantiations"]: + if inst["args"]: + cxx = f'{class_info["base"]}<{",".join(inst["args"])}>' + index[cxx] = inst["py_name"] + return index + + +def _stub_source( + base: str, + instantiations: list[dict], + diagonal_shorthand: bool = False, + cxx_to_pyname: dict = None, +) -> str: + """Render a ``class (TemplateClass)`` stub for a templated class. + + A template argument that is itself a wrapped templated type is keyed by its + Python concrete class name via ``cxx_to_pyname`` (``PottsMesh<2>`` -> + ``PottsMesh_2``), so ``MeshFactory[PottsMesh[2]]`` resolves. + + When ``diagonal_shorthand`` is set, a multi-argument instantiation whose + arguments are all equal (a "diagonal", e.g. ``Element<2, 2>``) also gets a + single-argument alias key (``Element[2] -> Element_2_2``). This is an + opt-in Python-layer convenience some packages (pychaste) adopt for their + ````-style classes; others (the cells example) keep + the explicit multi-argument form only. + """ + cxx_to_pyname = cxx_to_pyname or {} + lines = [f"class {base}(TemplateClass):", " _instantiations = {"] + for inst in instantiations: + args = [cxx_to_pyname.get(arg, arg) for arg in inst["args"]] + lines.append(f' {_key_repr(args)}: {inst["py_name"]},') + if diagonal_shorthand and len(args) > 1 and len(set(args)) == 1: + lines.append(f' {_key_repr(args[:1])}: {inst["py_name"]},') + lines.append(" }") + return "\n".join(lines) + + +def render_generated_module( + package: str, + compiled_import: str, + classes: list[dict], + templated_classes: list[dict], + diagonal_shorthand: bool = False, + cxx_to_pyname: dict = None, +) -> str: + """ + Render a subpackage's ``_generated.py`` content. + + Parameters + ---------- + package : str + The Python import root (e.g. ``pyshapes``, ``chaste``), used to import + the shared ``_syntax`` helper. + compiled_import : str + The import statement pulling in the compiled extension's names. + classes : list[dict] + The subpackage's classes (from the model); used to know whether any + TemplateClass import is needed. + templated_classes : list[dict] + The subset that is templated, each rendered as a stub. + + Returns + ------- + str + The file content (ending with a newline). + """ + # Build black-clean output: docstring, one blank line, the imports, then each + # stub separated by two blank lines. + lines = [GENERATED_HEADER.rstrip("\n"), "", compiled_import] + if templated_classes: + lines.append(f"from {package}._syntax import TemplateClass") + for class_info in templated_classes: + lines.extend(["", ""]) # two blank lines before each top-level class + lines.append( + _stub_source( + class_info["base"], + class_info["instantiations"], + diagonal_shorthand, + cxx_to_pyname, + ) + ) + return "\n".join(lines) + "\n" + + +def _concrete_names(class_info: dict) -> list[str]: + """Return the concrete py_names of a class's instantiations.""" + return [inst["py_name"] for inst in class_info["instantiations"]] + + +def _explicit_import(package: str, compiled_module: str, names: list[str]) -> str: + """Render an explicit ``from . import (...)`` statement.""" + body = "".join(f" {name},\n" for name in sorted(names)) + return f"from {package}.{compiled_module} import (\n{body})" + + +def _format(content: str) -> str: + """Format with black when available, so output is stable and idempotent. + + Long instantiation keys (e.g. pychaste's CellsGenerator) exceed black's line + length and must be wrapped exactly as black would, or a project's + ``black --check`` / ``git diff`` reproducibility gate fails. When black is + not installed the content is written as-is (still valid Python). + """ + try: + import black + except ImportError: # pragma: no cover - only when black is absent + return content + return black.format_str(content, mode=black.Mode()) + + +def _write_generated(path: str, content: str, overwrite: bool) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + content = _format(content) + if write_file_if_changed(path, content, overwrite): + print(f"wrote {path}") + else: + print(f"unchanged {path}") + + +def generate_module_per_subpackage(model: dict, layout: dict, overwrite: bool): + """Emit one ``_generated.py`` per cppwg module (each owns its own extension).""" + package = layout.get("package", model["package"]) + package_root = layout["package_root"] + module_dirs = layout.get("module_dirs", {}) + diagonal_shorthand = layout.get("diagonal_shorthand", False) + cxx_to_pyname = _build_cxx_index(model) + + for module in model["modules"]: + # The subpackage directory (relative to package_root); default = module + # name, overridable (e.g. a single "all" module living at the root). + subdir = module_dirs.get(module["name"], module["name"]) + compiled_import = f"from .{module['compiled_module']} import *" + templated = [c for c in module["classes"] if c["templated"]] + content = render_generated_module( + package, + compiled_import, + module["classes"], + templated, + diagonal_shorthand, + cxx_to_pyname, + ) + path = os.path.join(package_root, subdir, "_generated.py") + _write_generated(path, content, overwrite) + + +def generate_shared_module_split(model: dict, layout: dict, overwrite: bool): + """Emit a ``_generated.py`` per subpackage, splitting one shared extension.""" + package = layout.get("package", model["package"]) + package_root = layout["package_root"] + compiled_module = layout["compiled_module"] + diagonal_shorthand = layout.get("diagonal_shorthand", False) + cxx_to_pyname = _build_cxx_index(model) + + # Index every wrapped entity across the model by name so a layout entry can + # be matched to a class (with its instantiations), an enum or a free function. + classes_by_base = {} + other_names = set() + for module in model["modules"]: + for class_info in module["classes"]: + classes_by_base[class_info["base"]] = class_info + for name in module["enums"] + module["free_functions"]: + other_names.add(name) + + assigned = set() + flatten_names = {} # subpackage -> the top-level names it exposes (issue #73) + for subpkg, names in layout["subpackages"].items(): + classes = [] + import_names = [] + exported = [] + for name in names: + assigned.add(name) + if name in classes_by_base: + class_info = classes_by_base[name] + classes.append(class_info) + import_names.extend(_concrete_names(class_info)) + exported.append(name) + elif name in other_names: + import_names.append(name) + exported.append(name) + else: + print(f"warning: '{name}' ({subpkg}) not found in model", file=sys.stderr) + + templated = [c for c in classes if c["templated"]] + compiled_import = _explicit_import(package, compiled_module, import_names) + content = render_generated_module( + package, + compiled_import, + classes, + templated, + diagonal_shorthand, + cxx_to_pyname, + ) + path = os.path.join(package_root, subpkg, "_generated.py") + _write_generated(path, content, overwrite) + flatten_names[subpkg] = exported + + # Flag any wrapped class not placed in a subpackage, so nothing is silently + # dropped when the config gains a class. + unplaced = sorted(set(classes_by_base) | other_names) + for name in unplaced: + if name not in assigned: + print(f"warning: '{name}' is wrapped but not assigned to a subpackage", + file=sys.stderr) + + if layout.get("flatten_to_root"): + generate_root_flatten(package, package_root, flatten_names, overwrite) + + +def generate_root_flatten( + package: str, package_root: str, flatten_names: dict, overwrite: bool +) -> None: + """Emit a top-level ``_generated.py`` re-exporting every subpackage's names. + + Gives ``package.ClassName`` in addition to ``package.subpackage.ClassName`` + (issue #73). Names are expected to be unique across subpackages; a clash is + warned about (the last subpackage's binding would win at import time). + """ + owner = {} # name -> subpackage that first exported it, for a clash check + for subpkg in sorted(flatten_names): + for name in flatten_names[subpkg]: + if name in owner: + print( + f"warning: '{name}' is exported by both '{owner[name]}' and " + f"'{subpkg}'; top-level {package}.{name} would be ambiguous", + file=sys.stderr, + ) + else: + owner[name] = subpkg + + lines = [FLATTEN_HEADER.rstrip("\n"), ""] + for subpkg in sorted(flatten_names): + names = flatten_names[subpkg] + if names: + lines.append(_explicit_import(package, subpkg, names)) + lines.append("") + lines.append("__all__ = [") + for name in sorted(owner): + lines.append(f' "{name}",') + lines.append("]") + content = "\n".join(lines) + "\n" + _write_generated(os.path.join(package_root, "_generated.py"), content, overwrite) + + +def main(argv=None) -> int: + """Entry point: read the model + layout and write the _generated.py files.""" + parser = argparse.ArgumentParser(prog="cppwg genpackage", description=__doc__) + parser.add_argument( + "--model", required=True, help="Path to cppwg_package_model.yaml (from cppwg)." + ) + parser.add_argument( + "--layout", required=True, help="Path to the Python package-layout file (YAML)." + ) + parser.add_argument( + "--overwrite", + action="store_true", + help="Rewrite files even if unchanged.", + ) + args = parser.parse_args(argv) + + import yaml + + with open(args.model) as model_file: + model = yaml.safe_load(model_file) + with open(args.layout) as layout_file: + layout = yaml.safe_load(layout_file) + + # Resolve a relative package_root against the layout's own directory, so a + # layout is portable regardless of the caller's working directory. + if not os.path.isabs(layout["package_root"]): + layout_dir = os.path.dirname(os.path.abspath(args.layout)) + layout["package_root"] = os.path.normpath( + os.path.join(layout_dir, layout["package_root"]) + ) + + if layout.get("subpackages"): + generate_shared_module_split(model, layout, args.overwrite) + else: + generate_module_per_subpackage(model, layout, args.overwrite) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_genpackage.py b/tests/test_genpackage.py index 05866fb..c0f94c0 100644 --- a/tests/test_genpackage.py +++ b/tests/test_genpackage.py @@ -1,14 +1,11 @@ -"""Unit tests for tools/cppwg_genpackage.py.""" +"""Unit tests for cppwg.genpackage (the `cppwg genpackage` subcommand).""" -import importlib.util -import os +import sys -_GENPACKAGE_PATH = os.path.join( - os.path.dirname(os.path.dirname(__file__)), "tools", "cppwg_genpackage.py" -) -_spec = importlib.util.spec_from_file_location("cppwg_genpackage", _GENPACKAGE_PATH) -genpackage = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(genpackage) +import pytest + +import cppwg.__main__ as cppwg_main +from cppwg import genpackage def _class(base, instantiations, templated=True): @@ -309,3 +306,24 @@ def test_shared_split_warns_on_unknown_and_unassigned(tmp_path, capsys): err = capsys.readouterr().err assert "'DoesNotExist'" in err # listed but not in model assert "'Orphan'" in err # in model but not assigned to a subpackage + + +def test_main_dispatches_genpackage_subcommand(monkeypatch): + """`cppwg genpackage ...` routes to genpackage.main with the remaining args.""" + captured = {} + + def fake_genpackage_main(argv): + captured["argv"] = argv + return 0 + + monkeypatch.setattr(genpackage, "main", fake_genpackage_main) + monkeypatch.setattr( + sys, "argv", ["cppwg", "genpackage", "--model", "m.yaml", "--layout", "l.yaml"] + ) + + with pytest.raises(SystemExit) as exc_info: + cppwg_main.main() + + assert exc_info.value.code == 0 + # The "genpackage" token is stripped; only the sub-args reach genpackage.main. + assert captured["argv"] == ["--model", "m.yaml", "--layout", "l.yaml"] diff --git a/tools/cppwg_genpackage.py b/tools/cppwg_genpackage.py index b881462..c05c481 100644 --- a/tools/cppwg_genpackage.py +++ b/tools/cppwg_genpackage.py @@ -1,378 +1,15 @@ #!/usr/bin/env python3 -"""Generate the Python package layer (``_generated.py``) for a cppwg project. +"""Standalone launcher for the cppwg package-layer generator. -cppwg generates the C++/pybind11 wrappers and, alongside them, a YAML model of -what it produced (``cppwg_package_model.yaml`` in the wrapper root - see -``cppwg.utils.package_model``). This standalone script turns that model, plus a -small package-layout file, into a ``_generated.py`` per Python subpackage: the -compiled-extension import and the ``TemplateClass`` subscript stubs -(``Point[2] -> Point_2``). It never touches the hand-written ``__init__.py``, -which does ``from ._generated import *`` and adds the bespoke pieces -(``TemplateMethod`` attachments, package ``init()``, curation, comments). - -Two layouts: - -* module-per-subpackage (default): each cppwg module becomes a subpackage that - owns its own compiled extension, imported with ``from . import *``. - Used by the shapes/cells examples. -* shared-module split: one compiled extension (e.g. ``_pychaste_all``) is split - into several subpackages by a layout that lists which names each owns; - imported explicitly with ``from . import (...)``. Used by - pychaste. - -Usage:: - - cppwg_genpackage.py --model wrapper/cppwg_package_model.yaml --layout package_layout.yaml +The implementation lives in :mod:`cppwg.genpackage`; the canonical invocation is +``cppwg genpackage ...``. This launcher lets the generator also be run directly +from a checkout (``python tools/cppwg_genpackage.py ...``) when cppwg is +importable. """ -import argparse -import os import sys -# Reuse cppwg's idempotent writer when importable; fall back to a local copy so -# the script also runs standalone (e.g. from a build tree without cppwg on the -# path). -try: - from cppwg.utils.utils import write_file_if_changed -except ImportError: # pragma: no cover - exercised only without cppwg installed - - def write_file_if_changed(filepath, content, overwrite=False): - """Write content unless an identical file already exists.""" - if not overwrite and os.path.isfile(filepath): - with open(filepath) as in_file: - if in_file.read() == content: - return False - with open(filepath, "w") as out_file: - out_file.write(content) - return True - - -GENERATED_HEADER = ( - '"""Generated by tools/cppwg_genpackage.py - do not edit.\n' - "\n" - "Compiled-extension imports and TemplateClass subscript stubs for this\n" - "subpackage. Hand-written code (TemplateMethod attachments, package init,\n" - 'curation) belongs in the sibling __init__.py, which does `from ._generated\n' - 'import *`.\n' - '"""\n' -) - - -FLATTEN_HEADER = ( - '"""Generated by tools/cppwg_genpackage.py - do not edit.\n' - "\n" - "Re-exports every wrapped class, enum and free function from the subpackages\n" - "into the top-level package namespace, so `package.ClassName` works as well\n" - "as `package.subpackage.ClassName`. Importing this imports the subpackages.\n" - "The hand-written __init__.py does `from ._generated import *` and adds the\n" - "bespoke top-level pieces (package init, etc.).\n" - '"""\n' -) - - -def _key_repr(args: list[str]) -> str: - """Render a template-argument list as a Python tuple literal of strings. - - e.g. ``["2"]`` -> ``("2",)`` and ``["Cell", "2"]`` -> ``("Cell", "2")``. - Matches _syntax._normalize_key, which normalizes keys to string tuples. - """ - inner = ", ".join(f'"{arg}"' for arg in args) - if len(args) == 1: - inner += "," - return f"({inner})" - - -def _build_cxx_index(model: dict) -> dict: - """Map each wrapped instantiation's C++ type string to its Python class name. - - e.g. ``PottsMesh<2> -> PottsMesh_2``. Used to key a template argument that is - itself a wrapped templated type (``MeshFactory>``) by the Python - concrete class name, so ``MeshFactory[PottsMesh[2]]`` resolves: ``PottsMesh[2]`` - is the ``PottsMesh_2`` class and ``_normalize_key`` keys it by its ``__name__``. - """ - index = {} - for module in model["modules"]: - for class_info in module["classes"]: - for inst in class_info["instantiations"]: - if inst["args"]: - cxx = f'{class_info["base"]}<{",".join(inst["args"])}>' - index[cxx] = inst["py_name"] - return index - - -def _stub_source( - base: str, - instantiations: list[dict], - diagonal_shorthand: bool = False, - cxx_to_pyname: dict = None, -) -> str: - """Render a ``class (TemplateClass)`` stub for a templated class. - - A template argument that is itself a wrapped templated type is keyed by its - Python concrete class name via ``cxx_to_pyname`` (``PottsMesh<2>`` -> - ``PottsMesh_2``), so ``MeshFactory[PottsMesh[2]]`` resolves. - - When ``diagonal_shorthand`` is set, a multi-argument instantiation whose - arguments are all equal (a "diagonal", e.g. ``Element<2, 2>``) also gets a - single-argument alias key (``Element[2] -> Element_2_2``). This is an - opt-in Python-layer convenience some packages (pychaste) adopt for their - ````-style classes; others (the cells example) keep - the explicit multi-argument form only. - """ - cxx_to_pyname = cxx_to_pyname or {} - lines = [f"class {base}(TemplateClass):", " _instantiations = {"] - for inst in instantiations: - args = [cxx_to_pyname.get(arg, arg) for arg in inst["args"]] - lines.append(f' {_key_repr(args)}: {inst["py_name"]},') - if diagonal_shorthand and len(args) > 1 and len(set(args)) == 1: - lines.append(f' {_key_repr(args[:1])}: {inst["py_name"]},') - lines.append(" }") - return "\n".join(lines) - - -def render_generated_module( - package: str, - compiled_import: str, - classes: list[dict], - templated_classes: list[dict], - diagonal_shorthand: bool = False, - cxx_to_pyname: dict = None, -) -> str: - """ - Render a subpackage's ``_generated.py`` content. - - Parameters - ---------- - package : str - The Python import root (e.g. ``pyshapes``, ``chaste``), used to import - the shared ``_syntax`` helper. - compiled_import : str - The import statement pulling in the compiled extension's names. - classes : list[dict] - The subpackage's classes (from the model); used to know whether any - TemplateClass import is needed. - templated_classes : list[dict] - The subset that is templated, each rendered as a stub. - - Returns - ------- - str - The file content (ending with a newline). - """ - # Build black-clean output: docstring, one blank line, the imports, then each - # stub separated by two blank lines. - lines = [GENERATED_HEADER.rstrip("\n"), "", compiled_import] - if templated_classes: - lines.append(f"from {package}._syntax import TemplateClass") - for class_info in templated_classes: - lines.extend(["", ""]) # two blank lines before each top-level class - lines.append( - _stub_source( - class_info["base"], - class_info["instantiations"], - diagonal_shorthand, - cxx_to_pyname, - ) - ) - return "\n".join(lines) + "\n" - - -def _concrete_names(class_info: dict) -> list[str]: - """Return the concrete py_names of a class's instantiations.""" - return [inst["py_name"] for inst in class_info["instantiations"]] - - -def _explicit_import(package: str, compiled_module: str, names: list[str]) -> str: - """Render an explicit ``from . import (...)`` statement.""" - body = "".join(f" {name},\n" for name in sorted(names)) - return f"from {package}.{compiled_module} import (\n{body})" - - -def _format(content: str) -> str: - """Format with black when available, so output is stable and idempotent. - - Long instantiation keys (e.g. pychaste's CellsGenerator) exceed black's line - length and must be wrapped exactly as black would, or a project's - ``black --check`` / ``git diff`` reproducibility gate fails. When black is - not installed the content is written as-is (still valid Python). - """ - try: - import black - except ImportError: # pragma: no cover - only when black is absent - return content - return black.format_str(content, mode=black.Mode()) - - -def _write_generated(path: str, content: str, overwrite: bool) -> None: - os.makedirs(os.path.dirname(path), exist_ok=True) - content = _format(content) - if write_file_if_changed(path, content, overwrite): - print(f"wrote {path}") - else: - print(f"unchanged {path}") - - -def generate_module_per_subpackage(model: dict, layout: dict, overwrite: bool): - """Emit one ``_generated.py`` per cppwg module (each owns its own extension).""" - package = layout.get("package", model["package"]) - package_root = layout["package_root"] - module_dirs = layout.get("module_dirs", {}) - diagonal_shorthand = layout.get("diagonal_shorthand", False) - cxx_to_pyname = _build_cxx_index(model) - - for module in model["modules"]: - # The subpackage directory (relative to package_root); default = module - # name, overridable (e.g. a single "all" module living at the root). - subdir = module_dirs.get(module["name"], module["name"]) - compiled_import = f"from .{module['compiled_module']} import *" - templated = [c for c in module["classes"] if c["templated"]] - content = render_generated_module( - package, - compiled_import, - module["classes"], - templated, - diagonal_shorthand, - cxx_to_pyname, - ) - path = os.path.join(package_root, subdir, "_generated.py") - _write_generated(path, content, overwrite) - - -def generate_shared_module_split(model: dict, layout: dict, overwrite: bool): - """Emit a ``_generated.py`` per subpackage, splitting one shared extension.""" - package = layout.get("package", model["package"]) - package_root = layout["package_root"] - compiled_module = layout["compiled_module"] - diagonal_shorthand = layout.get("diagonal_shorthand", False) - cxx_to_pyname = _build_cxx_index(model) - - # Index every wrapped entity across the model by name so a layout entry can - # be matched to a class (with its instantiations), an enum or a free function. - classes_by_base = {} - other_names = set() - for module in model["modules"]: - for class_info in module["classes"]: - classes_by_base[class_info["base"]] = class_info - for name in module["enums"] + module["free_functions"]: - other_names.add(name) - - assigned = set() - flatten_names = {} # subpackage -> the top-level names it exposes (issue #73) - for subpkg, names in layout["subpackages"].items(): - classes = [] - import_names = [] - exported = [] - for name in names: - assigned.add(name) - if name in classes_by_base: - class_info = classes_by_base[name] - classes.append(class_info) - import_names.extend(_concrete_names(class_info)) - exported.append(name) - elif name in other_names: - import_names.append(name) - exported.append(name) - else: - print(f"warning: '{name}' ({subpkg}) not found in model", file=sys.stderr) - - templated = [c for c in classes if c["templated"]] - compiled_import = _explicit_import(package, compiled_module, import_names) - content = render_generated_module( - package, - compiled_import, - classes, - templated, - diagonal_shorthand, - cxx_to_pyname, - ) - path = os.path.join(package_root, subpkg, "_generated.py") - _write_generated(path, content, overwrite) - flatten_names[subpkg] = exported - - # Flag any wrapped class not placed in a subpackage, so nothing is silently - # dropped when the config gains a class. - unplaced = sorted(set(classes_by_base) | other_names) - for name in unplaced: - if name not in assigned: - print(f"warning: '{name}' is wrapped but not assigned to a subpackage", - file=sys.stderr) - - if layout.get("flatten_to_root"): - generate_root_flatten(package, package_root, flatten_names, overwrite) - - -def generate_root_flatten( - package: str, package_root: str, flatten_names: dict, overwrite: bool -) -> None: - """Emit a top-level ``_generated.py`` re-exporting every subpackage's names. - - Gives ``package.ClassName`` in addition to ``package.subpackage.ClassName`` - (issue #73). Names are expected to be unique across subpackages; a clash is - warned about (the last subpackage's binding would win at import time). - """ - owner = {} # name -> subpackage that first exported it, for a clash check - for subpkg in sorted(flatten_names): - for name in flatten_names[subpkg]: - if name in owner: - print( - f"warning: '{name}' is exported by both '{owner[name]}' and " - f"'{subpkg}'; top-level {package}.{name} would be ambiguous", - file=sys.stderr, - ) - else: - owner[name] = subpkg - - lines = [FLATTEN_HEADER.rstrip("\n"), ""] - for subpkg in sorted(flatten_names): - names = flatten_names[subpkg] - if names: - lines.append(_explicit_import(package, subpkg, names)) - lines.append("") - lines.append("__all__ = [") - for name in sorted(owner): - lines.append(f' "{name}",') - lines.append("]") - content = "\n".join(lines) + "\n" - _write_generated(os.path.join(package_root, "_generated.py"), content, overwrite) - - -def main(argv=None) -> int: - """Entry point: read the model + layout and write the _generated.py files.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--model", required=True, help="Path to cppwg_package_model.yaml (from cppwg)." - ) - parser.add_argument( - "--layout", required=True, help="Path to the Python package-layout file (YAML)." - ) - parser.add_argument( - "--overwrite", - action="store_true", - help="Rewrite files even if unchanged.", - ) - args = parser.parse_args(argv) - - import yaml - - with open(args.model) as model_file: - model = yaml.safe_load(model_file) - with open(args.layout) as layout_file: - layout = yaml.safe_load(layout_file) - - # Resolve a relative package_root against the layout's own directory, so a - # layout is portable regardless of the caller's working directory. - if not os.path.isabs(layout["package_root"]): - layout_dir = os.path.dirname(os.path.abspath(args.layout)) - layout["package_root"] = os.path.normpath( - os.path.join(layout_dir, layout["package_root"]) - ) - - if layout.get("subpackages"): - generate_shared_module_split(model, layout, args.overwrite) - else: - generate_module_per_subpackage(model, layout, args.overwrite) - return 0 - +from cppwg.genpackage import main if __name__ == "__main__": sys.exit(main()) From 160818b20fa20574483fc27efce5af94dd794d93 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 12:25:16 +0100 Subject: [PATCH 11/24] #102 Add a do-not-edit banner to the emitted package model The cppwg_package_model.yaml files carried no header, unlike the generated wrapper sources (and the _generated.py files). Prepend a two-line YAML-comment banner (CPPWG_PACKAGE_MODEL_HEADER) so the committed model reads clearly as a generated artifact. The banner is ignored on load, so the model still parses. Co-Authored-By: Claude Opus 4.8 --- cppwg/generators.py | 12 ++++++++---- cppwg/utils/constants.py | 8 ++++++++ .../cells/dynamic/wrappers/cppwg_package_model.yaml | 2 ++ examples/shapes/wrapper/cppwg_package_model.yaml | 2 ++ 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/cppwg/generators.py b/cppwg/generators.py index 7fd8500..e11f3c1 100644 --- a/cppwg/generators.py +++ b/cppwg/generators.py @@ -20,6 +20,7 @@ CPPWG_DEFAULT_WRAPPER_DIR, CPPWG_HEADER_COLLECTION_FILENAME, CPPWG_PACKAGE_MODEL_FILENAME, + CPPWG_PACKAGE_MODEL_HEADER, ) from cppwg.utils.package_model import build_package_model from cppwg.version import __version__ as cppwg_version @@ -430,13 +431,16 @@ def write_package_model(self) -> None: Write the package model (cppwg_package_model.yaml) to the wrapper root. A small YAML description of the generated modules and their classes / - instantiations / enums / free functions, so a separate step - (tools/cppwg_genpackage.py) can generate the Python package layer without - re-parsing the source. Written last, once the info tree is final. + instantiations / enums / free functions, so a separate step (the + ``cppwg genpackage`` subcommand) can generate the Python package layer + without re-parsing the source. Written last, once the info tree is final. """ model = build_package_model(self.package_info) model_path = os.path.join(self.wrapper_root, CPPWG_PACKAGE_MODEL_FILENAME) - content = yaml.safe_dump(model, default_flow_style=False, sort_keys=True) + # Prepend a do-not-edit banner (YAML comments, ignored on load). + content = CPPWG_PACKAGE_MODEL_HEADER + yaml.safe_dump( + model, default_flow_style=False, sort_keys=True + ) utils.write_file_if_changed(model_path, content, self.overwrite) def generate(self) -> None: diff --git a/cppwg/utils/constants.py b/cppwg/utils/constants.py index a122974..a2a2f88 100644 --- a/cppwg/utils/constants.py +++ b/cppwg/utils/constants.py @@ -14,6 +14,14 @@ # cppwg.utils.package_model and tools/cppwg_genpackage.py). CPPWG_PACKAGE_MODEL_FILENAME = "cppwg_package_model.yaml" +# Do-not-edit banner prepended to the emitted package model (YAML comments, +# ignored on load), mirroring the banner on the generated wrapper sources so the +# committed model reads clearly as a generated artifact. +CPPWG_PACKAGE_MODEL_HEADER = ( + "# This file is automatically generated by cppwg.\n" + "# Do not modify this file directly.\n" +) + CPPWG_TRUE_STRINGS = ["ON", "YES", "Y", "TRUE", "T", "1"] CPPWG_FALSE_STRINGS = ["OFF", "NO", "N", "FALSE", "F", "0", ""] diff --git a/examples/cells/dynamic/wrappers/cppwg_package_model.yaml b/examples/cells/dynamic/wrappers/cppwg_package_model.yaml index 41341d8..9594841 100644 --- a/examples/cells/dynamic/wrappers/cppwg_package_model.yaml +++ b/examples/cells/dynamic/wrappers/cppwg_package_model.yaml @@ -1,3 +1,5 @@ +# This file is automatically generated by cppwg. +# Do not modify this file directly. modules: - classes: - base: Cell diff --git a/examples/shapes/wrapper/cppwg_package_model.yaml b/examples/shapes/wrapper/cppwg_package_model.yaml index 2550420..a626951 100644 --- a/examples/shapes/wrapper/cppwg_package_model.yaml +++ b/examples/shapes/wrapper/cppwg_package_model.yaml @@ -1,3 +1,5 @@ +# This file is automatically generated by cppwg. +# Do not modify this file directly. modules: - classes: [] compiled_module: _pyshapes_math_funcs From ad1438cf7d67a7f7baa7c6530ac0e6567cb814d1 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 14:22:27 +0100 Subject: [PATCH 12/24] #102 Document Python package generation Add a "Python packages" doc page covering the `cppwg genpackage` subcommand: the emitted package model, the two layouts (one subpackage per module with `module_dirs`, and the shared-extension split with `subpackages`), the `TemplateClass` subscript stubs, the `diagonal_shorthand`/`flatten_to_root` options, and the hand-written `__init__.py` bits. Slot it into the toctree after custom-generators. Also lowercase the "CppWG" branding to "cppwg" in the index/basics/configuration pages for consistency with the command name. Co-Authored-By: Claude Opus 4.8 --- doc/basics.md | 2 +- doc/configuration.md | 2 +- doc/index.md | 3 +- doc/python-packages.md | 295 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 299 insertions(+), 3 deletions(-) create mode 100644 doc/python-packages.md diff --git a/doc/basics.md b/doc/basics.md index 8dee90e..59c3120 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -1,6 +1,6 @@ # Basics -CppWG is driven by a YAML configuration file that describes a **package** made +cppwg is driven by a YAML configuration file that describes a **package** made of one or more **modules**, and the **classes**, **free functions** and **enums** each module wraps. diff --git a/doc/configuration.md b/doc/configuration.md index 8b7e172..3333dbf 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -1,6 +1,6 @@ # Configuration -CppWG is driven by a YAML configuration file that describes what to wrap. +cppwg is driven by a YAML configuration file that describes what to wrap. This section describes how to create a configuration for your project. The [Basics](basics.md) page gets you started, the [Reference](reference.md) page diff --git a/doc/index.md b/doc/index.md index 52db553..9cf04a4 100644 --- a/doc/index.md +++ b/doc/index.md @@ -1,6 +1,6 @@ # CppWG -CppWG is a C++ wrapper generator for Python. It reads your C++ source code, +cppwg is a C++ wrapper generator for Python. It reads your C++ source code, together with a YAML config file, and emits [pybind11](https://pybind11.readthedocs.io/) wrapper code. Using the YAML config file, you describe what parts of your C++ project to expose to Python rather than writing pybind11 wrappers directly. @@ -13,4 +13,5 @@ installation first-steps configuration custom-generators +python-packages ``` diff --git a/doc/python-packages.md b/doc/python-packages.md new file mode 100644 index 0000000..799152f --- /dev/null +++ b/doc/python-packages.md @@ -0,0 +1,295 @@ +# Python packages + +A full Python package may need more than the compiled extension generated by cppwg. +For example, a package may want to add syntactic sugar for wrapped templated +classes to enable using the `Foo[Bar]` subscript syntax in Python instead of the +actually wrapped `Foo_Bar` and more closely mirror the `Foo` usage in C++. + +This can be added by hand in the Python package, but can drift from the config +whenever the wrapped instantiations change. To help address this, the +`cppwg genpackage` tool can be used to generate this **Python package layer** +from the same config that produced the wrappers, so they stay in step. + +## How it works + +Generation is a two-step flow: + +1. When cppwg generates the wrappers it also writes a **package model**, + `cppwg_package_model.yaml`. This is a small YAML description of what was + wrapped — each module, its compiled extension, and the wrapped classes (with + their template instantiations), enums and free functions. + +2. `cppwg genpackage` reads that model, together with a **layout file** you + provide describing the layout of the Python package, and emits a + `_generated.py` for each Python subpackage. In `_generated.py`, all the + wrapped elements are imported from the compiled-extension, and one + [`TemplateClass`](#templateclass) subscript stub is added per templated + class to allow the `Foo[Bar]` syntax from Python. The `_generated.py` can + then be imported inside an `__init__.py`, where other things can be manually + added that cannot be derived from the config + (see [Hand-written code](#hand-written-code)). + +(templateclass)= +### `TemplateClass` + +A templated class is exposed as a `TemplateClass` subclass whose `_instantiations` +maps template-argument tuples to the concrete wrapped classes. Subscripting the +class (`Point[2]`) looks up the concrete class (`Point_2`). These can be nested +so e.g. `MeshFactory>` is subscripted as `MeshFactory[PottsMesh[2]]` +(see `examples/cells`). + +:::{note} +The generated stubs import `TemplateClass` (and, where you use it, +`TemplateMethod`) from a `_syntax.py` helper in your package. This helper is not +generated — copy it from `examples/shapes/src/py/pyshapes/_syntax.py`. +::: + +## Layouts + +`cppwg genpackage` supports two layouts, and your supplied layout file chooses +between them: + +- **One subpackage per module** (default): each cppwg module has its **own** + compiled extension and becomes its own subpackage. The optional `module_dirs` + option controls *where* each subpackage is written. +- **One shared extension split into subpackages**: everything is compiled into a + **single** extension, which the `subpackages` option then divides across + several subpackages by name. + +## One subpackage per module + +Each cppwg [module](reference.md#module-options) has its own subpackage, which +star-imports the relevant compiled extension with `from . import *`. +This is the default, and what `examples/shapes` uses. + +**package_layout.yaml** + +`examples/shapes/wrapper/package_layout.yaml` + +```yaml +package: pyshapes +package_root: ../src/py/pyshapes +``` + +:::{note} +`package_root` is the directory holding the Python subpackage; a relative path +is resolved against the layout file's own directory. +::: + +Run: + +```bash +cppwg genpackage \ + --model wrapper/cppwg_package_model.yaml \ + --layout wrapper/package_layout.yaml +``` + +For the `geometry` module, this writes: + +**geometry/_generated.py** + +```python +"""Generated by tools/cppwg_genpackage.py - do not edit. +... +""" + +from ._pyshapes_geometry import * +from pyshapes._syntax import TemplateClass + + +class Point(TemplateClass): + _instantiations = { + ("2",): Point_2, + ("3",): Point_3, + } +``` + +This is imported into the hand-written `__init__.py` beside it: + +**geometry/__init__.py** + +```python +from ._generated import * # noqa: F401,F403 +``` + +The package can then be imported with both the concrete names and the subscript form: + +```python +from pyshapes.geometry import Point, Point_2 + +Point[2] # -> Point_2 +Point[3] # -> Point_3 +``` + +### Mapping modules to subpackage directories + +By default each module's subpackage directory is named after the module, under +`package_root` — so the `geometry` module above becomes `pyshapes/geometry/`. The +`module_dirs` option overrides that directory per module: the key is the module +name, the value is a directory relative to `package_root`. Use it to rename a +subpackage, nest it, or map a module to `"."` to place it at the package root. A +module not listed keeps the default. + +`examples/cells` has a single module, `all`, and maps it to `"."` so its extension +sits at the package root rather than in an `all/` subfolder: + +`examples/cells/dynamic/package_layout.yaml` + +```yaml +package: pycells +package_root: ../src/py/pycells +module_dirs: + all: "." +``` + +The `_generated.py` is then written to `pycells/_generated.py`, and +`from ._generated import *` in `pycells/__init__.py` surfaces everything at the top +level (`from pycells import Node`). + +`module_dirs` only relocates a module's files; every module still keeps its own +compiled extension. To split a *single* extension instead, use `subpackages` +([below](#splitting-one-shared-extension)). + +(splitting-one-shared-extension)= +## Splitting one shared extension into subpackages + +In the second layout all wrappers are compiled into a **single** extension, named +by `compiled_module`, but you still want several Python subpackages. The +`subpackages` option divides that one extension's contents manually by **name**. +For each subpackage, list the (base) class, enum and free-function names it should own. + +```yaml +package: chaste +package_root: src/py/chaste +compiled_module: _pychaste_all +subpackages: + core: [FileFinder, OutputFileHandler, RandomNumberGenerator, Timer, ...] + mesh: [ChastePoint, Element, MutableMesh, Node, ...] + # ... +``` + +Because they all depend on a single compiled extension, each subpackage's +`_generated.py` **explicitly** imports its own names from that extension rather +than star-importing it. A name listed under no subpackage, or listed but absent +from the model, is reported as a warning. + +For the `mesh` subpackage this writes the explicit imports plus a `TemplateClass` +stub per templated class: + +**mesh/_generated.py** + +```python +"""Generated by tools/cppwg_genpackage.py - do not edit. +... +""" + +from chaste._pychaste_all import ( + Element_1_1, + Element_1_2, + Element_1_3, + Element_2_2, + Element_2_3, + Element_3_3, + # ... +) +from chaste._syntax import TemplateClass + + +class Element(TemplateClass): + _instantiations = { + ("1", "1"): Element_1_1, + ("1", "2"): Element_1_2, + ("1", "3"): Element_1_3, + ("2", "2"): Element_2_2, + ("2", "3"): Element_2_3, + ("3", "3"): Element_3_3, + } + + +# ... one stub per templated class +``` + +As in the per-module layout, a hand-written `__init__.py` in each subpackage +star-imports this via `from ._generated import *`. + +(hand-written-code)= +## Hand-written code + +Anything that cannot be derived from the config stays in the hand-written +`__init__.py`, after the `from ._generated import *` line. A common case is a +templated *method* (`TemplateMethod`), which the model does not describe: + +**primitives/__init__.py** + +```python +from ._generated import * # noqa: F401,F403 +from pyshapes._syntax import TemplateMethod + +# UnitSquare.GetAreaIn[Unit]() — a templated method, so it cannot be +# auto-generated and is attached here. +UnitSquare.GetAreaIn = TemplateMethod("GetAreaIn", UnitSquare.GetAreaIn) +``` + +## Options + +### `diagonal_shorthand` + +For a multi-argument instantiation whose arguments are all equal (a "diagonal", +e.g. `Element<2, 2>`), also emit a single-argument alias so `Element[2]` resolves +to the same class as `Element[2, 2]`. Off by default; enable it in the layout file: + +```yaml +diagonal_shorthand: true +``` + +The `Element` stub from the split example above then gains a single-argument alias +key for each diagonal instantiation (the `("1",)`, `("2",)` and `("3",)` entries): + +```python +class Element(TemplateClass): + _instantiations = { + ("1", "1"): Element_1_1, + ("1",): Element_1_1, + ("1", "2"): Element_1_2, + ("1", "3"): Element_1_3, + ("2", "2"): Element_2_2, + ("2",): Element_2_2, + ("2", "3"): Element_2_3, + ("3", "3"): Element_3_3, + ("3",): Element_3_3, + } +``` + +so `Element[2]` now resolves to `Element_2_2` alongside `Element[2, 2]`. + +### `flatten_to_root` + +Also write a top-level `_generated.py` that re-exports every subpackage's class, +enum and free-function names into the package root, so `chaste.Node` works +alongside `chaste.mesh.Node`. A name exported by more than one subpackage is +reported as an ambiguity warning. Off by default: + +```yaml +flatten_to_root: true +``` + +## Command-line usage + +```text +usage: cppwg genpackage [-h] --model MODEL --layout LAYOUT [--overwrite] + +options: + -h, --help show this help message and exit + --model MODEL Path to cppwg_package_model.yaml (from cppwg). + --layout LAYOUT Path to the Python package-layout file (YAML). + --overwrite Rewrite files even if unchanged. +``` + +The generated `_generated.py` files are reproducible: re-running `cppwg genpackage` +after a config change updates only what changed, so the layer can be checked in and +verified with `git diff`. + +:::{seealso} +- See [First steps](first-steps.md) for generating the wrappers themselves. +- See [Templates](templates.md) for choosing which instantiations are wrapped. +::: From 47e196939387cf8c03de4a384df5fa373686a1ce Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 14:41:52 +0100 Subject: [PATCH 13/24] #102 Cover the genpackage subcommand's main() Moving the package-layer generator into the cppwg package put it under coverage measurement for the first time, exposing that main() (and a couple of branches) were untested. Add tests exercising main() end-to-end for both dispatch paths, the "unchanged" second-run branch, and the flatten skip for a subpackage that exports nothing. genpackage.py 88% -> 100%. Co-Authored-By: Claude Opus 4.8 --- tests/test_genpackage.py | 80 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/tests/test_genpackage.py b/tests/test_genpackage.py index c0f94c0..4e13d6c 100644 --- a/tests/test_genpackage.py +++ b/tests/test_genpackage.py @@ -1,5 +1,6 @@ """Unit tests for cppwg.genpackage (the `cppwg genpackage` subcommand).""" +import json import sys import pytest @@ -327,3 +328,82 @@ def fake_genpackage_main(argv): assert exc_info.value.code == 0 # The "genpackage" token is stripped; only the sub-args reach genpackage.main. assert captured["argv"] == ["--model", "m.yaml", "--layout", "l.yaml"] + + +def test_main_end_to_end(tmp_path, capsys): + """main() reads the model + layout files, resolves a relative package_root, writes files.""" + model = { + "package": "pyshapes", + "modules": [ + { + "name": "geometry", + "compiled_module": "_pyshapes_geometry", + "imports": [], + "enums": [], + "free_functions": [], + "classes": [_class("Point", [_inst(["2"], "Point_2")])], + } + ], + } + # A relative package_root is resolved against the layout file's own directory. + layout = {"package": "pyshapes", "package_root": "pkg"} + (tmp_path / "model.yaml").write_text(json.dumps(model)) # JSON is valid YAML + (tmp_path / "package_layout.yaml").write_text(json.dumps(layout)) + argv = [ + "--model", + str(tmp_path / "model.yaml"), + "--layout", + str(tmp_path / "package_layout.yaml"), + ] + + assert genpackage.main(argv) == 0 + generated = tmp_path / "pkg" / "geometry" / "_generated.py" + assert "class Point(TemplateClass):" in generated.read_text() + assert "wrote" in capsys.readouterr().out + + # A second run leaves the file untouched (the "unchanged" branch). + assert genpackage.main(argv) == 0 + assert "unchanged" in capsys.readouterr().out + + +def test_main_dispatches_shared_split(tmp_path): + """main() takes the shared-split path when the layout has a `subpackages` key.""" + model = { + "package": "pychaste", + "modules": [ + { + "name": "all", + "compiled_module": "_pychaste_all", + "imports": [], + "enums": [], + "free_functions": [], + "classes": [_class("Node", [_inst(["2"], "Node_2")])], + } + ], + } + layout = { + "package": "chaste", + "package_root": str(tmp_path), + "compiled_module": "_pychaste_all", + "subpackages": {"mesh": ["Node"]}, + } + (tmp_path / "model.yaml").write_text(json.dumps(model)) + (tmp_path / "layout.yaml").write_text(json.dumps(layout)) + + code = genpackage.main( + ["--model", str(tmp_path / "model.yaml"), "--layout", str(tmp_path / "layout.yaml")] + ) + assert code == 0 + assert "class Node(TemplateClass):" in (tmp_path / "mesh" / "_generated.py").read_text() + + +def test_flatten_skips_subpackage_with_no_exports(tmp_path): + """generate_root_flatten omits a subpackage that exports no names.""" + genpackage.generate_root_flatten( + "pkg", str(tmp_path), {"a": ["Widget"], "b": []}, overwrite=False + ) + + root = (tmp_path / "_generated.py").read_text() + assert "from pkg.a import (" in root # a exports a name + assert "from pkg.b import (" not in root # b exports nothing -> skipped + assert '"Widget",' in root # __all__ From 70745b588a9e23339851643dbccd1e3cd1bb6aa1 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 16:15:55 +0100 Subject: [PATCH 14/24] #102 Address review comments in the package-layer generator - _key_repr renders each template argument with json.dumps, so a value with special characters produces a valid, escaped Python string literal. - An empty (or all-unknown) subpackage now imports the extension module instead of emitting `from ... import ()`, which is a SyntaxError. - Drop the optional-black formatting pass: the generator emits its final layout directly, the way cppwg's C++ templates do, so output no longer depends on whether black happens to be installed. - Fix a typo in the BASE_INFO_OPTIONS comment ("became" -> "become"). Co-Authored-By: Claude Opus 4.8 --- cppwg/genpackage.py | 39 +++++++++++++++++---------------------- cppwg/info/base_info.py | 2 +- tests/test_genpackage.py | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 23 deletions(-) diff --git a/cppwg/genpackage.py b/cppwg/genpackage.py index 5e5867e..bdae9e9 100644 --- a/cppwg/genpackage.py +++ b/cppwg/genpackage.py @@ -23,10 +23,11 @@ Usage:: - cppwg genpackage --model wrapper/cppwg_package_model.yaml --layout package_layout.yaml + cppwg genpackage --model wrapper/cppwg_package_model.json --layout package_layout.yaml """ import argparse +import json import os import sys @@ -60,9 +61,11 @@ def _key_repr(args: list[str]) -> str: """Render a template-argument list as a Python tuple literal of strings. e.g. ``["2"]`` -> ``("2",)`` and ``["Cell", "2"]`` -> ``("Cell", "2")``. - Matches _syntax._normalize_key, which normalizes keys to string tuples. + Matches _syntax._normalize_key, which normalizes keys to string tuples. Each + value is rendered with ``json.dumps`` so any special characters are escaped + into a valid (double-quoted) Python string literal. """ - inner = ", ".join(f'"{arg}"' for arg in args) + inner = ", ".join(json.dumps(arg) for arg in args) if len(args) == 1: inner += "," return f"({inner})" @@ -145,8 +148,9 @@ def render_generated_module( str The file content (ending with a newline). """ - # Build black-clean output: docstring, one blank line, the imports, then each - # stub separated by two blank lines. + # Emit black-style output directly, the way the C++ templates emit clean C++: + # docstring, one blank line, the imports, then each stub separated by two + # blank lines. cppwg runs no formatter, so the layout built here is final. lines = [GENERATED_HEADER.rstrip("\n"), "", compiled_import] if templated_classes: lines.append(f"from {package}._syntax import TemplateClass") @@ -169,29 +173,20 @@ def _concrete_names(class_info: dict) -> list[str]: def _explicit_import(package: str, compiled_module: str, names: list[str]) -> str: - """Render an explicit ``from . import (...)`` statement.""" - body = "".join(f" {name},\n" for name in sorted(names)) - return f"from {package}.{compiled_module} import (\n{body})" - + """Render an import of ``names`` from the shared compiled extension. -def _format(content: str) -> str: - """Format with black when available, so output is stable and idempotent. - - Long instantiation keys (e.g. pychaste's CellsGenerator) exceed black's line - length and must be wrapped exactly as black would, or a project's - ``black --check`` / ``git diff`` reproducibility gate fails. When black is - not installed the content is written as-is (still valid Python). + With no names (an empty subpackage, or one whose names are all absent from + the model) a ``from ... import ()`` list would be empty and invalid Python, + so import the extension module itself instead, keeping the file valid. """ - try: - import black - except ImportError: # pragma: no cover - only when black is absent - return content - return black.format_str(content, mode=black.Mode()) + if not names: + return f"import {package}.{compiled_module} # noqa: F401" + body = "".join(f" {name},\n" for name in sorted(names)) + return f"from {package}.{compiled_module} import (\n{body})" def _write_generated(path: str, content: str, overwrite: bool) -> None: os.makedirs(os.path.dirname(path), exist_ok=True) - content = _format(content) if write_file_if_changed(path, content, overwrite): print(f"wrote {path}") else: diff --git a/cppwg/info/base_info.py b/cppwg/info/base_info.py index 8321b51..4f2bbeb 100644 --- a/cppwg/info/base_info.py +++ b/cppwg/info/base_info.py @@ -20,7 +20,7 @@ # (cppwg.parsers.package_info_parser) builds its config dicts from the same # schema. An option added here is therefore understood everywhere - defined in # one place instead of being restated in BaseInfo and the parser (which can -# cause options to became unreachable from the YAML). Mutable defaults are +# cause options to become unreachable from the YAML). Mutable defaults are # deep-copied per use so no two objects share a list/dict. See the class # Attributes docstring for what each option means; tri-state options default to # None, meaning "inherit from further up the info tree". diff --git a/tests/test_genpackage.py b/tests/test_genpackage.py index 4e13d6c..371f348 100644 --- a/tests/test_genpackage.py +++ b/tests/test_genpackage.py @@ -1,5 +1,6 @@ """Unit tests for cppwg.genpackage (the `cppwg genpackage` subcommand).""" +import ast import json import sys @@ -407,3 +408,42 @@ def test_flatten_skips_subpackage_with_no_exports(tmp_path): assert "from pkg.a import (" in root # a exports a name assert "from pkg.b import (" not in root # b exports nothing -> skipped assert '"Widget",' in root # __all__ + + +def test_key_repr_escapes_special_chars(): + """A value with special characters is escaped into a valid Python literal.""" + rendered = genpackage._key_repr(['a"b', "c"]) + # Round-trips: the emitted tuple literal is valid Python, not `("a"b",...`. + assert ast.literal_eval(rendered) == ('a"b', "c") + + +def test_shared_split_empty_subpackage_emits_valid_import(tmp_path): + """An empty subpackage imports the extension module, not `from ... import ()`.""" + model = { + "package": "pkg", + "modules": [ + { + "name": "all", + "compiled_module": "_pkg_all", + "imports": [], + "enums": [], + "free_functions": [], + "classes": [_class("Kept", [_inst([], "Kept")], templated=False)], + } + ], + } + layout = { + "package": "pkg", + "package_root": str(tmp_path), + "compiled_module": "_pkg_all", + "subpackages": {"a": ["Kept"], "b": []}, # b owns nothing + } + + genpackage.generate_shared_module_split(model, layout, overwrite=False) + + # b would otherwise be `from pkg._pkg_all import (\n)`, a SyntaxError; it + # must be a plain, valid module import instead. + b = (tmp_path / "b" / "_generated.py").read_text() + assert "import pkg._pkg_all" in b + assert "import (" not in b + ast.parse(b) # the whole file parses From 68dafbb87f1774685188ec9e425cf4c57ee2a5b8 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 16:16:31 +0100 Subject: [PATCH 15/24] #102 Fix nested template args for classes with a name_override _build_cxx_index reconstructed a class's C++ type as base using the Python (overridden) name, so a class with a name_override used as a nested template argument (e.g. Factory[NewName[2]]) failed to resolve and raised KeyError. Carry each instantiation's real C++ type name (cxx_name) in the package model and key on it, falling back to the reconstruction for older models. +2 tests. Co-Authored-By: Claude Opus 4.8 --- cppwg/genpackage.py | 9 ++++- cppwg/utils/package_model.py | 18 +++++++--- tests/test_genpackage.py | 27 ++++++++++++-- tests/test_package_model.py | 68 +++++++++++++++++++++++++++++++----- 4 files changed, 106 insertions(+), 16 deletions(-) diff --git a/cppwg/genpackage.py b/cppwg/genpackage.py index bdae9e9..b422b3d 100644 --- a/cppwg/genpackage.py +++ b/cppwg/genpackage.py @@ -78,13 +78,20 @@ def _build_cxx_index(model: dict) -> dict: itself a wrapped templated type (``MeshFactory>``) by the Python concrete class name, so ``MeshFactory[PottsMesh[2]]`` resolves: ``PottsMesh[2]`` is the ``PottsMesh_2`` class and ``_normalize_key`` keys it by its ``__name__``. + + Keyed by the instantiation's real C++ type name (``cxx_name``) so a class with + a name_override (whose Python name differs from its C++ name) still resolves. + Older models without ``cxx_name`` fall back to reconstructing it from the base + name and arguments. """ index = {} for module in model["modules"]: for class_info in module["classes"]: for inst in class_info["instantiations"]: if inst["args"]: - cxx = f'{class_info["base"]}<{",".join(inst["args"])}>' + cxx = inst.get("cxx_name") or ( + f'{class_info["base"]}<{",".join(inst["args"])}>' + ) index[cxx] = inst["py_name"] return index diff --git a/cppwg/utils/package_model.py b/cppwg/utils/package_model.py index e54bb86..e676dc8 100644 --- a/cppwg/utils/package_model.py +++ b/cppwg/utils/package_model.py @@ -43,7 +43,8 @@ def build_package_model(package_info: "PackageInfo") -> dict[str, Any]: "classes": [{"base", "templated", "instantiations": [{"args", "py_name"}]}], "enums": [...], "free_functions": [...]}]}``. Excluded entities are omitted (they are not wrapped). An untemplated class has ``templated: - false`` and a single instantiation with empty ``args``. + false`` and a single instantiation with empty ``args``; a templated + instantiation additionally carries ``cxx_name`` (its C++ type name). """ modules = [] for module in package_info.module_collection: @@ -53,10 +54,19 @@ def build_package_model(package_info: "PackageInfo") -> dict[str, Any]: continue if class_info.template_arg_lists: + # Carry each instantiation's actual C++ type name (cxx_name), which + # differs from base when the class has a name_override; the + # package-layer generator keys nested template arguments by it. instantiations = [ - {"args": [str(arg) for arg in args], "py_name": py_name} - for args, py_name in zip( - class_info.template_arg_lists, class_info.py_names + { + "args": [str(arg) for arg in args], + "cxx_name": cxx_name, + "py_name": py_name, + } + for args, cxx_name, py_name in zip( + class_info.template_arg_lists, + class_info.cpp_names, + class_info.py_names, ) ] templated = True diff --git a/tests/test_genpackage.py b/tests/test_genpackage.py index 371f348..ec31aab 100644 --- a/tests/test_genpackage.py +++ b/tests/test_genpackage.py @@ -14,8 +14,11 @@ def _class(base, instantiations, templated=True): return {"base": base, "templated": templated, "instantiations": instantiations} -def _inst(args, py_name): - return {"args": list(args), "py_name": py_name} +def _inst(args, py_name, cxx_name=None): + inst = {"args": list(args), "py_name": py_name} + if cxx_name is not None: + inst["cxx_name"] = cxx_name + return inst def test_key_repr_singleton_and_multi(): @@ -87,6 +90,26 @@ def test_build_cxx_index(): assert index == {"PottsMesh<2>": "PottsMesh_2"} # untemplated Cell has no <...> +def test_build_cxx_index_uses_cxx_name_for_name_override(): + """A class whose C++ name differs from its py name is keyed by the C++ name.""" + model = { + "modules": [ + { + "classes": [ + _class( + "NewName", + [_inst(["2"], "NewName_2", cxx_name="OldName<2>")], + ), + ] + } + ] + } + index = genpackage._build_cxx_index(model) + # Keyed by OldName<2> (the real C++ type), not NewName<2>, so a nested + # OldName<2> argument resolves to NewName_2. + assert index == {"OldName<2>": "NewName_2"} + + def test_render_generated_module_with_stub_imports_syntax(): point = _class("Point", [_inst(["2"], "Point_2"), _inst(["3"], "Point_3")]) content = genpackage.render_generated_module( diff --git a/tests/test_package_model.py b/tests/test_package_model.py index ce702a1..9f6765b 100644 --- a/tests/test_package_model.py +++ b/tests/test_package_model.py @@ -5,10 +5,11 @@ from cppwg.utils.package_model import build_package_model, compiled_module_name -def _class(base, py_names, template_arg_lists=(), excluded=False): +def _class(base, py_names, template_arg_lists=(), cpp_names=None, excluded=False): return SimpleNamespace( excluded=excluded, py_names=list(py_names), + cpp_names=list(cpp_names if cpp_names is not None else py_names), template_arg_lists=[list(a) for a in template_arg_lists], py_name_base=lambda base=base: base, ) @@ -47,12 +48,24 @@ def test_build_model_templated_and_untemplated(): [ _module( "geometry", - classes=[_class("Point", ["Point_2", "Point_3"], [[2], [3]])], + classes=[ + _class( + "Point", + ["Point_2", "Point_3"], + [[2], [3]], + cpp_names=["Point<2>", "Point<3>"], + ) + ], ), _module( "primitives", classes=[ - _class("Shape", ["Shape_2", "Shape_3"], [[2], [3]]), + _class( + "Shape", + ["Shape_2", "Shape_3"], + [[2], [3]], + cpp_names=["Shape<2>", "Shape<3>"], + ), _class("UnitSquare", ["UnitSquare"]), # untemplated ], enums=[_enum("ShapeKind")], @@ -72,8 +85,8 @@ def test_build_model_templated_and_untemplated(): "base": "Point", "templated": True, "instantiations": [ - {"args": ["2"], "py_name": "Point_2"}, - {"args": ["3"], "py_name": "Point_3"}, + {"args": ["2"], "cxx_name": "Point<2>", "py_name": "Point_2"}, + {"args": ["3"], "cxx_name": "Point<3>", "py_name": "Point_3"}, ], } @@ -123,8 +136,18 @@ def test_build_model_multi_arg_and_class_arg_keys(): _module( "all", classes=[ - _class("MacroMesh", ["MacroMesh_2_2"], [[2, 2]]), - _class("CellFactory", ["CellFactory_Cell_2"], [["Cell", 2]]), + _class( + "MacroMesh", + ["MacroMesh_2_2"], + [[2, 2]], + cpp_names=["MacroMesh<2, 2>"], + ), + _class( + "CellFactory", + ["CellFactory_Cell_2"], + [["Cell", 2]], + cpp_names=["CellFactory"], + ), ], ) ], @@ -132,9 +155,36 @@ def test_build_model_multi_arg_and_class_arg_keys(): (module,) = build_package_model(package)["modules"] macro, factory = module["classes"] - assert macro["instantiations"] == [{"args": ["2", "2"], "py_name": "MacroMesh_2_2"}] + assert macro["instantiations"] == [ + {"args": ["2", "2"], "cxx_name": "MacroMesh<2, 2>", "py_name": "MacroMesh_2_2"} + ] assert factory["instantiations"] == [ - {"args": ["Cell", "2"], "py_name": "CellFactory_Cell_2"} + { + "args": ["Cell", "2"], + "cxx_name": "CellFactory", + "py_name": "CellFactory_Cell_2", + } + ] + + +def test_instantiation_carries_cxx_name_for_name_override(): + """cxx_name is the C++ type name (from cpp_names), not the overridden py name.""" + package = _package( + "pkg", + [ + _module( + "mod", + classes=[ + _class("NewName", ["NewName_2"], [[2]], cpp_names=["OldName<2>"]), + ], + ) + ], + ) + + (module,) = build_package_model(package)["modules"] + (cls,) = module["classes"] + assert cls["instantiations"] == [ + {"args": ["2"], "cxx_name": "OldName<2>", "py_name": "NewName_2"} ] From 48860fd429b7efa64acd1154b73f79640fb9b7f8 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 16:17:06 +0100 Subject: [PATCH 16/24] #102 Emit the package model as JSON instead of YAML The package model is a machine-generated, git-diff-checked interchange artifact consumed by `cppwg genpackage`, so a deterministic serialization matters more than YAML's readability (the hand-authored *.yaml configs stay YAML, and YAML's implicit typing is a footgun for machine data). Write cppwg_package_model.json via json.dumps(sort_keys); the do-not-edit banner becomes a `_comment` field (JSON has no comments), which the generator ignores. genpackage reads it with json.load. Regenerated shapes/cells models. Co-Authored-By: Claude Opus 4.8 --- cppwg/generators.py | 21 +- cppwg/genpackage.py | 8 +- cppwg/utils/constants.py | 19 +- cppwg/utils/package_model.py | 2 +- doc/python-packages.md | 6 +- .../dynamic/wrappers/cppwg_package_model.json | 251 ++++++++++++++++++ .../dynamic/wrappers/cppwg_package_model.yaml | 123 --------- .../shapes/wrapper/cppwg_package_model.json | 232 ++++++++++++++++ .../shapes/wrapper/cppwg_package_model.yaml | 120 --------- 9 files changed, 513 insertions(+), 269 deletions(-) create mode 100644 examples/cells/dynamic/wrappers/cppwg_package_model.json delete mode 100644 examples/cells/dynamic/wrappers/cppwg_package_model.yaml create mode 100644 examples/shapes/wrapper/cppwg_package_model.json delete mode 100644 examples/shapes/wrapper/cppwg_package_model.yaml diff --git a/cppwg/generators.py b/cppwg/generators.py index e11f3c1..eff18f9 100644 --- a/cppwg/generators.py +++ b/cppwg/generators.py @@ -1,5 +1,6 @@ """The main interface for generating Python wrappers.""" +import json import logging import os import re @@ -9,7 +10,6 @@ from pathlib import Path import pygccxml -import yaml from cppwg.info.package_info import PackageInfo from cppwg.parsers.package_info_parser import PackageInfoParser @@ -20,7 +20,7 @@ CPPWG_DEFAULT_WRAPPER_DIR, CPPWG_HEADER_COLLECTION_FILENAME, CPPWG_PACKAGE_MODEL_FILENAME, - CPPWG_PACKAGE_MODEL_HEADER, + CPPWG_PACKAGE_MODEL_NOTICE, ) from cppwg.utils.package_model import build_package_model from cppwg.version import __version__ as cppwg_version @@ -428,19 +428,22 @@ def write_wrappers(self) -> None: def write_package_model(self) -> None: """ - Write the package model (cppwg_package_model.yaml) to the wrapper root. + Write the package model (cppwg_package_model.json) to the wrapper root. - A small YAML description of the generated modules and their classes / + A small JSON description of the generated modules and their classes / instantiations / enums / free functions, so a separate step (the ``cppwg genpackage`` subcommand) can generate the Python package layer without re-parsing the source. Written last, once the info tree is final. + JSON is used deliberately: this is a machine-generated, git-diff-checked + artifact, so a deterministic serialization matters more than YAML's + readability (the hand-authored configs stay YAML). """ model = build_package_model(self.package_info) + # A do-not-edit notice as `_comment` (JSON has no comments); the generator + # ignores unknown top-level keys. + model = {"_comment": CPPWG_PACKAGE_MODEL_NOTICE, **model} model_path = os.path.join(self.wrapper_root, CPPWG_PACKAGE_MODEL_FILENAME) - # Prepend a do-not-edit banner (YAML comments, ignored on load). - content = CPPWG_PACKAGE_MODEL_HEADER + yaml.safe_dump( - model, default_flow_style=False, sort_keys=True - ) + content = json.dumps(model, indent=2, sort_keys=True) + "\n" utils.write_file_if_changed(model_path, content, self.overwrite) def generate(self) -> None: @@ -504,6 +507,6 @@ def generate(self) -> None: # Write the wrapper code for the package self.write_wrappers() - # Write the package model (cppwg_package_model.yaml) for the package-layer + # Write the package model (cppwg_package_model.json) for the package-layer # generator (tools/cppwg_genpackage.py). self.write_package_model() diff --git a/cppwg/genpackage.py b/cppwg/genpackage.py index b422b3d..36bf015 100644 --- a/cppwg/genpackage.py +++ b/cppwg/genpackage.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """Generate the Python package layer (``_generated.py``) for a cppwg project. -cppwg generates the C++/pybind11 wrappers and, alongside them, a YAML model of -what it produced (``cppwg_package_model.yaml`` in the wrapper root - see +cppwg generates the C++/pybind11 wrappers and, alongside them, a JSON model of +what it produced (``cppwg_package_model.json`` in the wrapper root - see ``cppwg.utils.package_model``). This module (run via ``cppwg genpackage``) turns that model, plus a small package-layout file, into a ``_generated.py`` per Python subpackage: the @@ -328,7 +328,7 @@ def main(argv=None) -> int: """Entry point: read the model + layout and write the _generated.py files.""" parser = argparse.ArgumentParser(prog="cppwg genpackage", description=__doc__) parser.add_argument( - "--model", required=True, help="Path to cppwg_package_model.yaml (from cppwg)." + "--model", required=True, help="Path to cppwg_package_model.json (from cppwg)." ) parser.add_argument( "--layout", required=True, help="Path to the Python package-layout file (YAML)." @@ -343,7 +343,7 @@ def main(argv=None) -> int: import yaml with open(args.model) as model_file: - model = yaml.safe_load(model_file) + model = json.load(model_file) # the model is JSON (see cppwg.utils.constants) with open(args.layout) as layout_file: layout = yaml.safe_load(layout_file) diff --git a/cppwg/utils/constants.py b/cppwg/utils/constants.py index a2a2f88..18a190f 100644 --- a/cppwg/utils/constants.py +++ b/cppwg/utils/constants.py @@ -11,15 +11,16 @@ # The package model cppwg writes into the wrapper root, describing the generated # modules/classes so a separate step can build the Python package layer (see -# cppwg.utils.package_model and tools/cppwg_genpackage.py). -CPPWG_PACKAGE_MODEL_FILENAME = "cppwg_package_model.yaml" - -# Do-not-edit banner prepended to the emitted package model (YAML comments, -# ignored on load), mirroring the banner on the generated wrapper sources so the -# committed model reads clearly as a generated artifact. -CPPWG_PACKAGE_MODEL_HEADER = ( - "# This file is automatically generated by cppwg.\n" - "# Do not modify this file directly.\n" +# cppwg.utils.package_model and tools/cppwg_genpackage.py). It is JSON: a +# machine-generated, git-diff-checked artifact whose serialization is +# deterministic across environments (unlike the hand-authored *.yaml configs). +CPPWG_PACKAGE_MODEL_FILENAME = "cppwg_package_model.json" + +# Do-not-edit notice stored as the model's `_comment` field (JSON has no comment +# syntax; the generator ignores the field), mirroring the banner on the generated +# wrapper sources so the committed model reads clearly as a generated artifact. +CPPWG_PACKAGE_MODEL_NOTICE = ( + "This file is automatically generated by cppwg. Do not modify this file directly." ) CPPWG_TRUE_STRINGS = ["ON", "YES", "Y", "TRUE", "T", "1"] diff --git a/cppwg/utils/package_model.py b/cppwg/utils/package_model.py index e676dc8..fa485e9 100644 --- a/cppwg/utils/package_model.py +++ b/cppwg/utils/package_model.py @@ -7,7 +7,7 @@ module name, the wrapped classes with their template instantiations, and the enum / free-function names. All of this is on the finalized ``PackageInfo`` tree but not in a form a standalone script can consume, so ``build_package_model`` -distils it into a plain dict that cppwg writes out as ``cppwg_package_model.yaml``. +distils it into a plain dict that cppwg writes out as ``cppwg_package_model.json``. """ from typing import TYPE_CHECKING, Any diff --git a/doc/python-packages.md b/doc/python-packages.md index 799152f..09b91ec 100644 --- a/doc/python-packages.md +++ b/doc/python-packages.md @@ -15,7 +15,7 @@ from the same config that produced the wrappers, so they stay in step. Generation is a two-step flow: 1. When cppwg generates the wrappers it also writes a **package model**, - `cppwg_package_model.yaml`. This is a small YAML description of what was + `cppwg_package_model.json`. This is a small JSON description of what was wrapped — each module, its compiled extension, and the wrapped classes (with their template instantiations), enums and free functions. @@ -80,7 +80,7 @@ Run: ```bash cppwg genpackage \ - --model wrapper/cppwg_package_model.yaml \ + --model wrapper/cppwg_package_model.json \ --layout wrapper/package_layout.yaml ``` @@ -280,7 +280,7 @@ usage: cppwg genpackage [-h] --model MODEL --layout LAYOUT [--overwrite] options: -h, --help show this help message and exit - --model MODEL Path to cppwg_package_model.yaml (from cppwg). + --model MODEL Path to cppwg_package_model.json (from cppwg). --layout LAYOUT Path to the Python package-layout file (YAML). --overwrite Rewrite files even if unchanged. ``` diff --git a/examples/cells/dynamic/wrappers/cppwg_package_model.json b/examples/cells/dynamic/wrappers/cppwg_package_model.json new file mode 100644 index 0000000..96a67a0 --- /dev/null +++ b/examples/cells/dynamic/wrappers/cppwg_package_model.json @@ -0,0 +1,251 @@ +{ + "_comment": "This file is automatically generated by cppwg. Do not modify this file directly.", + "modules": [ + { + "classes": [ + { + "base": "Cell", + "instantiations": [ + { + "args": [], + "py_name": "Cell" + } + ], + "templated": false + }, + { + "base": "CellFactory", + "instantiations": [ + { + "args": [ + "Cell", + "2" + ], + "cxx_name": "CellFactory", + "py_name": "CellFactory_Cell_2" + }, + { + "args": [ + "Cell", + "3" + ], + "cxx_name": "CellFactory", + "py_name": "CellFactory_Cell_3" + } + ], + "templated": true + }, + { + "base": "Corner", + "instantiations": [ + { + "args": [ + "2" + ], + "cxx_name": "Corner<2>", + "py_name": "Corner_2" + } + ], + "templated": true + }, + { + "base": "Facet", + "instantiations": [ + { + "args": [ + "2" + ], + "cxx_name": "Facet<2>", + "py_name": "Facet_2" + } + ], + "templated": true + }, + { + "base": "MacroMesh", + "instantiations": [ + { + "args": [ + "2", + "2" + ], + "cxx_name": "MacroMesh<2, 2>", + "py_name": "MacroMesh_2_2" + }, + { + "args": [ + "3", + "3" + ], + "cxx_name": "MacroMesh<3, 3>", + "py_name": "MacroMesh_3_3" + } + ], + "templated": true + }, + { + "base": "Node", + "instantiations": [ + { + "args": [ + "2" + ], + "cxx_name": "Node<2>", + "py_name": "Node_2" + }, + { + "args": [ + "3" + ], + "cxx_name": "Node<3>", + "py_name": "Node_3" + } + ], + "templated": true + }, + { + "base": "AbstractMesh", + "instantiations": [ + { + "args": [ + "2", + "2" + ], + "cxx_name": "AbstractMesh<2, 2>", + "py_name": "AbstractMesh_2_2" + }, + { + "args": [ + "3", + "3" + ], + "cxx_name": "AbstractMesh<3, 3>", + "py_name": "AbstractMesh_3_3" + } + ], + "templated": true + }, + { + "base": "AbstractSphericalMesh", + "instantiations": [ + { + "args": [ + "2", + "2" + ], + "cxx_name": "AbstractSphericalMesh<2, 2>", + "py_name": "AbstractSphericalMesh_2_2" + }, + { + "args": [ + "3", + "3" + ], + "cxx_name": "AbstractSphericalMesh<3, 3>", + "py_name": "AbstractSphericalMesh_3_3" + } + ], + "templated": true + }, + { + "base": "PetscUtils", + "instantiations": [ + { + "args": [], + "py_name": "PetscUtils" + } + ], + "templated": false + }, + { + "base": "PottsMesh", + "instantiations": [ + { + "args": [ + "2" + ], + "cxx_name": "PottsMesh<2>", + "py_name": "PottsMesh_2" + }, + { + "args": [ + "3" + ], + "cxx_name": "PottsMesh<3>", + "py_name": "PottsMesh_3" + } + ], + "templated": true + }, + { + "base": "MeshFactory", + "instantiations": [ + { + "args": [ + "PottsMesh<2>" + ], + "cxx_name": "MeshFactory>", + "py_name": "MeshFactory_PottsMesh_2" + }, + { + "args": [ + "PottsMesh<3>" + ], + "cxx_name": "MeshFactory>", + "py_name": "MeshFactory_PottsMesh_3" + } + ], + "templated": true + }, + { + "base": "Scene", + "instantiations": [ + { + "args": [ + "2" + ], + "cxx_name": "Scene<2>", + "py_name": "Scene_2" + }, + { + "args": [ + "3" + ], + "cxx_name": "Scene<3>", + "py_name": "Scene_3" + } + ], + "templated": true + }, + { + "base": "SphericalMesh", + "instantiations": [ + { + "args": [ + "2", + "2" + ], + "cxx_name": "SphericalMesh<2, 2>", + "py_name": "SphericalMesh_2_2" + }, + { + "args": [ + "3", + "3" + ], + "cxx_name": "SphericalMesh<3, 3>", + "py_name": "SphericalMesh_3_3" + } + ], + "templated": true + } + ], + "compiled_module": "_pycells_all", + "enums": [], + "free_functions": [], + "imports": [], + "name": "all" + } + ], + "package": "pycells" +} diff --git a/examples/cells/dynamic/wrappers/cppwg_package_model.yaml b/examples/cells/dynamic/wrappers/cppwg_package_model.yaml deleted file mode 100644 index 9594841..0000000 --- a/examples/cells/dynamic/wrappers/cppwg_package_model.yaml +++ /dev/null @@ -1,123 +0,0 @@ -# This file is automatically generated by cppwg. -# Do not modify this file directly. -modules: -- classes: - - base: Cell - instantiations: - - args: [] - py_name: Cell - templated: false - - base: CellFactory - instantiations: - - args: - - Cell - - '2' - py_name: CellFactory_Cell_2 - - args: - - Cell - - '3' - py_name: CellFactory_Cell_3 - templated: true - - base: Corner - instantiations: - - args: - - '2' - py_name: Corner_2 - templated: true - - base: Facet - instantiations: - - args: - - '2' - py_name: Facet_2 - templated: true - - base: MacroMesh - instantiations: - - args: - - '2' - - '2' - py_name: MacroMesh_2_2 - - args: - - '3' - - '3' - py_name: MacroMesh_3_3 - templated: true - - base: Node - instantiations: - - args: - - '2' - py_name: Node_2 - - args: - - '3' - py_name: Node_3 - templated: true - - base: AbstractMesh - instantiations: - - args: - - '2' - - '2' - py_name: AbstractMesh_2_2 - - args: - - '3' - - '3' - py_name: AbstractMesh_3_3 - templated: true - - base: AbstractSphericalMesh - instantiations: - - args: - - '2' - - '2' - py_name: AbstractSphericalMesh_2_2 - - args: - - '3' - - '3' - py_name: AbstractSphericalMesh_3_3 - templated: true - - base: PetscUtils - instantiations: - - args: [] - py_name: PetscUtils - templated: false - - base: PottsMesh - instantiations: - - args: - - '2' - py_name: PottsMesh_2 - - args: - - '3' - py_name: PottsMesh_3 - templated: true - - base: MeshFactory - instantiations: - - args: - - PottsMesh<2> - py_name: MeshFactory_PottsMesh_2 - - args: - - PottsMesh<3> - py_name: MeshFactory_PottsMesh_3 - templated: true - - base: Scene - instantiations: - - args: - - '2' - py_name: Scene_2 - - args: - - '3' - py_name: Scene_3 - templated: true - - base: SphericalMesh - instantiations: - - args: - - '2' - - '2' - py_name: SphericalMesh_2_2 - - args: - - '3' - - '3' - py_name: SphericalMesh_3_3 - templated: true - compiled_module: _pycells_all - enums: [] - free_functions: [] - imports: [] - name: all -package: pycells diff --git a/examples/shapes/wrapper/cppwg_package_model.json b/examples/shapes/wrapper/cppwg_package_model.json new file mode 100644 index 0000000..77219f0 --- /dev/null +++ b/examples/shapes/wrapper/cppwg_package_model.json @@ -0,0 +1,232 @@ +{ + "_comment": "This file is automatically generated by cppwg. Do not modify this file directly.", + "modules": [ + { + "classes": [], + "compiled_module": "_pyshapes_math_funcs", + "enums": [], + "free_functions": [ + "add", + "throw_exception", + "throw_unwrapped_exception" + ], + "imports": [], + "name": "math_funcs" + }, + { + "classes": [ + { + "base": "Point", + "instantiations": [ + { + "args": [ + "2" + ], + "cxx_name": "Point<2>", + "py_name": "Point_2" + }, + { + "args": [ + "3" + ], + "cxx_name": "Point<3>", + "py_name": "Point_3" + } + ], + "templated": true + } + ], + "compiled_module": "_pyshapes_geometry", + "enums": [], + "free_functions": [], + "imports": [], + "name": "geometry" + }, + { + "classes": [ + { + "base": "AbstractShape", + "instantiations": [ + { + "args": [ + "2" + ], + "cxx_name": "AbstractShape<2>", + "py_name": "AbstractShape_2" + }, + { + "args": [ + "3" + ], + "cxx_name": "AbstractShape<3>", + "py_name": "AbstractShape_3" + } + ], + "templated": true + }, + { + "base": "AbstractPolygon", + "instantiations": [ + { + "args": [ + "2" + ], + "cxx_name": "AbstractPolygon<2>", + "py_name": "AbstractPolygon_2" + }, + { + "args": [ + "3" + ], + "cxx_name": "AbstractPolygon<3>", + "py_name": "AbstractPolygon_3" + } + ], + "templated": true + }, + { + "base": "RegularPolygon", + "instantiations": [ + { + "args": [ + "2" + ], + "cxx_name": "RegularPolygon<2>", + "py_name": "RegularPolygon_2" + }, + { + "args": [ + "3" + ], + "cxx_name": "RegularPolygon<3>", + "py_name": "RegularPolygon_3" + } + ], + "templated": true + }, + { + "base": "Shape", + "instantiations": [ + { + "args": [ + "2" + ], + "cxx_name": "Shape<2>", + "py_name": "Shape_2" + }, + { + "args": [ + "3" + ], + "cxx_name": "Shape<3>", + "py_name": "Shape_3" + } + ], + "templated": true + }, + { + "base": "Cuboid", + "instantiations": [ + { + "args": [], + "py_name": "Cuboid" + } + ], + "templated": false + }, + { + "base": "Rectangle", + "instantiations": [ + { + "args": [], + "py_name": "Rectangle" + } + ], + "templated": false + }, + { + "base": "ShapeClassifier", + "instantiations": [ + { + "args": [], + "py_name": "ShapeClassifier" + } + ], + "templated": false + }, + { + "base": "ShapeMetrics", + "instantiations": [ + { + "args": [], + "py_name": "ShapeMetrics" + } + ], + "templated": false + }, + { + "base": "SquareFeet", + "instantiations": [ + { + "args": [], + "py_name": "SquareFeet" + } + ], + "templated": false + }, + { + "base": "SquareMetres", + "instantiations": [ + { + "args": [], + "py_name": "SquareMetres" + } + ], + "templated": false + }, + { + "base": "UnitSquare", + "instantiations": [ + { + "args": [], + "py_name": "UnitSquare" + } + ], + "templated": false + } + ], + "compiled_module": "_pyshapes_primitives", + "enums": [ + "Handedness", + "ShapeKind" + ], + "free_functions": [], + "imports": [ + "pyshapes.geometry._pyshapes_geometry" + ], + "name": "primitives" + }, + { + "classes": [ + { + "base": "Square", + "instantiations": [ + { + "args": [], + "py_name": "Square" + } + ], + "templated": false + } + ], + "compiled_module": "_pyshapes_composites", + "enums": [], + "free_functions": [], + "imports": [ + "pyshapes.primitives._pyshapes_primitives" + ], + "name": "composites" + } + ], + "package": "pyshapes" +} diff --git a/examples/shapes/wrapper/cppwg_package_model.yaml b/examples/shapes/wrapper/cppwg_package_model.yaml deleted file mode 100644 index a626951..0000000 --- a/examples/shapes/wrapper/cppwg_package_model.yaml +++ /dev/null @@ -1,120 +0,0 @@ -# This file is automatically generated by cppwg. -# Do not modify this file directly. -modules: -- classes: [] - compiled_module: _pyshapes_math_funcs - enums: [] - free_functions: - - add - - throw_exception - - throw_unwrapped_exception - imports: [] - name: math_funcs -- classes: - - base: Point - instantiations: - - args: - - '2' - py_name: Point_2 - - args: - - '3' - py_name: Point_3 - templated: true - compiled_module: _pyshapes_geometry - enums: [] - free_functions: [] - imports: [] - name: geometry -- classes: - - base: AbstractShape - instantiations: - - args: - - '2' - py_name: AbstractShape_2 - - args: - - '3' - py_name: AbstractShape_3 - templated: true - - base: AbstractPolygon - instantiations: - - args: - - '2' - py_name: AbstractPolygon_2 - - args: - - '3' - py_name: AbstractPolygon_3 - templated: true - - base: RegularPolygon - instantiations: - - args: - - '2' - py_name: RegularPolygon_2 - - args: - - '3' - py_name: RegularPolygon_3 - templated: true - - base: Shape - instantiations: - - args: - - '2' - py_name: Shape_2 - - args: - - '3' - py_name: Shape_3 - templated: true - - base: Cuboid - instantiations: - - args: [] - py_name: Cuboid - templated: false - - base: Rectangle - instantiations: - - args: [] - py_name: Rectangle - templated: false - - base: ShapeClassifier - instantiations: - - args: [] - py_name: ShapeClassifier - templated: false - - base: ShapeMetrics - instantiations: - - args: [] - py_name: ShapeMetrics - templated: false - - base: SquareFeet - instantiations: - - args: [] - py_name: SquareFeet - templated: false - - base: SquareMetres - instantiations: - - args: [] - py_name: SquareMetres - templated: false - - base: UnitSquare - instantiations: - - args: [] - py_name: UnitSquare - templated: false - compiled_module: _pyshapes_primitives - enums: - - Handedness - - ShapeKind - free_functions: [] - imports: - - pyshapes.geometry._pyshapes_geometry - name: primitives -- classes: - - base: Square - instantiations: - - args: [] - py_name: Square - templated: false - compiled_module: _pyshapes_composites - enums: [] - free_functions: [] - imports: - - pyshapes.primitives._pyshapes_primitives - name: composites -package: pyshapes From 1ed9cd39ea5242774293fe553a9e73b1bb5da9a0 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 16:36:03 +0100 Subject: [PATCH 17/24] #102 Warn about orphaned _generated.py files When a module or subpackage is removed from the config, its previously generated _generated.py is left behind and can keep exporting removed bindings. After generating, walk the package root and warn about any _generated.py that carries the do-not-edit banner but was not written this run. cppwg never deletes files (it does not own the hand-written __init__.py), so it flags the orphan for the user to remove the containing directory rather than silently leaving stale bindings. +1 test. Co-Authored-By: Claude Opus 4.8 --- cppwg/genpackage.py | 67 +++++++++++++++++++++++++++++++++------- tests/test_genpackage.py | 24 ++++++++++++++ 2 files changed, 79 insertions(+), 12 deletions(-) diff --git a/cppwg/genpackage.py b/cppwg/genpackage.py index 36bf015..88b1dce 100644 --- a/cppwg/genpackage.py +++ b/cppwg/genpackage.py @@ -192,22 +192,27 @@ def _explicit_import(package: str, compiled_module: str, names: list[str]) -> st return f"from {package}.{compiled_module} import (\n{body})" -def _write_generated(path: str, content: str, overwrite: bool) -> None: +def _write_generated(path: str, content: str, overwrite: bool) -> str: os.makedirs(os.path.dirname(path), exist_ok=True) if write_file_if_changed(path, content, overwrite): print(f"wrote {path}") else: print(f"unchanged {path}") + return os.path.abspath(path) -def generate_module_per_subpackage(model: dict, layout: dict, overwrite: bool): - """Emit one ``_generated.py`` per cppwg module (each owns its own extension).""" +def generate_module_per_subpackage(model: dict, layout: dict, overwrite: bool) -> list: + """Emit one ``_generated.py`` per cppwg module (each owns its own extension). + + Returns the list of files written, so the caller can flag orphaned outputs. + """ package = layout.get("package", model["package"]) package_root = layout["package_root"] module_dirs = layout.get("module_dirs", {}) diagonal_shorthand = layout.get("diagonal_shorthand", False) cxx_to_pyname = _build_cxx_index(model) + written = [] for module in model["modules"]: # The subpackage directory (relative to package_root); default = module # name, overridable (e.g. a single "all" module living at the root). @@ -223,16 +228,21 @@ def generate_module_per_subpackage(model: dict, layout: dict, overwrite: bool): cxx_to_pyname, ) path = os.path.join(package_root, subdir, "_generated.py") - _write_generated(path, content, overwrite) + written.append(_write_generated(path, content, overwrite)) + return written + +def generate_shared_module_split(model: dict, layout: dict, overwrite: bool) -> list: + """Emit a ``_generated.py`` per subpackage, splitting one shared extension. -def generate_shared_module_split(model: dict, layout: dict, overwrite: bool): - """Emit a ``_generated.py`` per subpackage, splitting one shared extension.""" + Returns the list of files written, so the caller can flag orphaned outputs. + """ package = layout.get("package", model["package"]) package_root = layout["package_root"] compiled_module = layout["compiled_module"] diagonal_shorthand = layout.get("diagonal_shorthand", False) cxx_to_pyname = _build_cxx_index(model) + written = [] # Index every wrapped entity across the model by name so a layout entry can # be matched to a class (with its instantiations), an enum or a free function. @@ -274,7 +284,7 @@ def generate_shared_module_split(model: dict, layout: dict, overwrite: bool): cxx_to_pyname, ) path = os.path.join(package_root, subpkg, "_generated.py") - _write_generated(path, content, overwrite) + written.append(_write_generated(path, content, overwrite)) flatten_names[subpkg] = exported # Flag any wrapped class not placed in a subpackage, so nothing is silently @@ -286,12 +296,15 @@ def generate_shared_module_split(model: dict, layout: dict, overwrite: bool): file=sys.stderr) if layout.get("flatten_to_root"): - generate_root_flatten(package, package_root, flatten_names, overwrite) + written.append( + generate_root_flatten(package, package_root, flatten_names, overwrite) + ) + return written def generate_root_flatten( package: str, package_root: str, flatten_names: dict, overwrite: bool -) -> None: +) -> str: """Emit a top-level ``_generated.py`` re-exporting every subpackage's names. Gives ``package.ClassName`` in addition to ``package.subpackage.ClassName`` @@ -321,7 +334,36 @@ def generate_root_flatten( lines.append(f' "{name}",') lines.append("]") content = "\n".join(lines) + "\n" - _write_generated(os.path.join(package_root, "_generated.py"), content, overwrite) + return _write_generated( + os.path.join(package_root, "_generated.py"), content, overwrite + ) + + +def _warn_orphans(package_root: str, written: list) -> None: + """Warn about generated files under ``package_root`` this run did not write. + + A ``_generated.py`` carrying the do-not-edit banner but no longer a target + means its module/subpackage was removed from the config. cppwg only ever + generates ``_generated.py`` (never the hand-written ``__init__.py``), so it + cannot fully remove the orphaned subpackage; it flags it for the user to + delete rather than silently leaving stale bindings or deleting their files. + """ + written = {os.path.abspath(p) for p in written} + banner = "Generated by tools/cppwg_genpackage.py" + for dirpath, _dirs, files in os.walk(package_root): + if "_generated.py" not in files: + continue + path = os.path.abspath(os.path.join(dirpath, "_generated.py")) + if path in written: + continue + with open(path) as generated_file: + if banner not in generated_file.read(200): + continue # not one of ours -> never touch it + print( + f"warning: {path} is generated but no longer a target; its " + "module/subpackage was removed - delete the containing directory", + file=sys.stderr, + ) def main(argv=None) -> int: @@ -356,9 +398,10 @@ def main(argv=None) -> int: ) if layout.get("subpackages"): - generate_shared_module_split(model, layout, args.overwrite) + written = generate_shared_module_split(model, layout, args.overwrite) else: - generate_module_per_subpackage(model, layout, args.overwrite) + written = generate_module_per_subpackage(model, layout, args.overwrite) + _warn_orphans(layout["package_root"], written) return 0 diff --git a/tests/test_genpackage.py b/tests/test_genpackage.py index ec31aab..a88f05e 100644 --- a/tests/test_genpackage.py +++ b/tests/test_genpackage.py @@ -470,3 +470,27 @@ def test_shared_split_empty_subpackage_emits_valid_import(tmp_path): assert "import pkg._pkg_all" in b assert "import (" not in b ast.parse(b) # the whole file parses + + +def test_warn_orphans_flags_only_unwritten_banner_files(tmp_path, capsys): + """A banner-owned _generated.py that was not written this run is flagged.""" + banner = '"""Generated by tools/cppwg_genpackage.py - do not edit."""\n' + # A live target (in the written set) -> not flagged. + (tmp_path / "live").mkdir() + live = tmp_path / "live" / "_generated.py" + live.write_text(banner) + # An orphan: banner-owned but not written this run -> flagged. + (tmp_path / "orphan").mkdir() + orphan = tmp_path / "orphan" / "_generated.py" + orphan.write_text(banner) + # A hand-written _generated.py with no banner -> never flagged. + (tmp_path / "foreign").mkdir() + foreign = tmp_path / "foreign" / "_generated.py" + foreign.write_text("x = 1\n") + + genpackage._warn_orphans(str(tmp_path), [str(live)]) + + err = capsys.readouterr().err + assert str(orphan) in err # orphan flagged + assert str(live) not in err # live target not flagged + assert str(foreign) not in err # non-banner file left alone From f9e80d41d90c8cc51579b52116c2800265c8da11 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 16:55:58 +0100 Subject: [PATCH 18/24] #102 Drop the redundant __main__ guard in genpackage cppwg/genpackage.py is a library module, entered via the `cppwg genpackage` subcommand (__main__ dispatches to cppwg.genpackage.main) and via the tools/cppwg_genpackage.py launcher (which has its own guard). Its `if __name__ == "__main__"` block therefore only enabled the undocumented `python -m cppwg.genpackage` path, so remove it as dead code. Co-Authored-By: Claude Opus 4.8 --- cppwg/genpackage.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cppwg/genpackage.py b/cppwg/genpackage.py index 88b1dce..525a0d3 100644 --- a/cppwg/genpackage.py +++ b/cppwg/genpackage.py @@ -403,7 +403,3 @@ def main(argv=None) -> int: written = generate_module_per_subpackage(model, layout, args.overwrite) _warn_orphans(layout["package_root"], written) return 0 - - -if __name__ == "__main__": - sys.exit(main()) From 8398b2e6b1575966f20da1eca1f5d6828ef0987d Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 17:04:52 +0100 Subject: [PATCH 19/24] #102 Apply the repo coverage config in the shapes/cells flags The shapes/cells flags run `coverage xml` from inside the example directories, where the repo's [tool.coverage.report] exclude_also is not found - so excluded lines (e.g. an `if __name__` guard) in cppwg modules those flags never import were reported as executable-but-missed. Merged with the unit flag (which does exclude them), codecov surfaced them as uncovered patch lines and failed the 100%-patch gate. Point both `coverage xml` steps at the repo pyproject.toml so the exclusions apply consistently across every flag. Verified locally: an `if __name__` line in an unimported module is reported as missed without --rcfile and excluded with it. Co-Authored-By: Claude Opus 4.8 --- .github/workflows/test-cells-conda.yml | 5 ++++- .github/workflows/test-shapes-pip.yml | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-cells-conda.yml b/.github/workflows/test-cells-conda.yml index e4a7dc3..c94e4c1 100644 --- a/.github/workflows/test-cells-conda.yml +++ b/.github/workflows/test-cells-conda.yml @@ -105,7 +105,10 @@ jobs: COVERAGE_FILE: ${{ github.workspace }}/.coverage run: | python3 -m coverage combine --rcfile=.cov_sub.rc - python3 -m coverage xml --rcfile=.cov_sub.rc + # Report with the repo's coverage config so [tool.coverage.report] + # exclude_also applies (.cov_sub.rc only sets [run]); otherwise excluded + # lines in cppwg modules this flag never imports are reported as missed. + python3 -m coverage xml --rcfile="$GITHUB_WORKSPACE/pyproject.toml" - name: Upload coverage to Codecov if: matrix.python-version == '3.12' diff --git a/.github/workflows/test-shapes-pip.yml b/.github/workflows/test-shapes-pip.yml index 018eb4e..80a3620 100644 --- a/.github/workflows/test-shapes-pip.yml +++ b/.github/workflows/test-shapes-pip.yml @@ -60,7 +60,11 @@ jobs: - name: Generate coverage report if: matrix.python-version == '3.12' - run: python3 -m coverage xml + # Use the repo's coverage config so [tool.coverage.report] exclude_also + # applies here too (this runs from examples/shapes, where it isn't found); + # otherwise excluded lines (e.g. `if __name__`) in cppwg modules this flag + # never imports are wrongly reported as missed, failing codecov's patch gate. + run: python3 -m coverage xml --rcfile="$GITHUB_WORKSPACE/pyproject.toml" working-directory: examples/shapes - name: Upload coverage to Codecov From 02b2801dc552e4fce94b645738601f840d56acac Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 17:20:00 +0100 Subject: [PATCH 20/24] #102 Remove the tools/ launcher; generate via the cppwg genpackage subcommand tools/cppwg_genpackage.py became redundant once the generator was exposed as the `cppwg genpackage` subcommand: it only enabled `python tools/cppwg_genpackage.py` (which the subcommand supersedes), it is not part of the installed package, and the generated-file banner pointed at it rather than the canonical command. Remove it and retarget the banner, the orphan-detection marker, and every doc/comment reference to `cppwg genpackage`. Regenerated the shapes/cells _generated.py with the new banner. Co-Authored-By: Claude Opus 4.8 --- cppwg/generators.py | 2 +- cppwg/genpackage.py | 6 +++--- cppwg/utils/constants.py | 2 +- cppwg/utils/package_model.py | 2 +- doc/python-packages.md | 4 ++-- examples/cells/dynamic/package_layout.yaml | 2 +- examples/cells/src/py/pycells/__init__.py | 2 +- examples/cells/src/py/pycells/_generated.py | 2 +- .../shapes/src/py/pyshapes/composites/__init__.py | 2 +- .../src/py/pyshapes/composites/_generated.py | 2 +- .../shapes/src/py/pyshapes/geometry/__init__.py | 2 +- .../shapes/src/py/pyshapes/geometry/_generated.py | 2 +- .../shapes/src/py/pyshapes/math_funcs/__init__.py | 2 +- .../src/py/pyshapes/math_funcs/_generated.py | 2 +- .../shapes/src/py/pyshapes/primitives/__init__.py | 2 +- .../src/py/pyshapes/primitives/_generated.py | 2 +- examples/shapes/wrapper/package_layout.yaml | 2 +- tests/test_genpackage.py | 4 ++-- tools/cppwg_genpackage.py | 15 --------------- 19 files changed, 22 insertions(+), 37 deletions(-) delete mode 100644 tools/cppwg_genpackage.py diff --git a/cppwg/generators.py b/cppwg/generators.py index eff18f9..80f94a0 100644 --- a/cppwg/generators.py +++ b/cppwg/generators.py @@ -508,5 +508,5 @@ def generate(self) -> None: self.write_wrappers() # Write the package model (cppwg_package_model.json) for the package-layer - # generator (tools/cppwg_genpackage.py). + # generator (cppwg genpackage). self.write_package_model() diff --git a/cppwg/genpackage.py b/cppwg/genpackage.py index 525a0d3..4c1b436 100644 --- a/cppwg/genpackage.py +++ b/cppwg/genpackage.py @@ -35,7 +35,7 @@ GENERATED_HEADER = ( - '"""Generated by tools/cppwg_genpackage.py - do not edit.\n' + '"""Generated by cppwg genpackage - do not edit.\n' "\n" "Compiled-extension imports and TemplateClass subscript stubs for this\n" "subpackage. Hand-written code (TemplateMethod attachments, package init,\n" @@ -46,7 +46,7 @@ FLATTEN_HEADER = ( - '"""Generated by tools/cppwg_genpackage.py - do not edit.\n' + '"""Generated by cppwg genpackage - do not edit.\n' "\n" "Re-exports every wrapped class, enum and free function from the subpackages\n" "into the top-level package namespace, so `package.ClassName` works as well\n" @@ -349,7 +349,7 @@ def _warn_orphans(package_root: str, written: list) -> None: delete rather than silently leaving stale bindings or deleting their files. """ written = {os.path.abspath(p) for p in written} - banner = "Generated by tools/cppwg_genpackage.py" + banner = "Generated by cppwg genpackage" for dirpath, _dirs, files in os.walk(package_root): if "_generated.py" not in files: continue diff --git a/cppwg/utils/constants.py b/cppwg/utils/constants.py index 18a190f..b749e30 100644 --- a/cppwg/utils/constants.py +++ b/cppwg/utils/constants.py @@ -11,7 +11,7 @@ # The package model cppwg writes into the wrapper root, describing the generated # modules/classes so a separate step can build the Python package layer (see -# cppwg.utils.package_model and tools/cppwg_genpackage.py). It is JSON: a +# cppwg.utils.package_model and cppwg genpackage). It is JSON: a # machine-generated, git-diff-checked artifact whose serialization is # deterministic across environments (unlike the hand-authored *.yaml configs). CPPWG_PACKAGE_MODEL_FILENAME = "cppwg_package_model.json" diff --git a/cppwg/utils/package_model.py b/cppwg/utils/package_model.py index fa485e9..6c6ff37 100644 --- a/cppwg/utils/package_model.py +++ b/cppwg/utils/package_model.py @@ -1,7 +1,7 @@ """Build a serialisable (plain-dict) model of the Python package layer. cppwg generates the C++/pybind11 wrappers; a separate step (see -``tools/cppwg_genpackage.py``) generates the Python package layer - the +``cppwg genpackage``) generates the Python package layer - the ``_generated.py`` files that import each compiled extension and define the ``TemplateClass`` subscript stubs. That step needs, per module, the compiled module name, the wrapped classes with their template instantiations, and the diff --git a/doc/python-packages.md b/doc/python-packages.md index 09b91ec..16b6f51 100644 --- a/doc/python-packages.md +++ b/doc/python-packages.md @@ -89,7 +89,7 @@ For the `geometry` module, this writes: **geometry/_generated.py** ```python -"""Generated by tools/cppwg_genpackage.py - do not edit. +"""Generated by cppwg genpackage - do not edit. ... """ @@ -179,7 +179,7 @@ stub per templated class: **mesh/_generated.py** ```python -"""Generated by tools/cppwg_genpackage.py - do not edit. +"""Generated by cppwg genpackage - do not edit. ... """ diff --git a/examples/cells/dynamic/package_layout.yaml b/examples/cells/dynamic/package_layout.yaml index 7382e49..da6d332 100644 --- a/examples/cells/dynamic/package_layout.yaml +++ b/examples/cells/dynamic/package_layout.yaml @@ -1,4 +1,4 @@ -# Python-package layout for tools/cppwg_genpackage.py (cells example). +# Python-package layout for cppwg genpackage (cells example). # # A single cppwg module `all` -> the flat pycells package (its compiled # extension _pycells_all sits at the package root, not in an `all/` subdir), so diff --git a/examples/cells/src/py/pycells/__init__.py b/examples/cells/src/py/pycells/__init__.py index 7249079..0c51327 100644 --- a/examples/cells/src/py/pycells/__init__.py +++ b/examples/cells/src/py/pycells/__init__.py @@ -1,7 +1,7 @@ """Main pycells module. The compiled-extension imports and TemplateClass subscript stubs are generated -by tools/cppwg_genpackage.py into _generated.py; add any hand-written code here. +by cppwg genpackage into _generated.py; add any hand-written code here. Curation rationale (why Facet<1>/Corner<1>/AbstractMesh are or aren't wrapped) lives with the C++ sources and examples/cells/dynamic/config.yaml. """ diff --git a/examples/cells/src/py/pycells/_generated.py b/examples/cells/src/py/pycells/_generated.py index ff9cd90..0017a1c 100644 --- a/examples/cells/src/py/pycells/_generated.py +++ b/examples/cells/src/py/pycells/_generated.py @@ -1,4 +1,4 @@ -"""Generated by tools/cppwg_genpackage.py - do not edit. +"""Generated by cppwg genpackage - do not edit. Compiled-extension imports and TemplateClass subscript stubs for this subpackage. Hand-written code (TemplateMethod attachments, package init, diff --git a/examples/shapes/src/py/pyshapes/composites/__init__.py b/examples/shapes/src/py/pyshapes/composites/__init__.py index 7397b2f..19b2817 100644 --- a/examples/shapes/src/py/pyshapes/composites/__init__.py +++ b/examples/shapes/src/py/pyshapes/composites/__init__.py @@ -1,3 +1,3 @@ # The compiled-extension imports and TemplateClass stubs are generated by -# tools/cppwg_genpackage.py into _generated.py; add any hand-written code here. +# cppwg genpackage into _generated.py; add any hand-written code here. from ._generated import * # noqa: F401,F403 diff --git a/examples/shapes/src/py/pyshapes/composites/_generated.py b/examples/shapes/src/py/pyshapes/composites/_generated.py index 669681d..72ebe80 100644 --- a/examples/shapes/src/py/pyshapes/composites/_generated.py +++ b/examples/shapes/src/py/pyshapes/composites/_generated.py @@ -1,4 +1,4 @@ -"""Generated by tools/cppwg_genpackage.py - do not edit. +"""Generated by cppwg genpackage - do not edit. Compiled-extension imports and TemplateClass subscript stubs for this subpackage. Hand-written code (TemplateMethod attachments, package init, diff --git a/examples/shapes/src/py/pyshapes/geometry/__init__.py b/examples/shapes/src/py/pyshapes/geometry/__init__.py index 7397b2f..19b2817 100644 --- a/examples/shapes/src/py/pyshapes/geometry/__init__.py +++ b/examples/shapes/src/py/pyshapes/geometry/__init__.py @@ -1,3 +1,3 @@ # The compiled-extension imports and TemplateClass stubs are generated by -# tools/cppwg_genpackage.py into _generated.py; add any hand-written code here. +# cppwg genpackage into _generated.py; add any hand-written code here. from ._generated import * # noqa: F401,F403 diff --git a/examples/shapes/src/py/pyshapes/geometry/_generated.py b/examples/shapes/src/py/pyshapes/geometry/_generated.py index 5c6a3ab..0d855af 100644 --- a/examples/shapes/src/py/pyshapes/geometry/_generated.py +++ b/examples/shapes/src/py/pyshapes/geometry/_generated.py @@ -1,4 +1,4 @@ -"""Generated by tools/cppwg_genpackage.py - do not edit. +"""Generated by cppwg genpackage - do not edit. Compiled-extension imports and TemplateClass subscript stubs for this subpackage. Hand-written code (TemplateMethod attachments, package init, diff --git a/examples/shapes/src/py/pyshapes/math_funcs/__init__.py b/examples/shapes/src/py/pyshapes/math_funcs/__init__.py index 7397b2f..19b2817 100644 --- a/examples/shapes/src/py/pyshapes/math_funcs/__init__.py +++ b/examples/shapes/src/py/pyshapes/math_funcs/__init__.py @@ -1,3 +1,3 @@ # The compiled-extension imports and TemplateClass stubs are generated by -# tools/cppwg_genpackage.py into _generated.py; add any hand-written code here. +# cppwg genpackage into _generated.py; add any hand-written code here. from ._generated import * # noqa: F401,F403 diff --git a/examples/shapes/src/py/pyshapes/math_funcs/_generated.py b/examples/shapes/src/py/pyshapes/math_funcs/_generated.py index 6530fc3..3e7ad83 100644 --- a/examples/shapes/src/py/pyshapes/math_funcs/_generated.py +++ b/examples/shapes/src/py/pyshapes/math_funcs/_generated.py @@ -1,4 +1,4 @@ -"""Generated by tools/cppwg_genpackage.py - do not edit. +"""Generated by cppwg genpackage - do not edit. Compiled-extension imports and TemplateClass subscript stubs for this subpackage. Hand-written code (TemplateMethod attachments, package init, diff --git a/examples/shapes/src/py/pyshapes/primitives/__init__.py b/examples/shapes/src/py/pyshapes/primitives/__init__.py index 2a8dc92..b7e26a0 100644 --- a/examples/shapes/src/py/pyshapes/primitives/__init__.py +++ b/examples/shapes/src/py/pyshapes/primitives/__init__.py @@ -1,5 +1,5 @@ # The compiled-extension imports and TemplateClass stubs are generated by -# tools/cppwg_genpackage.py into _generated.py; add any hand-written code here. +# cppwg genpackage into _generated.py; add any hand-written code here. from ._generated import * # noqa: F401,F403 from pyshapes._syntax import TemplateMethod diff --git a/examples/shapes/src/py/pyshapes/primitives/_generated.py b/examples/shapes/src/py/pyshapes/primitives/_generated.py index 0b74e9b..9058a64 100644 --- a/examples/shapes/src/py/pyshapes/primitives/_generated.py +++ b/examples/shapes/src/py/pyshapes/primitives/_generated.py @@ -1,4 +1,4 @@ -"""Generated by tools/cppwg_genpackage.py - do not edit. +"""Generated by cppwg genpackage - do not edit. Compiled-extension imports and TemplateClass subscript stubs for this subpackage. Hand-written code (TemplateMethod attachments, package init, diff --git a/examples/shapes/wrapper/package_layout.yaml b/examples/shapes/wrapper/package_layout.yaml index 560a3b1..375443b 100644 --- a/examples/shapes/wrapper/package_layout.yaml +++ b/examples/shapes/wrapper/package_layout.yaml @@ -1,4 +1,4 @@ -# Python-package layout for tools/cppwg_genpackage.py (shapes example). +# Python-package layout for cppwg genpackage (shapes example). # # Module-per-subpackage: each cppwg module (geometry/primitives/composites/ # math_funcs) is a subpackage that owns its own compiled extension, so each diff --git a/tests/test_genpackage.py b/tests/test_genpackage.py index a88f05e..dbd43f8 100644 --- a/tests/test_genpackage.py +++ b/tests/test_genpackage.py @@ -115,7 +115,7 @@ def test_render_generated_module_with_stub_imports_syntax(): content = genpackage.render_generated_module( "pyshapes", "from ._pyshapes_geometry import *", [point], [point] ) - assert content.startswith('"""Generated by tools/cppwg_genpackage.py') + assert content.startswith('"""Generated by cppwg genpackage') assert "from ._pyshapes_geometry import *" in content assert "from pyshapes._syntax import TemplateClass" in content assert "class Point(TemplateClass):" in content @@ -474,7 +474,7 @@ def test_shared_split_empty_subpackage_emits_valid_import(tmp_path): def test_warn_orphans_flags_only_unwritten_banner_files(tmp_path, capsys): """A banner-owned _generated.py that was not written this run is flagged.""" - banner = '"""Generated by tools/cppwg_genpackage.py - do not edit."""\n' + banner = '"""Generated by cppwg genpackage - do not edit."""\n' # A live target (in the written set) -> not flagged. (tmp_path / "live").mkdir() live = tmp_path / "live" / "_generated.py" diff --git a/tools/cppwg_genpackage.py b/tools/cppwg_genpackage.py deleted file mode 100644 index c05c481..0000000 --- a/tools/cppwg_genpackage.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env python3 -"""Standalone launcher for the cppwg package-layer generator. - -The implementation lives in :mod:`cppwg.genpackage`; the canonical invocation is -``cppwg genpackage ...``. This launcher lets the generator also be run directly -from a checkout (``python tools/cppwg_genpackage.py ...``) when cppwg is -importable. -""" - -import sys - -from cppwg.genpackage import main - -if __name__ == "__main__": - sys.exit(main()) From 2cdb617b64458e86273857274807308b1cac7b6b Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 17:42:16 +0100 Subject: [PATCH 21/24] #102 Rename the model's cxx_name field to cpp_name for consistency The package model carried each templated instantiation's C++ type name as `cxx_name`, and genpackage used `cxx`-prefixed helpers (`_build_cxx_index`, `cxx_to_pyname`) - but the rest of cppwg uses `cpp` throughout (class_info.cpp_names, cpp_name, *.cppwg.cpp). The value comes straight from cpp_names, so rename `cxx_*` to `cpp_*` everywhere: the model field, the genpackage helpers, the tests, and the regenerated example models. No output change - _generated.py is unaffected. Co-Authored-By: Claude Opus 4.8 --- cppwg/genpackage.py | 30 +++++++------- cppwg/utils/package_model.py | 8 ++-- .../dynamic/wrappers/cppwg_package_model.json | 40 +++++++++---------- .../shapes/wrapper/cppwg_package_model.json | 20 +++++----- tests/test_genpackage.py | 20 +++++----- tests/test_package_model.py | 14 +++---- 6 files changed, 66 insertions(+), 66 deletions(-) diff --git a/cppwg/genpackage.py b/cppwg/genpackage.py index 4c1b436..6733656 100644 --- a/cppwg/genpackage.py +++ b/cppwg/genpackage.py @@ -71,7 +71,7 @@ def _key_repr(args: list[str]) -> str: return f"({inner})" -def _build_cxx_index(model: dict) -> dict: +def _build_cpp_index(model: dict) -> dict: """Map each wrapped instantiation's C++ type string to its Python class name. e.g. ``PottsMesh<2> -> PottsMesh_2``. Used to key a template argument that is @@ -79,9 +79,9 @@ def _build_cxx_index(model: dict) -> dict: concrete class name, so ``MeshFactory[PottsMesh[2]]`` resolves: ``PottsMesh[2]`` is the ``PottsMesh_2`` class and ``_normalize_key`` keys it by its ``__name__``. - Keyed by the instantiation's real C++ type name (``cxx_name``) so a class with + Keyed by the instantiation's real C++ type name (``cpp_name``) so a class with a name_override (whose Python name differs from its C++ name) still resolves. - Older models without ``cxx_name`` fall back to reconstructing it from the base + Older models without ``cpp_name`` fall back to reconstructing it from the base name and arguments. """ index = {} @@ -89,10 +89,10 @@ def _build_cxx_index(model: dict) -> dict: for class_info in module["classes"]: for inst in class_info["instantiations"]: if inst["args"]: - cxx = inst.get("cxx_name") or ( + cpp = inst.get("cpp_name") or ( f'{class_info["base"]}<{",".join(inst["args"])}>' ) - index[cxx] = inst["py_name"] + index[cpp] = inst["py_name"] return index @@ -100,12 +100,12 @@ def _stub_source( base: str, instantiations: list[dict], diagonal_shorthand: bool = False, - cxx_to_pyname: dict = None, + cpp_to_pyname: dict = None, ) -> str: """Render a ``class (TemplateClass)`` stub for a templated class. A template argument that is itself a wrapped templated type is keyed by its - Python concrete class name via ``cxx_to_pyname`` (``PottsMesh<2>`` -> + Python concrete class name via ``cpp_to_pyname`` (``PottsMesh<2>`` -> ``PottsMesh_2``), so ``MeshFactory[PottsMesh[2]]`` resolves. When ``diagonal_shorthand`` is set, a multi-argument instantiation whose @@ -115,10 +115,10 @@ def _stub_source( ````-style classes; others (the cells example) keep the explicit multi-argument form only. """ - cxx_to_pyname = cxx_to_pyname or {} + cpp_to_pyname = cpp_to_pyname or {} lines = [f"class {base}(TemplateClass):", " _instantiations = {"] for inst in instantiations: - args = [cxx_to_pyname.get(arg, arg) for arg in inst["args"]] + args = [cpp_to_pyname.get(arg, arg) for arg in inst["args"]] lines.append(f' {_key_repr(args)}: {inst["py_name"]},') if diagonal_shorthand and len(args) > 1 and len(set(args)) == 1: lines.append(f' {_key_repr(args[:1])}: {inst["py_name"]},') @@ -132,7 +132,7 @@ def render_generated_module( classes: list[dict], templated_classes: list[dict], diagonal_shorthand: bool = False, - cxx_to_pyname: dict = None, + cpp_to_pyname: dict = None, ) -> str: """ Render a subpackage's ``_generated.py`` content. @@ -168,7 +168,7 @@ def render_generated_module( class_info["base"], class_info["instantiations"], diagonal_shorthand, - cxx_to_pyname, + cpp_to_pyname, ) ) return "\n".join(lines) + "\n" @@ -210,7 +210,7 @@ def generate_module_per_subpackage(model: dict, layout: dict, overwrite: bool) - package_root = layout["package_root"] module_dirs = layout.get("module_dirs", {}) diagonal_shorthand = layout.get("diagonal_shorthand", False) - cxx_to_pyname = _build_cxx_index(model) + cpp_to_pyname = _build_cpp_index(model) written = [] for module in model["modules"]: @@ -225,7 +225,7 @@ def generate_module_per_subpackage(model: dict, layout: dict, overwrite: bool) - module["classes"], templated, diagonal_shorthand, - cxx_to_pyname, + cpp_to_pyname, ) path = os.path.join(package_root, subdir, "_generated.py") written.append(_write_generated(path, content, overwrite)) @@ -241,7 +241,7 @@ def generate_shared_module_split(model: dict, layout: dict, overwrite: bool) -> package_root = layout["package_root"] compiled_module = layout["compiled_module"] diagonal_shorthand = layout.get("diagonal_shorthand", False) - cxx_to_pyname = _build_cxx_index(model) + cpp_to_pyname = _build_cpp_index(model) written = [] # Index every wrapped entity across the model by name so a layout entry can @@ -281,7 +281,7 @@ def generate_shared_module_split(model: dict, layout: dict, overwrite: bool) -> classes, templated, diagonal_shorthand, - cxx_to_pyname, + cpp_to_pyname, ) path = os.path.join(package_root, subpkg, "_generated.py") written.append(_write_generated(path, content, overwrite)) diff --git a/cppwg/utils/package_model.py b/cppwg/utils/package_model.py index 6c6ff37..a62626e 100644 --- a/cppwg/utils/package_model.py +++ b/cppwg/utils/package_model.py @@ -44,7 +44,7 @@ def build_package_model(package_info: "PackageInfo") -> dict[str, Any]: "enums": [...], "free_functions": [...]}]}``. Excluded entities are omitted (they are not wrapped). An untemplated class has ``templated: false`` and a single instantiation with empty ``args``; a templated - instantiation additionally carries ``cxx_name`` (its C++ type name). + instantiation additionally carries ``cpp_name`` (its C++ type name). """ modules = [] for module in package_info.module_collection: @@ -54,16 +54,16 @@ def build_package_model(package_info: "PackageInfo") -> dict[str, Any]: continue if class_info.template_arg_lists: - # Carry each instantiation's actual C++ type name (cxx_name), which + # Carry each instantiation's actual C++ type name (cpp_name), which # differs from base when the class has a name_override; the # package-layer generator keys nested template arguments by it. instantiations = [ { "args": [str(arg) for arg in args], - "cxx_name": cxx_name, + "cpp_name": cpp_name, "py_name": py_name, } - for args, cxx_name, py_name in zip( + for args, cpp_name, py_name in zip( class_info.template_arg_lists, class_info.cpp_names, class_info.py_names, diff --git a/examples/cells/dynamic/wrappers/cppwg_package_model.json b/examples/cells/dynamic/wrappers/cppwg_package_model.json index 96a67a0..bc9caea 100644 --- a/examples/cells/dynamic/wrappers/cppwg_package_model.json +++ b/examples/cells/dynamic/wrappers/cppwg_package_model.json @@ -21,7 +21,7 @@ "Cell", "2" ], - "cxx_name": "CellFactory", + "cpp_name": "CellFactory", "py_name": "CellFactory_Cell_2" }, { @@ -29,7 +29,7 @@ "Cell", "3" ], - "cxx_name": "CellFactory", + "cpp_name": "CellFactory", "py_name": "CellFactory_Cell_3" } ], @@ -42,7 +42,7 @@ "args": [ "2" ], - "cxx_name": "Corner<2>", + "cpp_name": "Corner<2>", "py_name": "Corner_2" } ], @@ -55,7 +55,7 @@ "args": [ "2" ], - "cxx_name": "Facet<2>", + "cpp_name": "Facet<2>", "py_name": "Facet_2" } ], @@ -69,7 +69,7 @@ "2", "2" ], - "cxx_name": "MacroMesh<2, 2>", + "cpp_name": "MacroMesh<2, 2>", "py_name": "MacroMesh_2_2" }, { @@ -77,7 +77,7 @@ "3", "3" ], - "cxx_name": "MacroMesh<3, 3>", + "cpp_name": "MacroMesh<3, 3>", "py_name": "MacroMesh_3_3" } ], @@ -90,14 +90,14 @@ "args": [ "2" ], - "cxx_name": "Node<2>", + "cpp_name": "Node<2>", "py_name": "Node_2" }, { "args": [ "3" ], - "cxx_name": "Node<3>", + "cpp_name": "Node<3>", "py_name": "Node_3" } ], @@ -111,7 +111,7 @@ "2", "2" ], - "cxx_name": "AbstractMesh<2, 2>", + "cpp_name": "AbstractMesh<2, 2>", "py_name": "AbstractMesh_2_2" }, { @@ -119,7 +119,7 @@ "3", "3" ], - "cxx_name": "AbstractMesh<3, 3>", + "cpp_name": "AbstractMesh<3, 3>", "py_name": "AbstractMesh_3_3" } ], @@ -133,7 +133,7 @@ "2", "2" ], - "cxx_name": "AbstractSphericalMesh<2, 2>", + "cpp_name": "AbstractSphericalMesh<2, 2>", "py_name": "AbstractSphericalMesh_2_2" }, { @@ -141,7 +141,7 @@ "3", "3" ], - "cxx_name": "AbstractSphericalMesh<3, 3>", + "cpp_name": "AbstractSphericalMesh<3, 3>", "py_name": "AbstractSphericalMesh_3_3" } ], @@ -164,14 +164,14 @@ "args": [ "2" ], - "cxx_name": "PottsMesh<2>", + "cpp_name": "PottsMesh<2>", "py_name": "PottsMesh_2" }, { "args": [ "3" ], - "cxx_name": "PottsMesh<3>", + "cpp_name": "PottsMesh<3>", "py_name": "PottsMesh_3" } ], @@ -184,14 +184,14 @@ "args": [ "PottsMesh<2>" ], - "cxx_name": "MeshFactory>", + "cpp_name": "MeshFactory>", "py_name": "MeshFactory_PottsMesh_2" }, { "args": [ "PottsMesh<3>" ], - "cxx_name": "MeshFactory>", + "cpp_name": "MeshFactory>", "py_name": "MeshFactory_PottsMesh_3" } ], @@ -204,14 +204,14 @@ "args": [ "2" ], - "cxx_name": "Scene<2>", + "cpp_name": "Scene<2>", "py_name": "Scene_2" }, { "args": [ "3" ], - "cxx_name": "Scene<3>", + "cpp_name": "Scene<3>", "py_name": "Scene_3" } ], @@ -225,7 +225,7 @@ "2", "2" ], - "cxx_name": "SphericalMesh<2, 2>", + "cpp_name": "SphericalMesh<2, 2>", "py_name": "SphericalMesh_2_2" }, { @@ -233,7 +233,7 @@ "3", "3" ], - "cxx_name": "SphericalMesh<3, 3>", + "cpp_name": "SphericalMesh<3, 3>", "py_name": "SphericalMesh_3_3" } ], diff --git a/examples/shapes/wrapper/cppwg_package_model.json b/examples/shapes/wrapper/cppwg_package_model.json index 77219f0..a3246c1 100644 --- a/examples/shapes/wrapper/cppwg_package_model.json +++ b/examples/shapes/wrapper/cppwg_package_model.json @@ -22,14 +22,14 @@ "args": [ "2" ], - "cxx_name": "Point<2>", + "cpp_name": "Point<2>", "py_name": "Point_2" }, { "args": [ "3" ], - "cxx_name": "Point<3>", + "cpp_name": "Point<3>", "py_name": "Point_3" } ], @@ -51,14 +51,14 @@ "args": [ "2" ], - "cxx_name": "AbstractShape<2>", + "cpp_name": "AbstractShape<2>", "py_name": "AbstractShape_2" }, { "args": [ "3" ], - "cxx_name": "AbstractShape<3>", + "cpp_name": "AbstractShape<3>", "py_name": "AbstractShape_3" } ], @@ -71,14 +71,14 @@ "args": [ "2" ], - "cxx_name": "AbstractPolygon<2>", + "cpp_name": "AbstractPolygon<2>", "py_name": "AbstractPolygon_2" }, { "args": [ "3" ], - "cxx_name": "AbstractPolygon<3>", + "cpp_name": "AbstractPolygon<3>", "py_name": "AbstractPolygon_3" } ], @@ -91,14 +91,14 @@ "args": [ "2" ], - "cxx_name": "RegularPolygon<2>", + "cpp_name": "RegularPolygon<2>", "py_name": "RegularPolygon_2" }, { "args": [ "3" ], - "cxx_name": "RegularPolygon<3>", + "cpp_name": "RegularPolygon<3>", "py_name": "RegularPolygon_3" } ], @@ -111,14 +111,14 @@ "args": [ "2" ], - "cxx_name": "Shape<2>", + "cpp_name": "Shape<2>", "py_name": "Shape_2" }, { "args": [ "3" ], - "cxx_name": "Shape<3>", + "cpp_name": "Shape<3>", "py_name": "Shape_3" } ], diff --git a/tests/test_genpackage.py b/tests/test_genpackage.py index dbd43f8..28b98af 100644 --- a/tests/test_genpackage.py +++ b/tests/test_genpackage.py @@ -14,10 +14,10 @@ def _class(base, instantiations, templated=True): return {"base": base, "templated": templated, "instantiations": instantiations} -def _inst(args, py_name, cxx_name=None): +def _inst(args, py_name, cpp_name=None): inst = {"args": list(args), "py_name": py_name} - if cxx_name is not None: - inst["cxx_name"] = cxx_name + if cpp_name is not None: + inst["cpp_name"] = cpp_name return inst @@ -61,21 +61,21 @@ def test_stub_source_nested_templated_arg(): """A templated-type argument is keyed by its Python concrete class name.""" # MeshFactory> -> key ("PottsMesh_2",), not ("PottsMesh<2>",), # so MeshFactory[PottsMesh[2]] resolves (PottsMesh[2] is the PottsMesh_2 class). - cxx_to_pyname = {"PottsMesh<2>": "PottsMesh_2", "PottsMesh<3>": "PottsMesh_3"} + cpp_to_pyname = {"PottsMesh<2>": "PottsMesh_2", "PottsMesh<3>": "PottsMesh_3"} stub = genpackage._stub_source( "MeshFactory", [ _inst(["PottsMesh<2>"], "MeshFactory_PottsMesh_2"), _inst(["PottsMesh<3>"], "MeshFactory_PottsMesh_3"), ], - cxx_to_pyname=cxx_to_pyname, + cpp_to_pyname=cpp_to_pyname, ) assert '("PottsMesh_2",): MeshFactory_PottsMesh_2,' in stub assert '("PottsMesh_3",): MeshFactory_PottsMesh_3,' in stub assert "<" not in stub # the raw C++ type string is gone -def test_build_cxx_index(): +def test_build_cpp_index(): model = { "modules": [ { @@ -86,11 +86,11 @@ def test_build_cxx_index(): } ] } - index = genpackage._build_cxx_index(model) + index = genpackage._build_cpp_index(model) assert index == {"PottsMesh<2>": "PottsMesh_2"} # untemplated Cell has no <...> -def test_build_cxx_index_uses_cxx_name_for_name_override(): +def test_build_cpp_index_uses_cpp_name_for_name_override(): """A class whose C++ name differs from its py name is keyed by the C++ name.""" model = { "modules": [ @@ -98,13 +98,13 @@ def test_build_cxx_index_uses_cxx_name_for_name_override(): "classes": [ _class( "NewName", - [_inst(["2"], "NewName_2", cxx_name="OldName<2>")], + [_inst(["2"], "NewName_2", cpp_name="OldName<2>")], ), ] } ] } - index = genpackage._build_cxx_index(model) + index = genpackage._build_cpp_index(model) # Keyed by OldName<2> (the real C++ type), not NewName<2>, so a nested # OldName<2> argument resolves to NewName_2. assert index == {"OldName<2>": "NewName_2"} diff --git a/tests/test_package_model.py b/tests/test_package_model.py index 9f6765b..33848c3 100644 --- a/tests/test_package_model.py +++ b/tests/test_package_model.py @@ -85,8 +85,8 @@ def test_build_model_templated_and_untemplated(): "base": "Point", "templated": True, "instantiations": [ - {"args": ["2"], "cxx_name": "Point<2>", "py_name": "Point_2"}, - {"args": ["3"], "cxx_name": "Point<3>", "py_name": "Point_3"}, + {"args": ["2"], "cpp_name": "Point<2>", "py_name": "Point_2"}, + {"args": ["3"], "cpp_name": "Point<3>", "py_name": "Point_3"}, ], } @@ -156,19 +156,19 @@ def test_build_model_multi_arg_and_class_arg_keys(): (module,) = build_package_model(package)["modules"] macro, factory = module["classes"] assert macro["instantiations"] == [ - {"args": ["2", "2"], "cxx_name": "MacroMesh<2, 2>", "py_name": "MacroMesh_2_2"} + {"args": ["2", "2"], "cpp_name": "MacroMesh<2, 2>", "py_name": "MacroMesh_2_2"} ] assert factory["instantiations"] == [ { "args": ["Cell", "2"], - "cxx_name": "CellFactory", + "cpp_name": "CellFactory", "py_name": "CellFactory_Cell_2", } ] -def test_instantiation_carries_cxx_name_for_name_override(): - """cxx_name is the C++ type name (from cpp_names), not the overridden py name.""" +def test_instantiation_carries_cpp_name_for_name_override(): + """cpp_name is the C++ type name (from cpp_names), not the overridden py name.""" package = _package( "pkg", [ @@ -184,7 +184,7 @@ def test_instantiation_carries_cxx_name_for_name_override(): (module,) = build_package_model(package)["modules"] (cls,) = module["classes"] assert cls["instantiations"] == [ - {"args": ["2"], "cxx_name": "OldName<2>", "py_name": "NewName_2"} + {"args": ["2"], "cpp_name": "OldName<2>", "py_name": "NewName_2"} ] From 697c60b1c8a40f658f69a1ae78b4fe0c834c6ac7 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 17:55:46 +0100 Subject: [PATCH 22/24] #102 Clarify genpackage helper names; document _write_generated Rename internal helpers for readability, with no behaviour change (generated output is byte-identical): - _stub_source -> _render_template_stub and _key_repr -> _render_key, joining render_generated_module in a consistent "render" family. - _build_cpp_index -> _build_cpp_to_pyname, matching the cpp_to_pyname map it returns and dropping the vague "index". - _concrete_names -> _concrete_py_names (they are py_names, now sitting next to cpp_names in the model). Also add the missing _write_generated docstring. Co-Authored-By: Claude Opus 4.8 --- cppwg/genpackage.py | 26 ++++++++++++++++---------- tests/test_genpackage.py | 34 +++++++++++++++++----------------- 2 files changed, 33 insertions(+), 27 deletions(-) diff --git a/cppwg/genpackage.py b/cppwg/genpackage.py index 6733656..cdff7cb 100644 --- a/cppwg/genpackage.py +++ b/cppwg/genpackage.py @@ -57,7 +57,7 @@ ) -def _key_repr(args: list[str]) -> str: +def _render_key(args: list[str]) -> str: """Render a template-argument list as a Python tuple literal of strings. e.g. ``["2"]`` -> ``("2",)`` and ``["Cell", "2"]`` -> ``("Cell", "2")``. @@ -71,7 +71,7 @@ def _key_repr(args: list[str]) -> str: return f"({inner})" -def _build_cpp_index(model: dict) -> dict: +def _build_cpp_to_pyname(model: dict) -> dict: """Map each wrapped instantiation's C++ type string to its Python class name. e.g. ``PottsMesh<2> -> PottsMesh_2``. Used to key a template argument that is @@ -96,7 +96,7 @@ def _build_cpp_index(model: dict) -> dict: return index -def _stub_source( +def _render_template_stub( base: str, instantiations: list[dict], diagonal_shorthand: bool = False, @@ -119,9 +119,9 @@ def _stub_source( lines = [f"class {base}(TemplateClass):", " _instantiations = {"] for inst in instantiations: args = [cpp_to_pyname.get(arg, arg) for arg in inst["args"]] - lines.append(f' {_key_repr(args)}: {inst["py_name"]},') + lines.append(f' {_render_key(args)}: {inst["py_name"]},') if diagonal_shorthand and len(args) > 1 and len(set(args)) == 1: - lines.append(f' {_key_repr(args[:1])}: {inst["py_name"]},') + lines.append(f' {_render_key(args[:1])}: {inst["py_name"]},') lines.append(" }") return "\n".join(lines) @@ -164,7 +164,7 @@ def render_generated_module( for class_info in templated_classes: lines.extend(["", ""]) # two blank lines before each top-level class lines.append( - _stub_source( + _render_template_stub( class_info["base"], class_info["instantiations"], diagonal_shorthand, @@ -174,7 +174,7 @@ def render_generated_module( return "\n".join(lines) + "\n" -def _concrete_names(class_info: dict) -> list[str]: +def _concrete_py_names(class_info: dict) -> list[str]: """Return the concrete py_names of a class's instantiations.""" return [inst["py_name"] for inst in class_info["instantiations"]] @@ -193,6 +193,12 @@ def _explicit_import(package: str, compiled_module: str, names: list[str]) -> st def _write_generated(path: str, content: str, overwrite: bool) -> str: + """Write a ``_generated.py`` (creating parent dirs) and return its abspath. + + Reports whether the file was written or left unchanged. The returned path + lets the caller record which outputs are live, so stale ones can be flagged + (see ``_warn_orphans``). + """ os.makedirs(os.path.dirname(path), exist_ok=True) if write_file_if_changed(path, content, overwrite): print(f"wrote {path}") @@ -210,7 +216,7 @@ def generate_module_per_subpackage(model: dict, layout: dict, overwrite: bool) - package_root = layout["package_root"] module_dirs = layout.get("module_dirs", {}) diagonal_shorthand = layout.get("diagonal_shorthand", False) - cpp_to_pyname = _build_cpp_index(model) + cpp_to_pyname = _build_cpp_to_pyname(model) written = [] for module in model["modules"]: @@ -241,7 +247,7 @@ def generate_shared_module_split(model: dict, layout: dict, overwrite: bool) -> package_root = layout["package_root"] compiled_module = layout["compiled_module"] diagonal_shorthand = layout.get("diagonal_shorthand", False) - cpp_to_pyname = _build_cpp_index(model) + cpp_to_pyname = _build_cpp_to_pyname(model) written = [] # Index every wrapped entity across the model by name so a layout entry can @@ -265,7 +271,7 @@ def generate_shared_module_split(model: dict, layout: dict, overwrite: bool) -> if name in classes_by_base: class_info = classes_by_base[name] classes.append(class_info) - import_names.extend(_concrete_names(class_info)) + import_names.extend(_concrete_py_names(class_info)) exported.append(name) elif name in other_names: import_names.append(name) diff --git a/tests/test_genpackage.py b/tests/test_genpackage.py index 28b98af..ef47b8d 100644 --- a/tests/test_genpackage.py +++ b/tests/test_genpackage.py @@ -21,14 +21,14 @@ def _inst(args, py_name, cpp_name=None): return inst -def test_key_repr_singleton_and_multi(): - assert genpackage._key_repr(["2"]) == '("2",)' - assert genpackage._key_repr(["2", "2"]) == '("2", "2")' - assert genpackage._key_repr(["Cell", "2"]) == '("Cell", "2")' +def test_render_key_singleton_and_multi(): + assert genpackage._render_key(["2"]) == '("2",)' + assert genpackage._render_key(["2", "2"]) == '("2", "2")' + assert genpackage._render_key(["Cell", "2"]) == '("Cell", "2")' -def test_stub_source(): - stub = genpackage._stub_source( +def test_render_template_stub(): + stub = genpackage._render_template_stub( "Point", [_inst(["2"], "Point_2"), _inst(["3"], "Point_3")] ) assert stub == ( @@ -40,9 +40,9 @@ def test_stub_source(): ) -def test_stub_source_diagonal_shorthand(): +def test_render_template_stub_diagonal_shorthand(): """A multi-arg diagonal instantiation gains a single-arg alias when opted in.""" - stub = genpackage._stub_source( + stub = genpackage._render_template_stub( "Element", [_inst(["2", "2"], "Element_2_2"), _inst(["1", "2"], "Element_1_2")], diagonal_shorthand=True, @@ -53,16 +53,16 @@ def test_stub_source_diagonal_shorthand(): assert '("1", "2"): Element_1_2,' in stub assert '("1",):' not in stub # Off by default: no aliases. - plain = genpackage._stub_source("Element", [_inst(["2", "2"], "Element_2_2")]) + plain = genpackage._render_template_stub("Element", [_inst(["2", "2"], "Element_2_2")]) assert '("2",):' not in plain -def test_stub_source_nested_templated_arg(): +def test_render_template_stub_nested_templated_arg(): """A templated-type argument is keyed by its Python concrete class name.""" # MeshFactory> -> key ("PottsMesh_2",), not ("PottsMesh<2>",), # so MeshFactory[PottsMesh[2]] resolves (PottsMesh[2] is the PottsMesh_2 class). cpp_to_pyname = {"PottsMesh<2>": "PottsMesh_2", "PottsMesh<3>": "PottsMesh_3"} - stub = genpackage._stub_source( + stub = genpackage._render_template_stub( "MeshFactory", [ _inst(["PottsMesh<2>"], "MeshFactory_PottsMesh_2"), @@ -75,7 +75,7 @@ def test_stub_source_nested_templated_arg(): assert "<" not in stub # the raw C++ type string is gone -def test_build_cpp_index(): +def test_build_cpp_to_pyname(): model = { "modules": [ { @@ -86,11 +86,11 @@ def test_build_cpp_index(): } ] } - index = genpackage._build_cpp_index(model) + index = genpackage._build_cpp_to_pyname(model) assert index == {"PottsMesh<2>": "PottsMesh_2"} # untemplated Cell has no <...> -def test_build_cpp_index_uses_cpp_name_for_name_override(): +def test_build_cpp_to_pyname_uses_cpp_name_for_name_override(): """A class whose C++ name differs from its py name is keyed by the C++ name.""" model = { "modules": [ @@ -104,7 +104,7 @@ def test_build_cpp_index_uses_cpp_name_for_name_override(): } ] } - index = genpackage._build_cpp_index(model) + index = genpackage._build_cpp_to_pyname(model) # Keyed by OldName<2> (the real C++ type), not NewName<2>, so a nested # OldName<2> argument resolves to NewName_2. assert index == {"OldName<2>": "NewName_2"} @@ -433,9 +433,9 @@ def test_flatten_skips_subpackage_with_no_exports(tmp_path): assert '"Widget",' in root # __all__ -def test_key_repr_escapes_special_chars(): +def test_render_key_escapes_special_chars(): """A value with special characters is escaped into a valid Python literal.""" - rendered = genpackage._key_repr(['a"b', "c"]) + rendered = genpackage._render_key(['a"b', "c"]) # Round-trips: the emitted tuple literal is valid Python, not `("a"b",...`. assert ast.literal_eval(rendered) == ('a"b', "c") From 4ed2b34165c096c4f81ce7a37f5b33822b8ca574 Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 18:44:03 +0100 Subject: [PATCH 23/24] #102 Share the free-function exclusion predicate via the info layer Hoist the free-function type-based exclusion rule into cppwg.info.exclusions.free_function_is_excluded, mirroring the existing method/constructor/variable predicates, so the writer and the package model apply one rule and cannot drift. Previously the model recorded functions the writer dropped for an excluded arg/return type, so a shared-module layout would emit an import of a symbol the extension never bound. Also honor the per-function config `excluded` flag in the free-function writer's exclude() (as the enum writer does with enum_info.excluded), so a config-excluded function emits no binding and the writer agrees with the model. Co-Authored-By: Claude Opus 4.8 --- cppwg/info/exclusions.py | 50 +++++++++++++++++++++++++++ cppwg/utils/package_model.py | 6 ++++ cppwg/writers/free_function_writer.py | 40 +++++---------------- tests/test_free_function_writer.py | 13 +++++-- tests/test_package_model.py | 41 ++++++++++++++++++++-- 5 files changed, 114 insertions(+), 36 deletions(-) diff --git a/cppwg/info/exclusions.py b/cppwg/info/exclusions.py index c685ecf..13768e8 100644 --- a/cppwg/info/exclusions.py +++ b/cppwg/info/exclusions.py @@ -25,6 +25,7 @@ from pygccxml.declarations.variable import variable_t from cppwg.info.class_info import CppClassInfo + from cppwg.info.free_function_info import CppFreeFunctionInfo def method_is_excluded( @@ -291,3 +292,52 @@ def variable_is_excluded( True if the member should be excluded, False otherwise. """ return variable_exclusion_reason(class_info, class_decl, variable_decl) is not None + + +def free_function_is_excluded(free_function_info: "CppFreeFunctionInfo") -> bool: + """ + Return True if a free function would be excluded from the wrapper code. + + A free function is dropped when its return type or any argument type matches + return_type_excludes / arg_type_excludes (the deprecated calldef_excludes + applies to both). Shared by CppFreeFunctionWrapperWriter and the package model + so a function whose binding is never emitted is not recorded as wrapped. + + Returns + ------- + bool + True if the function should be excluded, False otherwise. + """ + decl = free_function_info.decls[0] + + # Exclude by return type. return_type_excludes targets return types; the + # deprecated calldef_excludes applies to both return and arg types. + calldef_excludes = free_function_info.hierarchy_attribute_gather_flat( + "calldef_excludes" + ) + return_type_excludes = ( + free_function_info.hierarchy_attribute_gather_flat("return_type_excludes") + + calldef_excludes + ) + return_type = decl.return_type.decl_string + if any( + utils.type_string_matches(return_type, pattern) + for pattern in return_type_excludes + ): + return True + + # Exclude by argument type. arg_type_excludes is the general arg-type exclude; + # the deprecated calldef_excludes applies too. + arg_type_excludes = ( + free_function_info.hierarchy_attribute_gather_flat("arg_type_excludes") + + calldef_excludes + ) + for argument_type in decl.argument_types: + arg_type = argument_type.decl_string + if any( + utils.type_string_matches(arg_type, pattern) + for pattern in arg_type_excludes + ): + return True + + return False diff --git a/cppwg/utils/package_model.py b/cppwg/utils/package_model.py index a62626e..ceadaa6 100644 --- a/cppwg/utils/package_model.py +++ b/cppwg/utils/package_model.py @@ -12,6 +12,8 @@ from typing import TYPE_CHECKING, Any +from cppwg.info.exclusions import free_function_is_excluded + if TYPE_CHECKING: from cppwg.info.package_info import PackageInfo @@ -93,10 +95,14 @@ def build_package_model(package_info: "PackageInfo") -> dict[str, Any]: for enum_info in module.enum_collection if not enum_info.excluded ] + # Skip config-excluded functions and those the writer drops for an + # excluded arg/return type (free_function_is_excluded) - a dropped + # function emits no binding, so it must not be recorded as wrapped. free_functions = [ free_function_info.name for free_function_info in module.free_function_collection if not free_function_info.excluded + and not free_function_is_excluded(free_function_info) ] modules.append( diff --git a/cppwg/writers/free_function_writer.py b/cppwg/writers/free_function_writer.py index 081c8f6..5e3cdd5 100644 --- a/cppwg/writers/free_function_writer.py +++ b/cppwg/writers/free_function_writer.py @@ -2,8 +2,8 @@ from typing import TYPE_CHECKING +from cppwg.info import exclusions from cppwg.info.free_function_info import CppFreeFunctionInfo -from cppwg.utils import utils from cppwg.writers.base_writer import CppBaseWrapperWriter if TYPE_CHECKING: @@ -73,34 +73,12 @@ def exclude(self) -> bool: bool True if the function should be excluded from wrapper code, False otherwise. """ - info = self.free_function_info - decl = info.decls[0] - - # Exclude by return type. return_type_excludes targets return types; the - # deprecated calldef_excludes applies to both return and arg types. - calldef_excludes = info.hierarchy_attribute_gather_flat("calldef_excludes") - return_type_excludes = ( - info.hierarchy_attribute_gather_flat("return_type_excludes") - + calldef_excludes + # Config-excluded (YAML `excluded: true`) or dropped for an excluded + # arg/return type. The type-based rule lives in free_function_is_excluded, + # shared with the package model; the config flag is folded in here (as the + # enum writer does with enum_info.excluded) so the writer and model agree + # on the wrapped set. + return ( + self.free_function_info.excluded + or exclusions.free_function_is_excluded(self.free_function_info) ) - return_type = decl.return_type.decl_string - if any( - utils.type_string_matches(return_type, pattern) - for pattern in return_type_excludes - ): - return True - - # Exclude by argument type. arg_type_excludes is the general arg-type - # exclude; the deprecated calldef_excludes applies too. - arg_type_excludes = ( - info.hierarchy_attribute_gather_flat("arg_type_excludes") + calldef_excludes - ) - for argument_type in decl.argument_types: - arg_type = argument_type.decl_string - if any( - utils.type_string_matches(arg_type, pattern) - for pattern in arg_type_excludes - ): - return True - - return False diff --git a/tests/test_free_function_writer.py b/tests/test_free_function_writer.py index de547aa..a6419b1 100644 --- a/tests/test_free_function_writer.py +++ b/tests/test_free_function_writer.py @@ -21,16 +21,17 @@ def __init__(self, return_type, arg_types): class _FreeFunctionInfo: """Minimal CppFreeFunctionInfo stand-in for exclusion lookups.""" - def __init__(self, return_type, arg_types, excludes=None): + def __init__(self, return_type, arg_types, excludes=None, excluded=False): self.decls = [_Decl(return_type, arg_types)] + self.excluded = excluded self._excludes = excludes or {} def hierarchy_attribute_gather_flat(self, name): return list(self._excludes.get(name, [])) -def _writer(return_type="void", arg_types=(), excludes=None): - info = _FreeFunctionInfo(return_type, list(arg_types), excludes) +def _writer(return_type="void", arg_types=(), excludes=None, excluded=False): + info = _FreeFunctionInfo(return_type, list(arg_types), excludes, excluded) writer = object.__new__(CppFreeFunctionWrapperWriter) writer.free_function_info = info return writer @@ -71,6 +72,12 @@ def test_free_function_not_excluded_without_options(): assert _writer(return_type="int", arg_types=["double"]).exclude() is False +def test_free_function_config_excluded(): + """A config-excluded function (YAML `excluded: true`) is excluded regardless + of its types, so its binding is never emitted.""" + assert _writer(return_type="int", arg_types=["double"], excluded=True).exclude() + + from string import Template # noqa: E402 diff --git a/tests/test_package_model.py b/tests/test_package_model.py index 33848c3..b71c3ef 100644 --- a/tests/test_package_model.py +++ b/tests/test_package_model.py @@ -19,8 +19,18 @@ def _enum(name, name_override="", excluded=False): return SimpleNamespace(name=name, name_override=name_override, excluded=excluded) -def _free_function(name, excluded=False): - return SimpleNamespace(name=name, excluded=excluded) +def _free_function(name, excluded=False, arg_types=(), return_type="void", excludes=None): + _ex = excludes or {} + decl = SimpleNamespace( + return_type=SimpleNamespace(decl_string=return_type), + argument_types=[SimpleNamespace(decl_string=a) for a in arg_types], + ) + return SimpleNamespace( + name=name, + excluded=excluded, + decls=[decl], + hierarchy_attribute_gather_flat=lambda key: list(_ex.get(key, [])), + ) def _module(name, classes=(), enums=(), free_functions=(), imports=()): @@ -188,6 +198,33 @@ def test_instantiation_carries_cpp_name_for_name_override(): ] +def test_build_model_omits_free_function_excluded_by_type(): + """A free function the writer drops for an excluded arg type is not recorded. + + Its binding is never emitted, so recording it would make a shared-module + layout import a symbol that does not exist in the compiled extension. + """ + package = _package( + "pkg", + [ + _module( + "mod", + free_functions=[ + _free_function("kept_fn"), + _free_function( + "shape_fn", + arg_types=["::Shape<2> const &"], + excludes={"arg_type_excludes": ["Shape"]}, + ), + ], + ) + ], + ) + + (module,) = build_package_model(package)["modules"] + assert module["free_functions"] == ["kept_fn"] # shape_fn dropped, not recorded + + def test_enum_name_override_used(): package = _package( "pkg", [_module("mod", enums=[_enum("RawName", name_override="PyName")])] From 86949177e57ea9f826fdad5c672037f782ad403c Mon Sep 17 00:00:00 2001 From: Kwabena Amponsah Date: Wed, 12 Aug 2026 19:07:29 +0100 Subject: [PATCH 24/24] #102 Address package-layer generator review comments Three fixes from the PR review of the genpackage package-layer generator: - Empty subpackage: `import package.extension` bound the top-level `package` name, which the sibling __init__'s `from ._generated import *` leaked into the package API. Alias it to a private `_extension` so nothing leaks. - Renamed untemplated class as a template argument: the model now carries cpp_name for untemplated instantiations too, and _build_cpp_to_pyname indexes them, so an OldName-exposed-as-NewName class used as `Factory` resolves to `Factory[NewName]` instead of raising KeyError. - Exported enum values under the shared-module split: a value-exporting enum also binds its enumerators at module scope (e.g. ShapeKind exports CIRCLE). The model records these per enum (enum_exports) and the split imports and re-exports them alongside the enum, so subpackage.CIRCLE keeps working. Regenerated the shapes and cells package models (additive cpp_name/enum_exports fields only; wrappers and _generated.py unchanged). Co-Authored-By: Claude Opus 4.8 --- cppwg/genpackage.py | 33 ++++++--- cppwg/utils/package_model.py | 32 +++++++-- .../dynamic/wrappers/cppwg_package_model.json | 3 + .../shapes/wrapper/cppwg_package_model.json | 18 +++++ tests/test_genpackage.py | 70 ++++++++++++++++++- tests/test_package_model.py | 53 +++++++++++++- 6 files changed, 190 insertions(+), 19 deletions(-) diff --git a/cppwg/genpackage.py b/cppwg/genpackage.py index cdff7cb..008b23f 100644 --- a/cppwg/genpackage.py +++ b/cppwg/genpackage.py @@ -80,18 +80,22 @@ def _build_cpp_to_pyname(model: dict) -> dict: is the ``PottsMesh_2`` class and ``_normalize_key`` keys it by its ``__name__``. Keyed by the instantiation's real C++ type name (``cpp_name``) so a class with - a name_override (whose Python name differs from its C++ name) still resolves. - Older models without ``cpp_name`` fall back to reconstructing it from the base - name and arguments. + a name_override (whose Python name differs from its C++ name) still resolves - + including an untemplated renamed class (``OldName`` exposed as ``NewName``) + used as a template argument. Older models without ``cpp_name`` fall back to + reconstructing it from the base name and arguments (templated only). """ index = {} for module in model["modules"]: for class_info in module["classes"]: for inst in class_info["instantiations"]: - if inst["args"]: - cpp = inst.get("cpp_name") or ( - f'{class_info["base"]}<{",".join(inst["args"])}>' - ) + cpp = inst.get("cpp_name") + if cpp is None and inst["args"]: + # Older model without cpp_name: reconstruct base. An + # untemplated class in such a model carries no C++ name, so + # it cannot be indexed. + cpp = f'{class_info["base"]}<{",".join(inst["args"])}>' + if cpp is not None: index[cpp] = inst["py_name"] return index @@ -184,10 +188,13 @@ def _explicit_import(package: str, compiled_module: str, names: list[str]) -> st With no names (an empty subpackage, or one whose names are all absent from the model) a ``from ... import ()`` list would be empty and invalid Python, - so import the extension module itself instead, keeping the file valid. + so import the extension module itself instead, keeping the file valid. The + import is aliased to a private name: a plain ``import package.extension`` + binds the top-level ``package`` name, which the sibling __init__'s + ``from ._generated import *`` would then leak into the package API. """ if not names: - return f"import {package}.{compiled_module} # noqa: F401" + return f"import {package}.{compiled_module} as _extension # noqa: F401" body = "".join(f" {name},\n" for name in sorted(names)) return f"from {package}.{compiled_module} import (\n{body})" @@ -254,11 +261,13 @@ def generate_shared_module_split(model: dict, layout: dict, overwrite: bool) -> # be matched to a class (with its instantiations), an enum or a free function. classes_by_base = {} other_names = set() + enum_exports = {} # enum name -> enumerators it binds at module scope for module in model["modules"]: for class_info in module["classes"]: classes_by_base[class_info["base"]] = class_info for name in module["enums"] + module["free_functions"]: other_names.add(name) + enum_exports.update(module.get("enum_exports", {})) assigned = set() flatten_names = {} # subpackage -> the top-level names it exposes (issue #73) @@ -276,6 +285,12 @@ def generate_shared_module_split(model: dict, layout: dict, overwrite: bool) -> elif name in other_names: import_names.append(name) exported.append(name) + # A value-exporting enum also binds its enumerators at module + # scope; import/re-export them alongside the enum so e.g. + # subpackage.CIRCLE keeps working under the split. + for enumerator in enum_exports.get(name, []): + import_names.append(enumerator) + exported.append(enumerator) else: print(f"warning: '{name}' ({subpkg}) not found in model", file=sys.stderr) diff --git a/cppwg/utils/package_model.py b/cppwg/utils/package_model.py index ceadaa6..0b78c45 100644 --- a/cppwg/utils/package_model.py +++ b/cppwg/utils/package_model.py @@ -43,10 +43,13 @@ def build_package_model(package_info: "PackageInfo") -> dict[str, Any]: dict[str, Any] ``{"package": name, "modules": [{"name", "compiled_module", "imports", "classes": [{"base", "templated", "instantiations": [{"args", "py_name"}]}], - "enums": [...], "free_functions": [...]}]}``. Excluded entities are + "enums": [...], "enum_exports": {enum: [enumerator, ...]}, + "free_functions": [...]}]}``. ``enum_exports`` maps each value-exporting + enum to the enumerators it binds at module scope. Excluded entities are omitted (they are not wrapped). An untemplated class has ``templated: - false`` and a single instantiation with empty ``args``; a templated - instantiation additionally carries ``cpp_name`` (its C++ type name). + false`` and a single instantiation with empty ``args``. Every + instantiation carries ``cpp_name`` (its C++ type name), which differs + from ``py_name`` when the class has a name_override. """ modules = [] for module in package_info.module_collection: @@ -73,8 +76,15 @@ def build_package_model(package_info: "PackageInfo") -> dict[str, Any]: ] templated = True else: + # Carry cpp_name for the untemplated case too, so a renamed + # untemplated class (name_override) used as a template argument + # resolves: the model records its C++ name (e.g. OldName) and the + # package-layer generator maps OldName -> the Python name. instantiations = [ - {"args": [], "py_name": py_name} for py_name in class_info.py_names + {"args": [], "cpp_name": cpp_name, "py_name": py_name} + for cpp_name, py_name in zip( + class_info.cpp_names, class_info.py_names + ) ] templated = False @@ -95,6 +105,19 @@ def build_package_model(package_info: "PackageInfo") -> dict[str, Any]: for enum_info in module.enum_collection if not enum_info.excluded ] + # An enum with .export_values() also binds its enumerators at module + # scope (e.g. ShapeKind exports CIRCLE). A shared-module split imports + # each name explicitly, so it must also import those enumerators or + # subpackage.CIRCLE would vanish. Record them per exporting enum. + enum_exports = { + (enum_info.name_override or enum_info.name): [ + value[0] for value in enum_info.decls[0].values + ] + for enum_info in module.enum_collection + if not enum_info.excluded + and enum_info.decls + and enum_info.should_export_values() + } # Skip config-excluded functions and those the writer drops for an # excluded arg/return type (free_function_is_excluded) - a dropped # function emits no binding, so it must not be recorded as wrapped. @@ -114,6 +137,7 @@ def build_package_model(package_info: "PackageInfo") -> dict[str, Any]: "imports": list(module.imports), "classes": classes, "enums": enums, + "enum_exports": enum_exports, "free_functions": free_functions, } ) diff --git a/examples/cells/dynamic/wrappers/cppwg_package_model.json b/examples/cells/dynamic/wrappers/cppwg_package_model.json index bc9caea..d9d5364 100644 --- a/examples/cells/dynamic/wrappers/cppwg_package_model.json +++ b/examples/cells/dynamic/wrappers/cppwg_package_model.json @@ -8,6 +8,7 @@ "instantiations": [ { "args": [], + "cpp_name": "Cell", "py_name": "Cell" } ], @@ -152,6 +153,7 @@ "instantiations": [ { "args": [], + "cpp_name": "PetscUtils", "py_name": "PetscUtils" } ], @@ -241,6 +243,7 @@ } ], "compiled_module": "_pycells_all", + "enum_exports": {}, "enums": [], "free_functions": [], "imports": [], diff --git a/examples/shapes/wrapper/cppwg_package_model.json b/examples/shapes/wrapper/cppwg_package_model.json index a3246c1..93d36d9 100644 --- a/examples/shapes/wrapper/cppwg_package_model.json +++ b/examples/shapes/wrapper/cppwg_package_model.json @@ -4,6 +4,7 @@ { "classes": [], "compiled_module": "_pyshapes_math_funcs", + "enum_exports": {}, "enums": [], "free_functions": [ "add", @@ -37,6 +38,7 @@ } ], "compiled_module": "_pyshapes_geometry", + "enum_exports": {}, "enums": [], "free_functions": [], "imports": [], @@ -129,6 +131,7 @@ "instantiations": [ { "args": [], + "cpp_name": "Cuboid", "py_name": "Cuboid" } ], @@ -139,6 +142,7 @@ "instantiations": [ { "args": [], + "cpp_name": "Rectangle", "py_name": "Rectangle" } ], @@ -149,6 +153,7 @@ "instantiations": [ { "args": [], + "cpp_name": "ShapeClassifier", "py_name": "ShapeClassifier" } ], @@ -159,6 +164,7 @@ "instantiations": [ { "args": [], + "cpp_name": "ShapeMetrics", "py_name": "ShapeMetrics" } ], @@ -169,6 +175,7 @@ "instantiations": [ { "args": [], + "cpp_name": "SquareFeet", "py_name": "SquareFeet" } ], @@ -179,6 +186,7 @@ "instantiations": [ { "args": [], + "cpp_name": "SquareMetres", "py_name": "SquareMetres" } ], @@ -189,6 +197,7 @@ "instantiations": [ { "args": [], + "cpp_name": "UnitSquare", "py_name": "UnitSquare" } ], @@ -196,6 +205,13 @@ } ], "compiled_module": "_pyshapes_primitives", + "enum_exports": { + "ShapeKind": [ + "CIRCLE", + "SQUARE", + "TRIANGLE" + ] + }, "enums": [ "Handedness", "ShapeKind" @@ -213,6 +229,7 @@ "instantiations": [ { "args": [], + "cpp_name": "Square", "py_name": "Square" } ], @@ -220,6 +237,7 @@ } ], "compiled_module": "_pyshapes_composites", + "enum_exports": {}, "enums": [], "free_functions": [], "imports": [ diff --git a/tests/test_genpackage.py b/tests/test_genpackage.py index ef47b8d..8843cc0 100644 --- a/tests/test_genpackage.py +++ b/tests/test_genpackage.py @@ -110,6 +110,26 @@ def test_build_cpp_to_pyname_uses_cpp_name_for_name_override(): assert index == {"OldName<2>": "NewName_2"} +def test_build_cpp_to_pyname_indexes_untemplated_name_override(): + """A renamed untemplated class is indexed by its C++ name, so it resolves as + a template argument (C++ OldName exposed as NewName).""" + model = { + "modules": [ + { + "classes": [ + _class( + "NewName", + [_inst([], "NewName", cpp_name="OldName")], + templated=False, + ), + ] + } + ] + } + index = genpackage._build_cpp_to_pyname(model) + assert index == {"OldName": "NewName"} # a nested OldName arg -> NewName + + def test_render_generated_module_with_stub_imports_syntax(): point = _class("Point", [_inst(["2"], "Point_2"), _inst(["3"], "Point_3")]) content = genpackage.render_generated_module( @@ -237,6 +257,41 @@ def test_generate_shared_module_split(tmp_path, capsys): assert "class PottsMesh(TemplateClass):" in mesh +def test_shared_split_imports_exported_enumerators(tmp_path): + """A value-exporting enum drags its enumerators into the owning subpackage.""" + model = { + "package": "pkg", + "modules": [ + { + "name": "all", + "compiled_module": "_pkg_all", + "imports": [], + "classes": [], + "enums": ["ShapeKind"], + "enum_exports": {"ShapeKind": ["CIRCLE", "SQUARE"]}, + "free_functions": [], + } + ], + } + layout = { + "package": "pkg", + "package_root": str(tmp_path), + "compiled_module": "_pkg_all", + "flatten_to_root": True, + "subpackages": {"geometry": ["ShapeKind"]}, + } + + genpackage.generate_shared_module_split(model, layout, overwrite=False) + + geometry = (tmp_path / "geometry" / "_generated.py").read_text() + # The enum and its exported enumerators are all imported into the subpackage. + assert "ShapeKind," in geometry + assert "CIRCLE," in geometry and "SQUARE," in geometry + # ...and re-exported at the top level so pkg.CIRCLE also works. + root = (tmp_path / "_generated.py").read_text() + assert '"CIRCLE",' in root and '"SQUARE",' in root and '"ShapeKind",' in root + + def test_flatten_to_root(tmp_path): """flatten_to_root emits a top-level _generated.py re-exporting every name.""" model = { @@ -465,11 +520,20 @@ def test_shared_split_empty_subpackage_emits_valid_import(tmp_path): genpackage.generate_shared_module_split(model, layout, overwrite=False) # b would otherwise be `from pkg._pkg_all import (\n)`, a SyntaxError; it - # must be a plain, valid module import instead. + # must be a plain, valid module import instead, aliased to a private name so + # `from ._generated import *` does not leak the top-level `pkg` name. b = (tmp_path / "b" / "_generated.py").read_text() - assert "import pkg._pkg_all" in b + assert "import pkg._pkg_all as _extension" in b assert "import (" not in b - ast.parse(b) # the whole file parses + tree = ast.parse(b) # the whole file parses + # The only name bound is private (underscore-prefixed) -> not star-exported. + bound = { + (alias.asname or alias.name.split(".")[0]) + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for alias in node.names + } + assert bound == {"_extension"} # `pkg` is not bound, so it cannot leak def test_warn_orphans_flags_only_unwritten_banner_files(tmp_path, capsys): diff --git a/tests/test_package_model.py b/tests/test_package_model.py index b71c3ef..f527fcb 100644 --- a/tests/test_package_model.py +++ b/tests/test_package_model.py @@ -15,8 +15,15 @@ def _class(base, py_names, template_arg_lists=(), cpp_names=None, excluded=False ) -def _enum(name, name_override="", excluded=False): - return SimpleNamespace(name=name, name_override=name_override, excluded=excluded) +def _enum(name, name_override="", excluded=False, exported_values=None): + values = list(exported_values or []) + return SimpleNamespace( + name=name, + name_override=name_override, + excluded=excluded, + decls=[SimpleNamespace(values=[(v, i) for i, v in enumerate(values)])], + should_export_values=lambda values=values: bool(values), + ) def _free_function(name, excluded=False, arg_types=(), return_type="void", excludes=None): @@ -106,7 +113,9 @@ def test_build_model_templated_and_untemplated(): assert unit_square == { "base": "UnitSquare", "templated": False, - "instantiations": [{"args": [], "py_name": "UnitSquare"}], + "instantiations": [ + {"args": [], "cpp_name": "UnitSquare", "py_name": "UnitSquare"} + ], } assert primitives["enums"] == ["ShapeKind"] @@ -198,6 +207,23 @@ def test_instantiation_carries_cpp_name_for_name_override(): ] +def test_untemplated_name_override_carries_cpp_name(): + """A renamed untemplated class records its C++ name, distinct from py_name. + + This lets the package-layer generator resolve the class when it appears as a + template argument spelled with its C++ name (OldName -> NewName). + """ + package = _package( + "pkg", + [_module("mod", classes=[_class("NewName", ["NewName"], cpp_names=["OldName"])])], + ) + (module,) = build_package_model(package)["modules"] + (cls,) = module["classes"] + assert cls["instantiations"] == [ + {"args": [], "cpp_name": "OldName", "py_name": "NewName"} + ] + + def test_build_model_omits_free_function_excluded_by_type(): """A free function the writer drops for an excluded arg type is not recorded. @@ -231,3 +257,24 @@ def test_enum_name_override_used(): ) (module,) = build_package_model(package)["modules"] assert module["enums"] == ["PyName"] + # A non-exporting enum contributes no enum_exports entry. + assert module["enum_exports"] == {} + + +def test_enum_exports_records_exported_enumerators(): + """A value-exporting enum records its enumerators, keyed by the enum py-name.""" + package = _package( + "pkg", + [ + _module( + "mod", + enums=[ + _enum("ShapeKind", exported_values=["CIRCLE", "SQUARE"]), + _enum("Scoped"), # scoped/non-exporting -> not recorded + ], + ) + ], + ) + (module,) = build_package_model(package)["modules"] + assert module["enums"] == ["ShapeKind", "Scoped"] + assert module["enum_exports"] == {"ShapeKind": ["CIRCLE", "SQUARE"]}