diff --git a/.github/workflows/test-cells-conda.yml b/.github/workflows/test-cells-conda.yml index e4a7dc3f..c94e4c14 100644 --- a/.github/workflows/test-cells-conda.yml +++ b/.github/workflows/test-cells-conda.yml @@ -105,7 +105,10 @@ jobs: COVERAGE_FILE: ${{ github.workspace }}/.coverage run: | python3 -m coverage combine --rcfile=.cov_sub.rc - python3 -m coverage xml --rcfile=.cov_sub.rc + # Report with the repo's coverage config so [tool.coverage.report] + # exclude_also applies (.cov_sub.rc only sets [run]); otherwise excluded + # lines in cppwg modules this flag never imports are reported as missed. + python3 -m coverage xml --rcfile="$GITHUB_WORKSPACE/pyproject.toml" - name: Upload coverage to Codecov if: matrix.python-version == '3.12' diff --git a/.github/workflows/test-shapes-pip.yml b/.github/workflows/test-shapes-pip.yml index 018eb4e7..80a36204 100644 --- a/.github/workflows/test-shapes-pip.yml +++ b/.github/workflows/test-shapes-pip.yml @@ -60,7 +60,11 @@ jobs: - name: Generate coverage report if: matrix.python-version == '3.12' - run: python3 -m coverage xml + # Use the repo's coverage config so [tool.coverage.report] exclude_also + # applies here too (this runs from examples/shapes, where it isn't found); + # otherwise excluded lines (e.g. `if __name__`) in cppwg modules this flag + # never imports are wrongly reported as missed, failing codecov's patch gate. + run: python3 -m coverage xml --rcfile="$GITHUB_WORKSPACE/pyproject.toml" working-directory: examples/shapes - name: Upload coverage to Codecov diff --git a/cppwg/__main__.py b/cppwg/__main__.py index 0c7293c7..b9daf056 100644 --- a/cppwg/__main__.py +++ b/cppwg/__main__.py @@ -2,6 +2,7 @@ import argparse import logging +import sys from datetime import datetime from pathlib import Path @@ -54,6 +55,7 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( prog="cppwg", description="Generate Python Wrappers for C++ code", + epilog="Run 'cppwg genpackage --help' for the Python package-layer generator.", ) parser.add_argument( @@ -193,7 +195,18 @@ def generate(args: argparse.Namespace) -> None: def main() -> None: - """Generate wrappers from command line arguments.""" + """Generate wrappers from command line arguments. + + ``cppwg genpackage ...`` is dispatched to the package-layer generator + (:mod:`cppwg.genpackage`); any other invocation runs wrapper generation as + usual. The subcommand is intercepted before argument parsing so the normal + CLI (a leading ``SOURCE_ROOT`` positional) is entirely unaffected. + """ + if len(sys.argv) > 1 and sys.argv[1] == "genpackage": + from cppwg.genpackage import main as genpackage_main + + raise SystemExit(genpackage_main(sys.argv[2:])) + args = parse_args() log_handlers = [] diff --git a/cppwg/generators.py b/cppwg/generators.py index 360528c5..80f94a0a 100644 --- a/cppwg/generators.py +++ b/cppwg/generators.py @@ -1,5 +1,6 @@ """The main interface for generating Python wrappers.""" +import json import logging import os import re @@ -18,7 +19,10 @@ from cppwg.utils.constants import ( CPPWG_DEFAULT_WRAPPER_DIR, CPPWG_HEADER_COLLECTION_FILENAME, + CPPWG_PACKAGE_MODEL_FILENAME, + CPPWG_PACKAGE_MODEL_NOTICE, ) +from cppwg.utils.package_model import build_package_model from cppwg.version import __version__ as cppwg_version from cppwg.writers.header_collection_writer import CppHeaderCollectionWriter from cppwg.writers.package_writer import CppPackageWrapperWriter @@ -422,6 +426,26 @@ def write_wrappers(self) -> None: ) package_writer.write() + def write_package_model(self) -> None: + """ + Write the package model (cppwg_package_model.json) to the wrapper root. + + A small JSON description of the generated modules and their classes / + instantiations / enums / free functions, so a separate step (the + ``cppwg genpackage`` subcommand) can generate the Python package layer + without re-parsing the source. Written last, once the info tree is final. + JSON is used deliberately: this is a machine-generated, git-diff-checked + artifact, so a deterministic serialization matters more than YAML's + readability (the hand-authored configs stay YAML). + """ + model = build_package_model(self.package_info) + # A do-not-edit notice as `_comment` (JSON has no comments); the generator + # ignores unknown top-level keys. + model = {"_comment": CPPWG_PACKAGE_MODEL_NOTICE, **model} + model_path = os.path.join(self.wrapper_root, CPPWG_PACKAGE_MODEL_FILENAME) + content = json.dumps(model, indent=2, sort_keys=True) + "\n" + utils.write_file_if_changed(model_path, content, self.overwrite) + def generate(self) -> None: """ Parse yaml configuration and C++ source to generate Python wrappers. @@ -482,3 +506,7 @@ def generate(self) -> None: # Write the wrapper code for the package self.write_wrappers() + + # Write the package model (cppwg_package_model.json) for the package-layer + # generator (cppwg genpackage). + self.write_package_model() diff --git a/cppwg/genpackage.py b/cppwg/genpackage.py new file mode 100644 index 00000000..008b23f8 --- /dev/null +++ b/cppwg/genpackage.py @@ -0,0 +1,426 @@ +#!/usr/bin/env python3 +"""Generate the Python package layer (``_generated.py``) for a cppwg project. + +cppwg generates the C++/pybind11 wrappers and, alongside them, a JSON model of +what it produced (``cppwg_package_model.json`` in the wrapper root - see +``cppwg.utils.package_model``). This module (run via ``cppwg genpackage``) turns +that model, plus a small package-layout file, into a ``_generated.py`` per +Python subpackage: the +compiled-extension import and the ``TemplateClass`` subscript stubs +(``Point[2] -> Point_2``). It never touches the hand-written ``__init__.py``, +which does ``from ._generated import *`` and adds the bespoke pieces +(``TemplateMethod`` attachments, package ``init()``, curation, comments). + +Two layouts: + +* module-per-subpackage (default): each cppwg module becomes a subpackage that + owns its own compiled extension, imported with ``from . import *``. + Used by the shapes/cells examples. +* shared-module split: one compiled extension (e.g. ``_pychaste_all``) is split + into several subpackages by a layout that lists which names each owns; + imported explicitly with ``from . import (...)``. Used by + pychaste. + +Usage:: + + cppwg genpackage --model wrapper/cppwg_package_model.json --layout package_layout.yaml +""" + +import argparse +import json +import os +import sys + +from cppwg.utils.utils import write_file_if_changed + + +GENERATED_HEADER = ( + '"""Generated by cppwg genpackage - do not edit.\n' + "\n" + "Compiled-extension imports and TemplateClass subscript stubs for this\n" + "subpackage. Hand-written code (TemplateMethod attachments, package init,\n" + 'curation) belongs in the sibling __init__.py, which does `from ._generated\n' + 'import *`.\n' + '"""\n' +) + + +FLATTEN_HEADER = ( + '"""Generated by cppwg genpackage - do not edit.\n' + "\n" + "Re-exports every wrapped class, enum and free function from the subpackages\n" + "into the top-level package namespace, so `package.ClassName` works as well\n" + "as `package.subpackage.ClassName`. Importing this imports the subpackages.\n" + "The hand-written __init__.py does `from ._generated import *` and adds the\n" + "bespoke top-level pieces (package init, etc.).\n" + '"""\n' +) + + +def _render_key(args: list[str]) -> str: + """Render a template-argument list as a Python tuple literal of strings. + + e.g. ``["2"]`` -> ``("2",)`` and ``["Cell", "2"]`` -> ``("Cell", "2")``. + Matches _syntax._normalize_key, which normalizes keys to string tuples. Each + value is rendered with ``json.dumps`` so any special characters are escaped + into a valid (double-quoted) Python string literal. + """ + inner = ", ".join(json.dumps(arg) for arg in args) + if len(args) == 1: + inner += "," + return f"({inner})" + + +def _build_cpp_to_pyname(model: dict) -> dict: + """Map each wrapped instantiation's C++ type string to its Python class name. + + e.g. ``PottsMesh<2> -> PottsMesh_2``. Used to key a template argument that is + itself a wrapped templated type (``MeshFactory>``) by the Python + concrete class name, so ``MeshFactory[PottsMesh[2]]`` resolves: ``PottsMesh[2]`` + is the ``PottsMesh_2`` class and ``_normalize_key`` keys it by its ``__name__``. + + Keyed by the instantiation's real C++ type name (``cpp_name``) so a class with + a name_override (whose Python name differs from its C++ name) still resolves - + including an untemplated renamed class (``OldName`` exposed as ``NewName``) + used as a template argument. Older models without ``cpp_name`` fall back to + reconstructing it from the base name and arguments (templated only). + """ + index = {} + for module in model["modules"]: + for class_info in module["classes"]: + for inst in class_info["instantiations"]: + cpp = inst.get("cpp_name") + if cpp is None and inst["args"]: + # Older model without cpp_name: reconstruct base. An + # untemplated class in such a model carries no C++ name, so + # it cannot be indexed. + cpp = f'{class_info["base"]}<{",".join(inst["args"])}>' + if cpp is not None: + index[cpp] = inst["py_name"] + return index + + +def _render_template_stub( + base: str, + instantiations: list[dict], + diagonal_shorthand: bool = False, + cpp_to_pyname: dict = None, +) -> str: + """Render a ``class (TemplateClass)`` stub for a templated class. + + A template argument that is itself a wrapped templated type is keyed by its + Python concrete class name via ``cpp_to_pyname`` (``PottsMesh<2>`` -> + ``PottsMesh_2``), so ``MeshFactory[PottsMesh[2]]`` resolves. + + When ``diagonal_shorthand`` is set, a multi-argument instantiation whose + arguments are all equal (a "diagonal", e.g. ``Element<2, 2>``) also gets a + single-argument alias key (``Element[2] -> Element_2_2``). This is an + opt-in Python-layer convenience some packages (pychaste) adopt for their + ````-style classes; others (the cells example) keep + the explicit multi-argument form only. + """ + cpp_to_pyname = cpp_to_pyname or {} + lines = [f"class {base}(TemplateClass):", " _instantiations = {"] + for inst in instantiations: + args = [cpp_to_pyname.get(arg, arg) for arg in inst["args"]] + lines.append(f' {_render_key(args)}: {inst["py_name"]},') + if diagonal_shorthand and len(args) > 1 and len(set(args)) == 1: + lines.append(f' {_render_key(args[:1])}: {inst["py_name"]},') + lines.append(" }") + return "\n".join(lines) + + +def render_generated_module( + package: str, + compiled_import: str, + classes: list[dict], + templated_classes: list[dict], + diagonal_shorthand: bool = False, + cpp_to_pyname: dict = None, +) -> str: + """ + Render a subpackage's ``_generated.py`` content. + + Parameters + ---------- + package : str + The Python import root (e.g. ``pyshapes``, ``chaste``), used to import + the shared ``_syntax`` helper. + compiled_import : str + The import statement pulling in the compiled extension's names. + classes : list[dict] + The subpackage's classes (from the model); used to know whether any + TemplateClass import is needed. + templated_classes : list[dict] + The subset that is templated, each rendered as a stub. + + Returns + ------- + str + The file content (ending with a newline). + """ + # Emit black-style output directly, the way the C++ templates emit clean C++: + # docstring, one blank line, the imports, then each stub separated by two + # blank lines. cppwg runs no formatter, so the layout built here is final. + lines = [GENERATED_HEADER.rstrip("\n"), "", compiled_import] + if templated_classes: + lines.append(f"from {package}._syntax import TemplateClass") + for class_info in templated_classes: + lines.extend(["", ""]) # two blank lines before each top-level class + lines.append( + _render_template_stub( + class_info["base"], + class_info["instantiations"], + diagonal_shorthand, + cpp_to_pyname, + ) + ) + return "\n".join(lines) + "\n" + + +def _concrete_py_names(class_info: dict) -> list[str]: + """Return the concrete py_names of a class's instantiations.""" + return [inst["py_name"] for inst in class_info["instantiations"]] + + +def _explicit_import(package: str, compiled_module: str, names: list[str]) -> str: + """Render an import of ``names`` from the shared compiled extension. + + With no names (an empty subpackage, or one whose names are all absent from + the model) a ``from ... import ()`` list would be empty and invalid Python, + so import the extension module itself instead, keeping the file valid. The + import is aliased to a private name: a plain ``import package.extension`` + binds the top-level ``package`` name, which the sibling __init__'s + ``from ._generated import *`` would then leak into the package API. + """ + if not names: + return f"import {package}.{compiled_module} as _extension # noqa: F401" + body = "".join(f" {name},\n" for name in sorted(names)) + return f"from {package}.{compiled_module} import (\n{body})" + + +def _write_generated(path: str, content: str, overwrite: bool) -> str: + """Write a ``_generated.py`` (creating parent dirs) and return its abspath. + + Reports whether the file was written or left unchanged. The returned path + lets the caller record which outputs are live, so stale ones can be flagged + (see ``_warn_orphans``). + """ + os.makedirs(os.path.dirname(path), exist_ok=True) + if write_file_if_changed(path, content, overwrite): + print(f"wrote {path}") + else: + print(f"unchanged {path}") + return os.path.abspath(path) + + +def generate_module_per_subpackage(model: dict, layout: dict, overwrite: bool) -> list: + """Emit one ``_generated.py`` per cppwg module (each owns its own extension). + + Returns the list of files written, so the caller can flag orphaned outputs. + """ + package = layout.get("package", model["package"]) + package_root = layout["package_root"] + module_dirs = layout.get("module_dirs", {}) + diagonal_shorthand = layout.get("diagonal_shorthand", False) + cpp_to_pyname = _build_cpp_to_pyname(model) + + written = [] + for module in model["modules"]: + # The subpackage directory (relative to package_root); default = module + # name, overridable (e.g. a single "all" module living at the root). + subdir = module_dirs.get(module["name"], module["name"]) + compiled_import = f"from .{module['compiled_module']} import *" + templated = [c for c in module["classes"] if c["templated"]] + content = render_generated_module( + package, + compiled_import, + module["classes"], + templated, + diagonal_shorthand, + cpp_to_pyname, + ) + path = os.path.join(package_root, subdir, "_generated.py") + written.append(_write_generated(path, content, overwrite)) + return written + + +def generate_shared_module_split(model: dict, layout: dict, overwrite: bool) -> list: + """Emit a ``_generated.py`` per subpackage, splitting one shared extension. + + Returns the list of files written, so the caller can flag orphaned outputs. + """ + package = layout.get("package", model["package"]) + package_root = layout["package_root"] + compiled_module = layout["compiled_module"] + diagonal_shorthand = layout.get("diagonal_shorthand", False) + cpp_to_pyname = _build_cpp_to_pyname(model) + written = [] + + # Index every wrapped entity across the model by name so a layout entry can + # be matched to a class (with its instantiations), an enum or a free function. + classes_by_base = {} + other_names = set() + enum_exports = {} # enum name -> enumerators it binds at module scope + for module in model["modules"]: + for class_info in module["classes"]: + classes_by_base[class_info["base"]] = class_info + for name in module["enums"] + module["free_functions"]: + other_names.add(name) + enum_exports.update(module.get("enum_exports", {})) + + assigned = set() + flatten_names = {} # subpackage -> the top-level names it exposes (issue #73) + for subpkg, names in layout["subpackages"].items(): + classes = [] + import_names = [] + exported = [] + for name in names: + assigned.add(name) + if name in classes_by_base: + class_info = classes_by_base[name] + classes.append(class_info) + import_names.extend(_concrete_py_names(class_info)) + exported.append(name) + elif name in other_names: + import_names.append(name) + exported.append(name) + # A value-exporting enum also binds its enumerators at module + # scope; import/re-export them alongside the enum so e.g. + # subpackage.CIRCLE keeps working under the split. + for enumerator in enum_exports.get(name, []): + import_names.append(enumerator) + exported.append(enumerator) + else: + print(f"warning: '{name}' ({subpkg}) not found in model", file=sys.stderr) + + templated = [c for c in classes if c["templated"]] + compiled_import = _explicit_import(package, compiled_module, import_names) + content = render_generated_module( + package, + compiled_import, + classes, + templated, + diagonal_shorthand, + cpp_to_pyname, + ) + path = os.path.join(package_root, subpkg, "_generated.py") + written.append(_write_generated(path, content, overwrite)) + flatten_names[subpkg] = exported + + # Flag any wrapped class not placed in a subpackage, so nothing is silently + # dropped when the config gains a class. + unplaced = sorted(set(classes_by_base) | other_names) + for name in unplaced: + if name not in assigned: + print(f"warning: '{name}' is wrapped but not assigned to a subpackage", + file=sys.stderr) + + if layout.get("flatten_to_root"): + written.append( + generate_root_flatten(package, package_root, flatten_names, overwrite) + ) + return written + + +def generate_root_flatten( + package: str, package_root: str, flatten_names: dict, overwrite: bool +) -> str: + """Emit a top-level ``_generated.py`` re-exporting every subpackage's names. + + Gives ``package.ClassName`` in addition to ``package.subpackage.ClassName`` + (issue #73). Names are expected to be unique across subpackages; a clash is + warned about (the last subpackage's binding would win at import time). + """ + owner = {} # name -> subpackage that first exported it, for a clash check + for subpkg in sorted(flatten_names): + for name in flatten_names[subpkg]: + if name in owner: + print( + f"warning: '{name}' is exported by both '{owner[name]}' and " + f"'{subpkg}'; top-level {package}.{name} would be ambiguous", + file=sys.stderr, + ) + else: + owner[name] = subpkg + + lines = [FLATTEN_HEADER.rstrip("\n"), ""] + for subpkg in sorted(flatten_names): + names = flatten_names[subpkg] + if names: + lines.append(_explicit_import(package, subpkg, names)) + lines.append("") + lines.append("__all__ = [") + for name in sorted(owner): + lines.append(f' "{name}",') + lines.append("]") + content = "\n".join(lines) + "\n" + return _write_generated( + os.path.join(package_root, "_generated.py"), content, overwrite + ) + + +def _warn_orphans(package_root: str, written: list) -> None: + """Warn about generated files under ``package_root`` this run did not write. + + A ``_generated.py`` carrying the do-not-edit banner but no longer a target + means its module/subpackage was removed from the config. cppwg only ever + generates ``_generated.py`` (never the hand-written ``__init__.py``), so it + cannot fully remove the orphaned subpackage; it flags it for the user to + delete rather than silently leaving stale bindings or deleting their files. + """ + written = {os.path.abspath(p) for p in written} + banner = "Generated by cppwg genpackage" + for dirpath, _dirs, files in os.walk(package_root): + if "_generated.py" not in files: + continue + path = os.path.abspath(os.path.join(dirpath, "_generated.py")) + if path in written: + continue + with open(path) as generated_file: + if banner not in generated_file.read(200): + continue # not one of ours -> never touch it + print( + f"warning: {path} is generated but no longer a target; its " + "module/subpackage was removed - delete the containing directory", + file=sys.stderr, + ) + + +def main(argv=None) -> int: + """Entry point: read the model + layout and write the _generated.py files.""" + parser = argparse.ArgumentParser(prog="cppwg genpackage", description=__doc__) + parser.add_argument( + "--model", required=True, help="Path to cppwg_package_model.json (from cppwg)." + ) + parser.add_argument( + "--layout", required=True, help="Path to the Python package-layout file (YAML)." + ) + parser.add_argument( + "--overwrite", + action="store_true", + help="Rewrite files even if unchanged.", + ) + args = parser.parse_args(argv) + + import yaml + + with open(args.model) as model_file: + model = json.load(model_file) # the model is JSON (see cppwg.utils.constants) + with open(args.layout) as layout_file: + layout = yaml.safe_load(layout_file) + + # Resolve a relative package_root against the layout's own directory, so a + # layout is portable regardless of the caller's working directory. + if not os.path.isabs(layout["package_root"]): + layout_dir = os.path.dirname(os.path.abspath(args.layout)) + layout["package_root"] = os.path.normpath( + os.path.join(layout_dir, layout["package_root"]) + ) + + if layout.get("subpackages"): + written = generate_shared_module_split(model, layout, args.overwrite) + else: + written = generate_module_per_subpackage(model, layout, args.overwrite) + _warn_orphans(layout["package_root"], written) + return 0 diff --git a/cppwg/info/base_info.py b/cppwg/info/base_info.py index dcc26fa8..4f2bbeb1 100644 --- a/cppwg/info/base_info.py +++ b/cppwg/info/base_info.py @@ -19,11 +19,11 @@ # defaults and copies any the config overrides, and the parser # (cppwg.parsers.package_info_parser) builds its config dicts from the same # schema. An option added here is therefore understood everywhere - defined in -# one place instead of being restated in BaseInfo and the parser (which is how -# options such as name_replacements previously became unreachable from the YAML). -# Mutable defaults are deep-copied per use so no two objects share a list/dict. -# See the class Attributes docstring for what each option means; tri-state -# options default to None, meaning "inherit from further up the info tree". +# one place instead of being restated in BaseInfo and the parser (which can +# cause options to become unreachable from the YAML). Mutable defaults are +# deep-copied per use so no two objects share a list/dict. See the class +# Attributes docstring for what each option means; tri-state options default to +# None, meaning "inherit from further up the info tree". BASE_INFO_OPTIONS: dict[str, Any] = { "arg_type_excludes": [], "auto_includes": None, diff --git a/cppwg/info/class_info.py b/cppwg/info/class_info.py index a00dbbec..9c6ce407 100644 --- a/cppwg/info/class_info.py +++ b/cppwg/info/class_info.py @@ -9,7 +9,7 @@ from cppwg.info.cpp_entity_info import CppEntityInfo from cppwg.utils import utils -from cppwg.utils.constants import CPPWG_EXT +from cppwg.utils.constants import CPPWG_EXT, CPPWG_TEMPLATE_ARG_SEPARATOR if TYPE_CHECKING: from pygccxml.declarations import declaration_t @@ -589,10 +589,11 @@ def update_py_names(self) -> None: Set the Python names for the class, accounting for template args. Set the name(s) of the class as it should appear in Python. This - collapses template arguments, separates them by underscores, and removes - special characters. There can be multiple names, one for each template - class instantiation. For example, class "Foo" with template arguments - [[2, 2], [3, 3]] will have a Python name list ["Foo_2_2", "Foo_3_3"]. + collapses template arguments, separates them by CPPWG_TEMPLATE_ARG_SEPARATOR + (default "_"), and removes special characters. There can be multiple names, + one for each template class instantiation. For example, class "Foo" with + template arguments [[2, 2], [3, 3]] will have a Python name list + ["Foo_2_2", "Foo_3_3"]. """ class_name = self.py_name_base() @@ -601,22 +602,24 @@ class instantiation. For example, class "Foo" with template arguments self.py_names.append(class_name) return - # Create a string of template args separated by "_" e.g. 2_2 + separator = CPPWG_TEMPLATE_ARG_SEPARATOR + + # Create a string of template args separated by `separator`, e.g. 2_2 for template_arg_list in self.template_arg_lists: # Example template_arg_list : [2, 2] template_string = "" for idx, arg in enumerate(template_arg_list): - # A nested template arg keeps its structure via "_" separators, - # e.g. PottsMesh<2> -> PottsMesh_2 (separator="_"). - arg_str = self._mangle_py_token(str(arg), separator="_") + # A nested template arg keeps its structure via `separator`, + # e.g. PottsMesh<2> -> PottsMesh_2. + arg_str = self._mangle_py_token(str(arg), separator=separator) - # Add "_" between template arguments + # Add the separator between template arguments template_string += arg_str if idx < len(template_arg_list) - 1: - template_string += "_" + template_string += separator - self.py_names.append(class_name + "_" + template_string) + self.py_names.append(class_name + separator + template_string) def update_cpp_names(self) -> None: """ diff --git a/cppwg/info/exclusions.py b/cppwg/info/exclusions.py index c685ecfa..13768e85 100644 --- a/cppwg/info/exclusions.py +++ b/cppwg/info/exclusions.py @@ -25,6 +25,7 @@ from pygccxml.declarations.variable import variable_t from cppwg.info.class_info import CppClassInfo + from cppwg.info.free_function_info import CppFreeFunctionInfo def method_is_excluded( @@ -291,3 +292,52 @@ def variable_is_excluded( True if the member should be excluded, False otherwise. """ return variable_exclusion_reason(class_info, class_decl, variable_decl) is not None + + +def free_function_is_excluded(free_function_info: "CppFreeFunctionInfo") -> bool: + """ + Return True if a free function would be excluded from the wrapper code. + + A free function is dropped when its return type or any argument type matches + return_type_excludes / arg_type_excludes (the deprecated calldef_excludes + applies to both). Shared by CppFreeFunctionWrapperWriter and the package model + so a function whose binding is never emitted is not recorded as wrapped. + + Returns + ------- + bool + True if the function should be excluded, False otherwise. + """ + decl = free_function_info.decls[0] + + # Exclude by return type. return_type_excludes targets return types; the + # deprecated calldef_excludes applies to both return and arg types. + calldef_excludes = free_function_info.hierarchy_attribute_gather_flat( + "calldef_excludes" + ) + return_type_excludes = ( + free_function_info.hierarchy_attribute_gather_flat("return_type_excludes") + + calldef_excludes + ) + return_type = decl.return_type.decl_string + if any( + utils.type_string_matches(return_type, pattern) + for pattern in return_type_excludes + ): + return True + + # Exclude by argument type. arg_type_excludes is the general arg-type exclude; + # the deprecated calldef_excludes applies too. + arg_type_excludes = ( + free_function_info.hierarchy_attribute_gather_flat("arg_type_excludes") + + calldef_excludes + ) + for argument_type in decl.argument_types: + arg_type = argument_type.decl_string + if any( + utils.type_string_matches(arg_type, pattern) + for pattern in arg_type_excludes + ): + return True + + return False diff --git a/cppwg/utils/constants.py b/cppwg/utils/constants.py index c52527bd..b749e300 100644 --- a/cppwg/utils/constants.py +++ b/cppwg/utils/constants.py @@ -9,9 +9,30 @@ # Default log file name used when --logfile is passed without a value. CPPWG_DEFAULT_LOGFILE = f"{CPPWG_EXT}.log" +# The package model cppwg writes into the wrapper root, describing the generated +# modules/classes so a separate step can build the Python package layer (see +# cppwg.utils.package_model and cppwg genpackage). It is JSON: a +# machine-generated, git-diff-checked artifact whose serialization is +# deterministic across environments (unlike the hand-authored *.yaml configs). +CPPWG_PACKAGE_MODEL_FILENAME = "cppwg_package_model.json" + +# Do-not-edit notice stored as the model's `_comment` field (JSON has no comment +# syntax; the generator ignores the field), mirroring the banner on the generated +# wrapper sources so the committed model reads clearly as a generated artifact. +CPPWG_PACKAGE_MODEL_NOTICE = ( + "This file is automatically generated by cppwg. Do not modify this file directly." +) + CPPWG_TRUE_STRINGS = ["ON", "YES", "Y", "TRUE", "T", "1"] CPPWG_FALSE_STRINGS = ["OFF", "NO", "N", "FALSE", "F", "0", ""] CPPWG_DEFAULT_WRAPPER_DIR = "cppwg_wrappers" CPPWG_CLASS_OVERRIDE_SUFFIX = "_Overrides" + +# Separator inserted between a templated class's base name and its template +# arguments, between successive arguments, and within a nested template argument +# when building the class's Python name (e.g. Foo<2, 2> -> "Foo_2_2", +# MeshFactory> -> "MeshFactory_PottsMesh_2"). A project can widen +# this (e.g. "__") to avoid Python-name clashes with similarly-named types. +CPPWG_TEMPLATE_ARG_SEPARATOR = "_" diff --git a/cppwg/utils/package_model.py b/cppwg/utils/package_model.py new file mode 100644 index 00000000..0b78c45f --- /dev/null +++ b/cppwg/utils/package_model.py @@ -0,0 +1,145 @@ +"""Build a serialisable (plain-dict) model of the Python package layer. + +cppwg generates the C++/pybind11 wrappers; a separate step (see +``cppwg genpackage``) generates the Python package layer - the +``_generated.py`` files that import each compiled extension and define the +``TemplateClass`` subscript stubs. That step needs, per module, the compiled +module name, the wrapped classes with their template instantiations, and the +enum / free-function names. All of this is on the finalized ``PackageInfo`` tree +but not in a form a standalone script can consume, so ``build_package_model`` +distils it into a plain dict that cppwg writes out as ``cppwg_package_model.json``. +""" + +from typing import TYPE_CHECKING, Any + +from cppwg.info.exclusions import free_function_is_excluded + +if TYPE_CHECKING: + from cppwg.info.package_info import PackageInfo + + +def compiled_module_name(package_name: str, module_name: str) -> str: + """ + Return the pybind11 module name for a package/module, e.g. ``_pyshapes_geometry``. + + Mirrors CppModuleWrapperWriter.full_module_name so the model and the C++ + wrappers agree on the compiled extension name. + """ + return f"_{package_name}_{module_name}" + + +def build_package_model(package_info: "PackageInfo") -> dict[str, Any]: + """ + Distil a PackageInfo tree into a serialisable (plain-dict) Python-package model. + + Parameters + ---------- + package_info : PackageInfo + The finalized package info (after wrappers are written, so py_names and + template_arg_lists are complete). + + Returns + ------- + dict[str, Any] + ``{"package": name, "modules": [{"name", "compiled_module", "imports", + "classes": [{"base", "templated", "instantiations": [{"args", "py_name"}]}], + "enums": [...], "enum_exports": {enum: [enumerator, ...]}, + "free_functions": [...]}]}``. ``enum_exports`` maps each value-exporting + enum to the enumerators it binds at module scope. Excluded entities are + omitted (they are not wrapped). An untemplated class has ``templated: + false`` and a single instantiation with empty ``args``. Every + instantiation carries ``cpp_name`` (its C++ type name), which differs + from ``py_name`` when the class has a name_override. + """ + modules = [] + for module in package_info.module_collection: + classes = [] + for class_info in module.class_collection: + if class_info.excluded: + continue + + if class_info.template_arg_lists: + # Carry each instantiation's actual C++ type name (cpp_name), which + # differs from base when the class has a name_override; the + # package-layer generator keys nested template arguments by it. + instantiations = [ + { + "args": [str(arg) for arg in args], + "cpp_name": cpp_name, + "py_name": py_name, + } + for args, cpp_name, py_name in zip( + class_info.template_arg_lists, + class_info.cpp_names, + class_info.py_names, + ) + ] + templated = True + else: + # Carry cpp_name for the untemplated case too, so a renamed + # untemplated class (name_override) used as a template argument + # resolves: the model records its C++ name (e.g. OldName) and the + # package-layer generator maps OldName -> the Python name. + instantiations = [ + {"args": [], "cpp_name": cpp_name, "py_name": py_name} + for cpp_name, py_name in zip( + class_info.cpp_names, class_info.py_names + ) + ] + templated = False + + if not instantiations: + # A class whose every instantiation was pruned contributes nothing. + continue + + classes.append( + { + "base": class_info.py_name_base(), + "templated": templated, + "instantiations": instantiations, + } + ) + + enums = [ + enum_info.name_override or enum_info.name + for enum_info in module.enum_collection + if not enum_info.excluded + ] + # An enum with .export_values() also binds its enumerators at module + # scope (e.g. ShapeKind exports CIRCLE). A shared-module split imports + # each name explicitly, so it must also import those enumerators or + # subpackage.CIRCLE would vanish. Record them per exporting enum. + enum_exports = { + (enum_info.name_override or enum_info.name): [ + value[0] for value in enum_info.decls[0].values + ] + for enum_info in module.enum_collection + if not enum_info.excluded + and enum_info.decls + and enum_info.should_export_values() + } + # Skip config-excluded functions and those the writer drops for an + # excluded arg/return type (free_function_is_excluded) - a dropped + # function emits no binding, so it must not be recorded as wrapped. + free_functions = [ + free_function_info.name + for free_function_info in module.free_function_collection + if not free_function_info.excluded + and not free_function_is_excluded(free_function_info) + ] + + modules.append( + { + "name": module.name, + "compiled_module": compiled_module_name( + package_info.name, module.name + ), + "imports": list(module.imports), + "classes": classes, + "enums": enums, + "enum_exports": enum_exports, + "free_functions": free_functions, + } + ) + + return {"package": package_info.name, "modules": modules} diff --git a/cppwg/writers/free_function_writer.py b/cppwg/writers/free_function_writer.py index 081c8f64..5e3cdd51 100644 --- a/cppwg/writers/free_function_writer.py +++ b/cppwg/writers/free_function_writer.py @@ -2,8 +2,8 @@ from typing import TYPE_CHECKING +from cppwg.info import exclusions from cppwg.info.free_function_info import CppFreeFunctionInfo -from cppwg.utils import utils from cppwg.writers.base_writer import CppBaseWrapperWriter if TYPE_CHECKING: @@ -73,34 +73,12 @@ def exclude(self) -> bool: bool True if the function should be excluded from wrapper code, False otherwise. """ - info = self.free_function_info - decl = info.decls[0] - - # Exclude by return type. return_type_excludes targets return types; the - # deprecated calldef_excludes applies to both return and arg types. - calldef_excludes = info.hierarchy_attribute_gather_flat("calldef_excludes") - return_type_excludes = ( - info.hierarchy_attribute_gather_flat("return_type_excludes") - + calldef_excludes + # Config-excluded (YAML `excluded: true`) or dropped for an excluded + # arg/return type. The type-based rule lives in free_function_is_excluded, + # shared with the package model; the config flag is folded in here (as the + # enum writer does with enum_info.excluded) so the writer and model agree + # on the wrapped set. + return ( + self.free_function_info.excluded + or exclusions.free_function_is_excluded(self.free_function_info) ) - return_type = decl.return_type.decl_string - if any( - utils.type_string_matches(return_type, pattern) - for pattern in return_type_excludes - ): - return True - - # Exclude by argument type. arg_type_excludes is the general arg-type - # exclude; the deprecated calldef_excludes applies too. - arg_type_excludes = ( - info.hierarchy_attribute_gather_flat("arg_type_excludes") + calldef_excludes - ) - for argument_type in decl.argument_types: - arg_type = argument_type.decl_string - if any( - utils.type_string_matches(arg_type, pattern) - for pattern in arg_type_excludes - ): - return True - - return False diff --git a/doc/basics.md b/doc/basics.md index 8dee90e6..59c3120a 100644 --- a/doc/basics.md +++ b/doc/basics.md @@ -1,6 +1,6 @@ # Basics -CppWG is driven by a YAML configuration file that describes a **package** made +cppwg is driven by a YAML configuration file that describes a **package** made of one or more **modules**, and the **classes**, **free functions** and **enums** each module wraps. diff --git a/doc/configuration.md b/doc/configuration.md index 8b7e1720..3333dbf1 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -1,6 +1,6 @@ # Configuration -CppWG is driven by a YAML configuration file that describes what to wrap. +cppwg is driven by a YAML configuration file that describes what to wrap. This section describes how to create a configuration for your project. The [Basics](basics.md) page gets you started, the [Reference](reference.md) page diff --git a/doc/index.md b/doc/index.md index 52db5537..9cf04a47 100644 --- a/doc/index.md +++ b/doc/index.md @@ -1,6 +1,6 @@ # CppWG -CppWG is a C++ wrapper generator for Python. It reads your C++ source code, +cppwg is a C++ wrapper generator for Python. It reads your C++ source code, together with a YAML config file, and emits [pybind11](https://pybind11.readthedocs.io/) wrapper code. Using the YAML config file, you describe what parts of your C++ project to expose to Python rather than writing pybind11 wrappers directly. @@ -13,4 +13,5 @@ installation first-steps configuration custom-generators +python-packages ``` diff --git a/doc/python-packages.md b/doc/python-packages.md new file mode 100644 index 00000000..16b6f51e --- /dev/null +++ b/doc/python-packages.md @@ -0,0 +1,295 @@ +# Python packages + +A full Python package may need more than the compiled extension generated by cppwg. +For example, a package may want to add syntactic sugar for wrapped templated +classes to enable using the `Foo[Bar]` subscript syntax in Python instead of the +actually wrapped `Foo_Bar` and more closely mirror the `Foo` usage in C++. + +This can be added by hand in the Python package, but can drift from the config +whenever the wrapped instantiations change. To help address this, the +`cppwg genpackage` tool can be used to generate this **Python package layer** +from the same config that produced the wrappers, so they stay in step. + +## How it works + +Generation is a two-step flow: + +1. When cppwg generates the wrappers it also writes a **package model**, + `cppwg_package_model.json`. This is a small JSON description of what was + wrapped — each module, its compiled extension, and the wrapped classes (with + their template instantiations), enums and free functions. + +2. `cppwg genpackage` reads that model, together with a **layout file** you + provide describing the layout of the Python package, and emits a + `_generated.py` for each Python subpackage. In `_generated.py`, all the + wrapped elements are imported from the compiled-extension, and one + [`TemplateClass`](#templateclass) subscript stub is added per templated + class to allow the `Foo[Bar]` syntax from Python. The `_generated.py` can + then be imported inside an `__init__.py`, where other things can be manually + added that cannot be derived from the config + (see [Hand-written code](#hand-written-code)). + +(templateclass)= +### `TemplateClass` + +A templated class is exposed as a `TemplateClass` subclass whose `_instantiations` +maps template-argument tuples to the concrete wrapped classes. Subscripting the +class (`Point[2]`) looks up the concrete class (`Point_2`). These can be nested +so e.g. `MeshFactory>` is subscripted as `MeshFactory[PottsMesh[2]]` +(see `examples/cells`). + +:::{note} +The generated stubs import `TemplateClass` (and, where you use it, +`TemplateMethod`) from a `_syntax.py` helper in your package. This helper is not +generated — copy it from `examples/shapes/src/py/pyshapes/_syntax.py`. +::: + +## Layouts + +`cppwg genpackage` supports two layouts, and your supplied layout file chooses +between them: + +- **One subpackage per module** (default): each cppwg module has its **own** + compiled extension and becomes its own subpackage. The optional `module_dirs` + option controls *where* each subpackage is written. +- **One shared extension split into subpackages**: everything is compiled into a + **single** extension, which the `subpackages` option then divides across + several subpackages by name. + +## One subpackage per module + +Each cppwg [module](reference.md#module-options) has its own subpackage, which +star-imports the relevant compiled extension with `from . import *`. +This is the default, and what `examples/shapes` uses. + +**package_layout.yaml** + +`examples/shapes/wrapper/package_layout.yaml` + +```yaml +package: pyshapes +package_root: ../src/py/pyshapes +``` + +:::{note} +`package_root` is the directory holding the Python subpackage; a relative path +is resolved against the layout file's own directory. +::: + +Run: + +```bash +cppwg genpackage \ + --model wrapper/cppwg_package_model.json \ + --layout wrapper/package_layout.yaml +``` + +For the `geometry` module, this writes: + +**geometry/_generated.py** + +```python +"""Generated by cppwg genpackage - do not edit. +... +""" + +from ._pyshapes_geometry import * +from pyshapes._syntax import TemplateClass + + +class Point(TemplateClass): + _instantiations = { + ("2",): Point_2, + ("3",): Point_3, + } +``` + +This is imported into the hand-written `__init__.py` beside it: + +**geometry/__init__.py** + +```python +from ._generated import * # noqa: F401,F403 +``` + +The package can then be imported with both the concrete names and the subscript form: + +```python +from pyshapes.geometry import Point, Point_2 + +Point[2] # -> Point_2 +Point[3] # -> Point_3 +``` + +### Mapping modules to subpackage directories + +By default each module's subpackage directory is named after the module, under +`package_root` — so the `geometry` module above becomes `pyshapes/geometry/`. The +`module_dirs` option overrides that directory per module: the key is the module +name, the value is a directory relative to `package_root`. Use it to rename a +subpackage, nest it, or map a module to `"."` to place it at the package root. A +module not listed keeps the default. + +`examples/cells` has a single module, `all`, and maps it to `"."` so its extension +sits at the package root rather than in an `all/` subfolder: + +`examples/cells/dynamic/package_layout.yaml` + +```yaml +package: pycells +package_root: ../src/py/pycells +module_dirs: + all: "." +``` + +The `_generated.py` is then written to `pycells/_generated.py`, and +`from ._generated import *` in `pycells/__init__.py` surfaces everything at the top +level (`from pycells import Node`). + +`module_dirs` only relocates a module's files; every module still keeps its own +compiled extension. To split a *single* extension instead, use `subpackages` +([below](#splitting-one-shared-extension)). + +(splitting-one-shared-extension)= +## Splitting one shared extension into subpackages + +In the second layout all wrappers are compiled into a **single** extension, named +by `compiled_module`, but you still want several Python subpackages. The +`subpackages` option divides that one extension's contents manually by **name**. +For each subpackage, list the (base) class, enum and free-function names it should own. + +```yaml +package: chaste +package_root: src/py/chaste +compiled_module: _pychaste_all +subpackages: + core: [FileFinder, OutputFileHandler, RandomNumberGenerator, Timer, ...] + mesh: [ChastePoint, Element, MutableMesh, Node, ...] + # ... +``` + +Because they all depend on a single compiled extension, each subpackage's +`_generated.py` **explicitly** imports its own names from that extension rather +than star-importing it. A name listed under no subpackage, or listed but absent +from the model, is reported as a warning. + +For the `mesh` subpackage this writes the explicit imports plus a `TemplateClass` +stub per templated class: + +**mesh/_generated.py** + +```python +"""Generated by cppwg genpackage - do not edit. +... +""" + +from chaste._pychaste_all import ( + Element_1_1, + Element_1_2, + Element_1_3, + Element_2_2, + Element_2_3, + Element_3_3, + # ... +) +from chaste._syntax import TemplateClass + + +class Element(TemplateClass): + _instantiations = { + ("1", "1"): Element_1_1, + ("1", "2"): Element_1_2, + ("1", "3"): Element_1_3, + ("2", "2"): Element_2_2, + ("2", "3"): Element_2_3, + ("3", "3"): Element_3_3, + } + + +# ... one stub per templated class +``` + +As in the per-module layout, a hand-written `__init__.py` in each subpackage +star-imports this via `from ._generated import *`. + +(hand-written-code)= +## Hand-written code + +Anything that cannot be derived from the config stays in the hand-written +`__init__.py`, after the `from ._generated import *` line. A common case is a +templated *method* (`TemplateMethod`), which the model does not describe: + +**primitives/__init__.py** + +```python +from ._generated import * # noqa: F401,F403 +from pyshapes._syntax import TemplateMethod + +# UnitSquare.GetAreaIn[Unit]() — a templated method, so it cannot be +# auto-generated and is attached here. +UnitSquare.GetAreaIn = TemplateMethod("GetAreaIn", UnitSquare.GetAreaIn) +``` + +## Options + +### `diagonal_shorthand` + +For a multi-argument instantiation whose arguments are all equal (a "diagonal", +e.g. `Element<2, 2>`), also emit a single-argument alias so `Element[2]` resolves +to the same class as `Element[2, 2]`. Off by default; enable it in the layout file: + +```yaml +diagonal_shorthand: true +``` + +The `Element` stub from the split example above then gains a single-argument alias +key for each diagonal instantiation (the `("1",)`, `("2",)` and `("3",)` entries): + +```python +class Element(TemplateClass): + _instantiations = { + ("1", "1"): Element_1_1, + ("1",): Element_1_1, + ("1", "2"): Element_1_2, + ("1", "3"): Element_1_3, + ("2", "2"): Element_2_2, + ("2",): Element_2_2, + ("2", "3"): Element_2_3, + ("3", "3"): Element_3_3, + ("3",): Element_3_3, + } +``` + +so `Element[2]` now resolves to `Element_2_2` alongside `Element[2, 2]`. + +### `flatten_to_root` + +Also write a top-level `_generated.py` that re-exports every subpackage's class, +enum and free-function names into the package root, so `chaste.Node` works +alongside `chaste.mesh.Node`. A name exported by more than one subpackage is +reported as an ambiguity warning. Off by default: + +```yaml +flatten_to_root: true +``` + +## Command-line usage + +```text +usage: cppwg genpackage [-h] --model MODEL --layout LAYOUT [--overwrite] + +options: + -h, --help show this help message and exit + --model MODEL Path to cppwg_package_model.json (from cppwg). + --layout LAYOUT Path to the Python package-layout file (YAML). + --overwrite Rewrite files even if unchanged. +``` + +The generated `_generated.py` files are reproducible: re-running `cppwg genpackage` +after a config change updates only what changed, so the layer can be checked in and +verified with `git diff`. + +:::{seealso} +- See [First steps](first-steps.md) for generating the wrappers themselves. +- See [Templates](templates.md) for choosing which instantiations are wrapped. +::: diff --git a/examples/cells/dynamic/package_layout.yaml b/examples/cells/dynamic/package_layout.yaml new file mode 100644 index 00000000..da6d3325 --- /dev/null +++ b/examples/cells/dynamic/package_layout.yaml @@ -0,0 +1,11 @@ +# Python-package layout for cppwg genpackage (cells example). +# +# A single cppwg module `all` -> the flat pycells package (its compiled +# extension _pycells_all sits at the package root, not in an `all/` subdir), so +# module_dirs maps `all` to ".". +# +# package_root is resolved relative to this layout file's directory. +package: pycells +package_root: ../src/py/pycells +module_dirs: + all: "." diff --git a/examples/cells/dynamic/wrappers/cppwg_package_model.json b/examples/cells/dynamic/wrappers/cppwg_package_model.json new file mode 100644 index 00000000..d9d53644 --- /dev/null +++ b/examples/cells/dynamic/wrappers/cppwg_package_model.json @@ -0,0 +1,254 @@ +{ + "_comment": "This file is automatically generated by cppwg. Do not modify this file directly.", + "modules": [ + { + "classes": [ + { + "base": "Cell", + "instantiations": [ + { + "args": [], + "cpp_name": "Cell", + "py_name": "Cell" + } + ], + "templated": false + }, + { + "base": "CellFactory", + "instantiations": [ + { + "args": [ + "Cell", + "2" + ], + "cpp_name": "CellFactory", + "py_name": "CellFactory_Cell_2" + }, + { + "args": [ + "Cell", + "3" + ], + "cpp_name": "CellFactory", + "py_name": "CellFactory_Cell_3" + } + ], + "templated": true + }, + { + "base": "Corner", + "instantiations": [ + { + "args": [ + "2" + ], + "cpp_name": "Corner<2>", + "py_name": "Corner_2" + } + ], + "templated": true + }, + { + "base": "Facet", + "instantiations": [ + { + "args": [ + "2" + ], + "cpp_name": "Facet<2>", + "py_name": "Facet_2" + } + ], + "templated": true + }, + { + "base": "MacroMesh", + "instantiations": [ + { + "args": [ + "2", + "2" + ], + "cpp_name": "MacroMesh<2, 2>", + "py_name": "MacroMesh_2_2" + }, + { + "args": [ + "3", + "3" + ], + "cpp_name": "MacroMesh<3, 3>", + "py_name": "MacroMesh_3_3" + } + ], + "templated": true + }, + { + "base": "Node", + "instantiations": [ + { + "args": [ + "2" + ], + "cpp_name": "Node<2>", + "py_name": "Node_2" + }, + { + "args": [ + "3" + ], + "cpp_name": "Node<3>", + "py_name": "Node_3" + } + ], + "templated": true + }, + { + "base": "AbstractMesh", + "instantiations": [ + { + "args": [ + "2", + "2" + ], + "cpp_name": "AbstractMesh<2, 2>", + "py_name": "AbstractMesh_2_2" + }, + { + "args": [ + "3", + "3" + ], + "cpp_name": "AbstractMesh<3, 3>", + "py_name": "AbstractMesh_3_3" + } + ], + "templated": true + }, + { + "base": "AbstractSphericalMesh", + "instantiations": [ + { + "args": [ + "2", + "2" + ], + "cpp_name": "AbstractSphericalMesh<2, 2>", + "py_name": "AbstractSphericalMesh_2_2" + }, + { + "args": [ + "3", + "3" + ], + "cpp_name": "AbstractSphericalMesh<3, 3>", + "py_name": "AbstractSphericalMesh_3_3" + } + ], + "templated": true + }, + { + "base": "PetscUtils", + "instantiations": [ + { + "args": [], + "cpp_name": "PetscUtils", + "py_name": "PetscUtils" + } + ], + "templated": false + }, + { + "base": "PottsMesh", + "instantiations": [ + { + "args": [ + "2" + ], + "cpp_name": "PottsMesh<2>", + "py_name": "PottsMesh_2" + }, + { + "args": [ + "3" + ], + "cpp_name": "PottsMesh<3>", + "py_name": "PottsMesh_3" + } + ], + "templated": true + }, + { + "base": "MeshFactory", + "instantiations": [ + { + "args": [ + "PottsMesh<2>" + ], + "cpp_name": "MeshFactory>", + "py_name": "MeshFactory_PottsMesh_2" + }, + { + "args": [ + "PottsMesh<3>" + ], + "cpp_name": "MeshFactory>", + "py_name": "MeshFactory_PottsMesh_3" + } + ], + "templated": true + }, + { + "base": "Scene", + "instantiations": [ + { + "args": [ + "2" + ], + "cpp_name": "Scene<2>", + "py_name": "Scene_2" + }, + { + "args": [ + "3" + ], + "cpp_name": "Scene<3>", + "py_name": "Scene_3" + } + ], + "templated": true + }, + { + "base": "SphericalMesh", + "instantiations": [ + { + "args": [ + "2", + "2" + ], + "cpp_name": "SphericalMesh<2, 2>", + "py_name": "SphericalMesh_2_2" + }, + { + "args": [ + "3", + "3" + ], + "cpp_name": "SphericalMesh<3, 3>", + "py_name": "SphericalMesh_3_3" + } + ], + "templated": true + } + ], + "compiled_module": "_pycells_all", + "enum_exports": {}, + "enums": [], + "free_functions": [], + "imports": [], + "name": "all" + } + ], + "package": "pycells" +} diff --git a/examples/cells/src/py/pycells/__init__.py b/examples/cells/src/py/pycells/__init__.py index 17464d7d..0c513273 100644 --- a/examples/cells/src/py/pycells/__init__.py +++ b/examples/cells/src/py/pycells/__init__.py @@ -1,113 +1,9 @@ -"""Main pycells module.""" +"""Main pycells module. -from ._pycells_all import ( - Cell, - CellFactory_Cell_2, - CellFactory_Cell_3, - Corner_2, - Facet_2, - MacroMesh_2_2, - MacroMesh_3_3, - MeshFactory_PottsMesh_2, - MeshFactory_PottsMesh_3, - Node_2, - Node_3, - PetscUtils, - PottsMesh_2, - PottsMesh_3, - Scene_2, - Scene_3, - SphericalMesh_2_2, - SphericalMesh_3_3, -) -from ._syntax import TemplateClass +The compiled-extension imports and TemplateClass subscript stubs are generated +by cppwg genpackage into _generated.py; add any hand-written code here. +Curation rationale (why Facet<1>/Corner<1>/AbstractMesh are or aren't wrapped) +lives with the C++ sources and examples/cells/dynamic/config.yaml. +""" - -# CellFactory names CELL_TYPE only as a template argument, so -# cppwg's auto_includes resolved Cell.hpp from the instantiation arguments (not -# a signature) - see CellFactory.hpp and examples/cells/dynamic/config.yaml. -class CellFactory(TemplateClass): - _instantiations = { - ("Cell", "2"): CellFactory_Cell_2, - ("Cell", "3"): CellFactory_Cell_3, - } - - -# Only Facet<2> is wrapped (curated). Facet<1> is deliberately left unwrapped; -# wrapping it would reference the never-instantiated Facet<0> and fail to import -# (see Facet.hpp and examples/cells/dynamic/config.yaml). -class Facet(TemplateClass): - _instantiations = { - ("2",): Facet_2, - } - - -# Non-curated counterpart to Facet: discovery finds Corner<1> and Corner<2>, and -# cppwg auto-drops Corner<1> (its GetSub returns the never-instantiated -# Corner<0>), so only Corner<2> is wrapped (see Corner.hpp). -class Corner(TemplateClass): - _instantiations = { - ("2",): Corner_2, - } - - -class MacroMesh(TemplateClass): - _instantiations = { - ("2", "2"): MacroMesh_2_2, - ("3", "3"): MacroMesh_3_3, - } - - -class MeshFactory(TemplateClass): - _instantiations = { - ("PottsMesh", "2"): MeshFactory_PottsMesh_2, - ("PottsMesh", "3"): MeshFactory_PottsMesh_3, - } - - -class Node(TemplateClass): - _instantiations = { - ("2",): Node_2, - ("3",): Node_3, - } - - -class PottsMesh(TemplateClass): - _instantiations = { - ("2",): PottsMesh_2, - ("3",): PottsMesh_3, - } - - -# SphericalMesh is the concrete leaf of AbstractMesh -> AbstractSphericalMesh -# -> SphericalMesh. Its inherited overrides (Scale, GetNumElements) are bound -# on the abstract bases and remain callable through inheritance despite being -# skipped on the leaf by exclude_inherited_overrides. The abstract bases are not -# constructible, so - like AbstractMesh - they are not surfaced here. -class SphericalMesh(TemplateClass): - _instantiations = { - ("2", "2"): SphericalMesh_2_2, - ("3", "3"): SphericalMesh_3_3, - } - - -class Scene(TemplateClass): - _instantiations = { - ("2",): Scene_2, - ("3",): Scene_3, - } - - -__all__ = [ - "Cell", - "CellFactory", - "Corner", - "Facet", - "MacroMesh", - "MeshFactory", - "Node", - "PetscUtils", - "PottsMesh", - "Scene", - "SphericalMesh", -] +from ._generated import * # noqa: F401,F403 diff --git a/examples/cells/src/py/pycells/_generated.py b/examples/cells/src/py/pycells/_generated.py new file mode 100644 index 00000000..0017a1cb --- /dev/null +++ b/examples/cells/src/py/pycells/_generated.py @@ -0,0 +1,85 @@ +"""Generated by cppwg genpackage - do not edit. + +Compiled-extension imports and TemplateClass subscript stubs for this +subpackage. Hand-written code (TemplateMethod attachments, package init, +curation) belongs in the sibling __init__.py, which does `from ._generated +import *`. +""" + +from ._pycells_all import * +from pycells._syntax import TemplateClass + + +class CellFactory(TemplateClass): + _instantiations = { + ("Cell", "2"): CellFactory_Cell_2, + ("Cell", "3"): CellFactory_Cell_3, + } + + +class Corner(TemplateClass): + _instantiations = { + ("2",): Corner_2, + } + + +class Facet(TemplateClass): + _instantiations = { + ("2",): Facet_2, + } + + +class MacroMesh(TemplateClass): + _instantiations = { + ("2", "2"): MacroMesh_2_2, + ("3", "3"): MacroMesh_3_3, + } + + +class Node(TemplateClass): + _instantiations = { + ("2",): Node_2, + ("3",): Node_3, + } + + +class AbstractMesh(TemplateClass): + _instantiations = { + ("2", "2"): AbstractMesh_2_2, + ("3", "3"): AbstractMesh_3_3, + } + + +class AbstractSphericalMesh(TemplateClass): + _instantiations = { + ("2", "2"): AbstractSphericalMesh_2_2, + ("3", "3"): AbstractSphericalMesh_3_3, + } + + +class PottsMesh(TemplateClass): + _instantiations = { + ("2",): PottsMesh_2, + ("3",): PottsMesh_3, + } + + +class MeshFactory(TemplateClass): + _instantiations = { + ("PottsMesh_2",): MeshFactory_PottsMesh_2, + ("PottsMesh_3",): MeshFactory_PottsMesh_3, + } + + +class Scene(TemplateClass): + _instantiations = { + ("2",): Scene_2, + ("3",): Scene_3, + } + + +class SphericalMesh(TemplateClass): + _instantiations = { + ("2", "2"): SphericalMesh_2_2, + ("3", "3"): SphericalMesh_3_3, + } diff --git a/examples/cells/tests/test_cells.py b/examples/cells/tests/test_cells.py index 5fe46c0c..b44e70b8 100644 --- a/examples/cells/tests/test_cells.py +++ b/examples/cells/tests/test_cells.py @@ -9,8 +9,10 @@ Corner, Facet, MacroMesh, + MeshFactory, Node, PetscUtils, + PottsMesh, Scene, SphericalMesh, ) @@ -55,6 +57,19 @@ def testMacroInstantiationFallback(self): self.assertEqual(MacroMesh[2, 2]().GetDimension(), 2) self.assertEqual(MacroMesh[3, 3]().GetDimension(), 3) + def testNestedTemplateArgSubscript(self): + # MeshFactory has a single template argument that is itself a + # templated type - MeshFactory>. It is subscripted with the + # Python mesh class, MeshFactory[PottsMesh[2]] (PottsMesh[2] is the + # PottsMesh_2 class), rather than splitting the argument as + # MeshFactory[PottsMesh, 2]. + factory = MeshFactory[PottsMesh[2]]() + self.assertIsInstance(factory.generateMesh(), PottsMesh[2]) + self.assertIsInstance(MeshFactory[PottsMesh[3]]().generateMesh(), PottsMesh[3]) + # The split-argument form is not a valid key. + with self.assertRaises(KeyError): + _ = MeshFactory[PottsMesh, 2] + def testInheritedOverrideStillCallable(self): # SphericalMesh is the concrete leaf of an abstract chain # (AbstractMesh -> AbstractSphericalMesh -> SphericalMesh). Its diff --git a/examples/shapes/src/py/pyshapes/composites/__init__.py b/examples/shapes/src/py/pyshapes/composites/__init__.py index 35aa80f7..19b28174 100644 --- a/examples/shapes/src/py/pyshapes/composites/__init__.py +++ b/examples/shapes/src/py/pyshapes/composites/__init__.py @@ -1,2 +1,3 @@ -# Bring in everything from the shared module -from pyshapes.composites._pyshapes_composites import * +# The compiled-extension imports and TemplateClass stubs are generated by +# cppwg genpackage into _generated.py; add any hand-written code here. +from ._generated import * # noqa: F401,F403 diff --git a/examples/shapes/src/py/pyshapes/composites/_generated.py b/examples/shapes/src/py/pyshapes/composites/_generated.py new file mode 100644 index 00000000..72ebe804 --- /dev/null +++ b/examples/shapes/src/py/pyshapes/composites/_generated.py @@ -0,0 +1,9 @@ +"""Generated by cppwg genpackage - do not edit. + +Compiled-extension imports and TemplateClass subscript stubs for this +subpackage. Hand-written code (TemplateMethod attachments, package init, +curation) belongs in the sibling __init__.py, which does `from ._generated +import *`. +""" + +from ._pyshapes_composites import * diff --git a/examples/shapes/src/py/pyshapes/geometry/__init__.py b/examples/shapes/src/py/pyshapes/geometry/__init__.py index 9cd02bb1..19b28174 100644 --- a/examples/shapes/src/py/pyshapes/geometry/__init__.py +++ b/examples/shapes/src/py/pyshapes/geometry/__init__.py @@ -1,10 +1,3 @@ -# Bring in everything from the shared module -from pyshapes._syntax import TemplateClass -from pyshapes.geometry._pyshapes_geometry import * - - -class Point(TemplateClass): - _instantiations = { - 2: Point_2, - 3: Point_3, - } +# The compiled-extension imports and TemplateClass stubs are generated by +# cppwg genpackage into _generated.py; add any hand-written code here. +from ._generated import * # noqa: F401,F403 diff --git a/examples/shapes/src/py/pyshapes/geometry/_generated.py b/examples/shapes/src/py/pyshapes/geometry/_generated.py new file mode 100644 index 00000000..0d855af2 --- /dev/null +++ b/examples/shapes/src/py/pyshapes/geometry/_generated.py @@ -0,0 +1,17 @@ +"""Generated by cppwg genpackage - do not edit. + +Compiled-extension imports and TemplateClass subscript stubs for this +subpackage. Hand-written code (TemplateMethod attachments, package init, +curation) belongs in the sibling __init__.py, which does `from ._generated +import *`. +""" + +from ._pyshapes_geometry import * +from pyshapes._syntax import TemplateClass + + +class Point(TemplateClass): + _instantiations = { + ("2",): Point_2, + ("3",): Point_3, + } diff --git a/examples/shapes/src/py/pyshapes/math_funcs/__init__.py b/examples/shapes/src/py/pyshapes/math_funcs/__init__.py index 53f3fe06..19b28174 100644 --- a/examples/shapes/src/py/pyshapes/math_funcs/__init__.py +++ b/examples/shapes/src/py/pyshapes/math_funcs/__init__.py @@ -1,2 +1,3 @@ -# Bring in everything from the shared module -from pyshapes.math_funcs._pyshapes_math_funcs import * +# The compiled-extension imports and TemplateClass stubs are generated by +# cppwg genpackage into _generated.py; add any hand-written code here. +from ._generated import * # noqa: F401,F403 diff --git a/examples/shapes/src/py/pyshapes/math_funcs/_generated.py b/examples/shapes/src/py/pyshapes/math_funcs/_generated.py new file mode 100644 index 00000000..3e7ad835 --- /dev/null +++ b/examples/shapes/src/py/pyshapes/math_funcs/_generated.py @@ -0,0 +1,9 @@ +"""Generated by cppwg genpackage - do not edit. + +Compiled-extension imports and TemplateClass subscript stubs for this +subpackage. Hand-written code (TemplateMethod attachments, package init, +curation) belongs in the sibling __init__.py, which does `from ._generated +import *`. +""" + +from ._pyshapes_math_funcs import * diff --git a/examples/shapes/src/py/pyshapes/primitives/__init__.py b/examples/shapes/src/py/pyshapes/primitives/__init__.py index 872ae915..b7e26a00 100644 --- a/examples/shapes/src/py/pyshapes/primitives/__init__.py +++ b/examples/shapes/src/py/pyshapes/primitives/__init__.py @@ -1,17 +1,11 @@ -# Bring in everything from the shared module -from pyshapes._syntax import TemplateClass, TemplateMethod -from pyshapes.primitives._pyshapes_primitives import * - - -class Shape(TemplateClass): - _instantiations = { - 2: Shape_2, - 3: Shape_3, - } - +# The compiled-extension imports and TemplateClass stubs are generated by +# cppwg genpackage into _generated.py; add any hand-written code here. +from ._generated import * # noqa: F401,F403 +from pyshapes._syntax import TemplateMethod # UnitSquare::GetAreaIn() is a templated method (see GetAreaInCustomTemplate.py); # expose its per-unit bindings as GetAreaIn[Unit](). GetAreaIn is also a plain # overload (GetAreaIn(perSquareMetre)); pass it as the fallback so the descriptor # does not shadow it and UnitSquare.GetAreaIn(factor) keeps working. +# (TemplateMethod attachments cannot be auto-generated, so they live here.) UnitSquare.GetAreaIn = TemplateMethod("GetAreaIn", UnitSquare.GetAreaIn) diff --git a/examples/shapes/src/py/pyshapes/primitives/_generated.py b/examples/shapes/src/py/pyshapes/primitives/_generated.py new file mode 100644 index 00000000..9058a64d --- /dev/null +++ b/examples/shapes/src/py/pyshapes/primitives/_generated.py @@ -0,0 +1,38 @@ +"""Generated by cppwg genpackage - do not edit. + +Compiled-extension imports and TemplateClass subscript stubs for this +subpackage. Hand-written code (TemplateMethod attachments, package init, +curation) belongs in the sibling __init__.py, which does `from ._generated +import *`. +""" + +from ._pyshapes_primitives import * +from pyshapes._syntax import TemplateClass + + +class AbstractShape(TemplateClass): + _instantiations = { + ("2",): AbstractShape_2, + ("3",): AbstractShape_3, + } + + +class AbstractPolygon(TemplateClass): + _instantiations = { + ("2",): AbstractPolygon_2, + ("3",): AbstractPolygon_3, + } + + +class RegularPolygon(TemplateClass): + _instantiations = { + ("2",): RegularPolygon_2, + ("3",): RegularPolygon_3, + } + + +class Shape(TemplateClass): + _instantiations = { + ("2",): Shape_2, + ("3",): Shape_3, + } diff --git a/examples/shapes/wrapper/cppwg_package_model.json b/examples/shapes/wrapper/cppwg_package_model.json new file mode 100644 index 00000000..93d36d99 --- /dev/null +++ b/examples/shapes/wrapper/cppwg_package_model.json @@ -0,0 +1,250 @@ +{ + "_comment": "This file is automatically generated by cppwg. Do not modify this file directly.", + "modules": [ + { + "classes": [], + "compiled_module": "_pyshapes_math_funcs", + "enum_exports": {}, + "enums": [], + "free_functions": [ + "add", + "throw_exception", + "throw_unwrapped_exception" + ], + "imports": [], + "name": "math_funcs" + }, + { + "classes": [ + { + "base": "Point", + "instantiations": [ + { + "args": [ + "2" + ], + "cpp_name": "Point<2>", + "py_name": "Point_2" + }, + { + "args": [ + "3" + ], + "cpp_name": "Point<3>", + "py_name": "Point_3" + } + ], + "templated": true + } + ], + "compiled_module": "_pyshapes_geometry", + "enum_exports": {}, + "enums": [], + "free_functions": [], + "imports": [], + "name": "geometry" + }, + { + "classes": [ + { + "base": "AbstractShape", + "instantiations": [ + { + "args": [ + "2" + ], + "cpp_name": "AbstractShape<2>", + "py_name": "AbstractShape_2" + }, + { + "args": [ + "3" + ], + "cpp_name": "AbstractShape<3>", + "py_name": "AbstractShape_3" + } + ], + "templated": true + }, + { + "base": "AbstractPolygon", + "instantiations": [ + { + "args": [ + "2" + ], + "cpp_name": "AbstractPolygon<2>", + "py_name": "AbstractPolygon_2" + }, + { + "args": [ + "3" + ], + "cpp_name": "AbstractPolygon<3>", + "py_name": "AbstractPolygon_3" + } + ], + "templated": true + }, + { + "base": "RegularPolygon", + "instantiations": [ + { + "args": [ + "2" + ], + "cpp_name": "RegularPolygon<2>", + "py_name": "RegularPolygon_2" + }, + { + "args": [ + "3" + ], + "cpp_name": "RegularPolygon<3>", + "py_name": "RegularPolygon_3" + } + ], + "templated": true + }, + { + "base": "Shape", + "instantiations": [ + { + "args": [ + "2" + ], + "cpp_name": "Shape<2>", + "py_name": "Shape_2" + }, + { + "args": [ + "3" + ], + "cpp_name": "Shape<3>", + "py_name": "Shape_3" + } + ], + "templated": true + }, + { + "base": "Cuboid", + "instantiations": [ + { + "args": [], + "cpp_name": "Cuboid", + "py_name": "Cuboid" + } + ], + "templated": false + }, + { + "base": "Rectangle", + "instantiations": [ + { + "args": [], + "cpp_name": "Rectangle", + "py_name": "Rectangle" + } + ], + "templated": false + }, + { + "base": "ShapeClassifier", + "instantiations": [ + { + "args": [], + "cpp_name": "ShapeClassifier", + "py_name": "ShapeClassifier" + } + ], + "templated": false + }, + { + "base": "ShapeMetrics", + "instantiations": [ + { + "args": [], + "cpp_name": "ShapeMetrics", + "py_name": "ShapeMetrics" + } + ], + "templated": false + }, + { + "base": "SquareFeet", + "instantiations": [ + { + "args": [], + "cpp_name": "SquareFeet", + "py_name": "SquareFeet" + } + ], + "templated": false + }, + { + "base": "SquareMetres", + "instantiations": [ + { + "args": [], + "cpp_name": "SquareMetres", + "py_name": "SquareMetres" + } + ], + "templated": false + }, + { + "base": "UnitSquare", + "instantiations": [ + { + "args": [], + "cpp_name": "UnitSquare", + "py_name": "UnitSquare" + } + ], + "templated": false + } + ], + "compiled_module": "_pyshapes_primitives", + "enum_exports": { + "ShapeKind": [ + "CIRCLE", + "SQUARE", + "TRIANGLE" + ] + }, + "enums": [ + "Handedness", + "ShapeKind" + ], + "free_functions": [], + "imports": [ + "pyshapes.geometry._pyshapes_geometry" + ], + "name": "primitives" + }, + { + "classes": [ + { + "base": "Square", + "instantiations": [ + { + "args": [], + "cpp_name": "Square", + "py_name": "Square" + } + ], + "templated": false + } + ], + "compiled_module": "_pyshapes_composites", + "enum_exports": {}, + "enums": [], + "free_functions": [], + "imports": [ + "pyshapes.primitives._pyshapes_primitives" + ], + "name": "composites" + } + ], + "package": "pyshapes" +} diff --git a/examples/shapes/wrapper/package_layout.yaml b/examples/shapes/wrapper/package_layout.yaml new file mode 100644 index 00000000..375443ba --- /dev/null +++ b/examples/shapes/wrapper/package_layout.yaml @@ -0,0 +1,9 @@ +# Python-package layout for cppwg genpackage (shapes example). +# +# Module-per-subpackage: each cppwg module (geometry/primitives/composites/ +# math_funcs) is a subpackage that owns its own compiled extension, so each +# _generated.py does `from .<_pyshapes_module> import *`. +# +# package_root is resolved relative to this layout file's directory. +package: pyshapes +package_root: ../src/py/pyshapes diff --git a/tests/test_free_function_writer.py b/tests/test_free_function_writer.py index de547aae..a6419b1e 100644 --- a/tests/test_free_function_writer.py +++ b/tests/test_free_function_writer.py @@ -21,16 +21,17 @@ def __init__(self, return_type, arg_types): class _FreeFunctionInfo: """Minimal CppFreeFunctionInfo stand-in for exclusion lookups.""" - def __init__(self, return_type, arg_types, excludes=None): + def __init__(self, return_type, arg_types, excludes=None, excluded=False): self.decls = [_Decl(return_type, arg_types)] + self.excluded = excluded self._excludes = excludes or {} def hierarchy_attribute_gather_flat(self, name): return list(self._excludes.get(name, [])) -def _writer(return_type="void", arg_types=(), excludes=None): - info = _FreeFunctionInfo(return_type, list(arg_types), excludes) +def _writer(return_type="void", arg_types=(), excludes=None, excluded=False): + info = _FreeFunctionInfo(return_type, list(arg_types), excludes, excluded) writer = object.__new__(CppFreeFunctionWrapperWriter) writer.free_function_info = info return writer @@ -71,6 +72,12 @@ def test_free_function_not_excluded_without_options(): assert _writer(return_type="int", arg_types=["double"]).exclude() is False +def test_free_function_config_excluded(): + """A config-excluded function (YAML `excluded: true`) is excluded regardless + of its types, so its binding is never emitted.""" + assert _writer(return_type="int", arg_types=["double"], excluded=True).exclude() + + from string import Template # noqa: E402 diff --git a/tests/test_genpackage.py b/tests/test_genpackage.py new file mode 100644 index 00000000..8843cc04 --- /dev/null +++ b/tests/test_genpackage.py @@ -0,0 +1,560 @@ +"""Unit tests for cppwg.genpackage (the `cppwg genpackage` subcommand).""" + +import ast +import json +import sys + +import pytest + +import cppwg.__main__ as cppwg_main +from cppwg import genpackage + + +def _class(base, instantiations, templated=True): + return {"base": base, "templated": templated, "instantiations": instantiations} + + +def _inst(args, py_name, cpp_name=None): + inst = {"args": list(args), "py_name": py_name} + if cpp_name is not None: + inst["cpp_name"] = cpp_name + return inst + + +def test_render_key_singleton_and_multi(): + assert genpackage._render_key(["2"]) == '("2",)' + assert genpackage._render_key(["2", "2"]) == '("2", "2")' + assert genpackage._render_key(["Cell", "2"]) == '("Cell", "2")' + + +def test_render_template_stub(): + stub = genpackage._render_template_stub( + "Point", [_inst(["2"], "Point_2"), _inst(["3"], "Point_3")] + ) + assert stub == ( + "class Point(TemplateClass):\n" + " _instantiations = {\n" + ' ("2",): Point_2,\n' + ' ("3",): Point_3,\n' + " }" + ) + + +def test_render_template_stub_diagonal_shorthand(): + """A multi-arg diagonal instantiation gains a single-arg alias when opted in.""" + stub = genpackage._render_template_stub( + "Element", + [_inst(["2", "2"], "Element_2_2"), _inst(["1", "2"], "Element_1_2")], + diagonal_shorthand=True, + ) + # <2, 2> is diagonal -> add the Element[2] alias; <1, 2> is not -> no alias. + assert '("2", "2"): Element_2_2,' in stub + assert '("2",): Element_2_2,' in stub + assert '("1", "2"): Element_1_2,' in stub + assert '("1",):' not in stub + # Off by default: no aliases. + plain = genpackage._render_template_stub("Element", [_inst(["2", "2"], "Element_2_2")]) + assert '("2",):' not in plain + + +def test_render_template_stub_nested_templated_arg(): + """A templated-type argument is keyed by its Python concrete class name.""" + # MeshFactory> -> key ("PottsMesh_2",), not ("PottsMesh<2>",), + # so MeshFactory[PottsMesh[2]] resolves (PottsMesh[2] is the PottsMesh_2 class). + cpp_to_pyname = {"PottsMesh<2>": "PottsMesh_2", "PottsMesh<3>": "PottsMesh_3"} + stub = genpackage._render_template_stub( + "MeshFactory", + [ + _inst(["PottsMesh<2>"], "MeshFactory_PottsMesh_2"), + _inst(["PottsMesh<3>"], "MeshFactory_PottsMesh_3"), + ], + cpp_to_pyname=cpp_to_pyname, + ) + assert '("PottsMesh_2",): MeshFactory_PottsMesh_2,' in stub + assert '("PottsMesh_3",): MeshFactory_PottsMesh_3,' in stub + assert "<" not in stub # the raw C++ type string is gone + + +def test_build_cpp_to_pyname(): + model = { + "modules": [ + { + "classes": [ + _class("PottsMesh", [_inst(["2"], "PottsMesh_2")]), + _class("Cell", [_inst([], "Cell")], templated=False), + ] + } + ] + } + index = genpackage._build_cpp_to_pyname(model) + assert index == {"PottsMesh<2>": "PottsMesh_2"} # untemplated Cell has no <...> + + +def test_build_cpp_to_pyname_uses_cpp_name_for_name_override(): + """A class whose C++ name differs from its py name is keyed by the C++ name.""" + model = { + "modules": [ + { + "classes": [ + _class( + "NewName", + [_inst(["2"], "NewName_2", cpp_name="OldName<2>")], + ), + ] + } + ] + } + index = genpackage._build_cpp_to_pyname(model) + # Keyed by OldName<2> (the real C++ type), not NewName<2>, so a nested + # OldName<2> argument resolves to NewName_2. + assert index == {"OldName<2>": "NewName_2"} + + +def test_build_cpp_to_pyname_indexes_untemplated_name_override(): + """A renamed untemplated class is indexed by its C++ name, so it resolves as + a template argument (C++ OldName exposed as NewName).""" + model = { + "modules": [ + { + "classes": [ + _class( + "NewName", + [_inst([], "NewName", cpp_name="OldName")], + templated=False, + ), + ] + } + ] + } + index = genpackage._build_cpp_to_pyname(model) + assert index == {"OldName": "NewName"} # a nested OldName arg -> NewName + + +def test_render_generated_module_with_stub_imports_syntax(): + point = _class("Point", [_inst(["2"], "Point_2"), _inst(["3"], "Point_3")]) + content = genpackage.render_generated_module( + "pyshapes", "from ._pyshapes_geometry import *", [point], [point] + ) + assert content.startswith('"""Generated by cppwg genpackage') + assert "from ._pyshapes_geometry import *" in content + assert "from pyshapes._syntax import TemplateClass" in content + assert "class Point(TemplateClass):" in content + assert content.endswith("\n") + + +def test_render_generated_module_no_templates_omits_syntax(): + plain = _class("Square", [_inst([], "Square")], templated=False) + content = genpackage.render_generated_module( + "pyshapes", "from ._pyshapes_composites import *", [plain], [] + ) + assert "from ._pyshapes_composites import *" in content + assert "import TemplateClass" not in content # no templated -> no syntax import + + +def test_generate_module_per_subpackage(tmp_path): + model = { + "package": "pyshapes", + "modules": [ + { + "name": "geometry", + "compiled_module": "_pyshapes_geometry", + "imports": [], + "classes": [ + _class("Point", [_inst(["2"], "Point_2"), _inst(["3"], "Point_3")]) + ], + "enums": [], + "free_functions": [], + }, + { + "name": "math_funcs", + "compiled_module": "_pyshapes_math_funcs", + "imports": [], + "classes": [], + "enums": [], + "free_functions": ["add"], + }, + ], + } + layout = {"package": "pyshapes", "package_root": str(tmp_path)} + + genpackage.generate_module_per_subpackage(model, layout, overwrite=False) + + geometry = (tmp_path / "geometry" / "_generated.py").read_text() + assert "from ._pyshapes_geometry import *" in geometry + assert "class Point(TemplateClass):" in geometry + + math_funcs = (tmp_path / "math_funcs" / "_generated.py").read_text() + assert "from ._pyshapes_math_funcs import *" in math_funcs + assert "import TemplateClass" not in math_funcs # free functions only + + +def test_module_dirs_override_places_single_module_at_root(tmp_path): + """A `module_dirs` entry redirects a module's _generated.py (e.g. cells' root).""" + model = { + "package": "pycells", + "modules": [ + { + "name": "all", + "compiled_module": "_pycells_all", + "imports": [], + "classes": [_class("Node", [_inst(["2"], "Node_2")])], + "enums": [], + "free_functions": [], + } + ], + } + layout = {"package_root": str(tmp_path), "module_dirs": {"all": "."}} + + genpackage.generate_module_per_subpackage(model, layout, overwrite=False) + + generated = (tmp_path / "_generated.py").read_text() # at the root, not all/ + assert "from ._pycells_all import *" in generated + assert "class Node(TemplateClass):" in generated + + +def test_generate_shared_module_split(tmp_path, capsys): + """Split one shared extension into subpackages by an explicit membership list.""" + model = { + "package": "pychaste", + "modules": [ + { + "name": "all", + "compiled_module": "_pychaste_all", + "imports": [], + "classes": [ + _class("Node", [_inst(["2"], "Node_2"), _inst(["3"], "Node_3")]), + _class("FileFinder", [_inst([], "FileFinder")], templated=False), + _class("PottsMesh", [_inst(["2"], "PottsMesh_2")]), + ], + "enums": ["RelativeTo"], + "free_functions": [], + } + ], + } + layout = { + "package": "chaste", + "package_root": str(tmp_path), + "compiled_module": "_pychaste_all", + "subpackages": { + "core": ["FileFinder", "RelativeTo"], + "mesh": ["Node", "PottsMesh"], + }, + } + + genpackage.generate_shared_module_split(model, layout, overwrite=False) + + core = (tmp_path / "core" / "_generated.py").read_text() + assert "from chaste._pychaste_all import (" in core + assert "FileFinder," in core and "RelativeTo," in core + assert "import TemplateClass" not in core # core has no templated class + + mesh = (tmp_path / "mesh" / "_generated.py").read_text() + assert "from chaste._pychaste_all import (" in mesh + # concrete names imported, sorted + assert "Node_2," in mesh and "Node_3," in mesh and "PottsMesh_2," in mesh + assert "from chaste._syntax import TemplateClass" in mesh + assert "class Node(TemplateClass):" in mesh + assert "class PottsMesh(TemplateClass):" in mesh + + +def test_shared_split_imports_exported_enumerators(tmp_path): + """A value-exporting enum drags its enumerators into the owning subpackage.""" + model = { + "package": "pkg", + "modules": [ + { + "name": "all", + "compiled_module": "_pkg_all", + "imports": [], + "classes": [], + "enums": ["ShapeKind"], + "enum_exports": {"ShapeKind": ["CIRCLE", "SQUARE"]}, + "free_functions": [], + } + ], + } + layout = { + "package": "pkg", + "package_root": str(tmp_path), + "compiled_module": "_pkg_all", + "flatten_to_root": True, + "subpackages": {"geometry": ["ShapeKind"]}, + } + + genpackage.generate_shared_module_split(model, layout, overwrite=False) + + geometry = (tmp_path / "geometry" / "_generated.py").read_text() + # The enum and its exported enumerators are all imported into the subpackage. + assert "ShapeKind," in geometry + assert "CIRCLE," in geometry and "SQUARE," in geometry + # ...and re-exported at the top level so pkg.CIRCLE also works. + root = (tmp_path / "_generated.py").read_text() + assert '"CIRCLE",' in root and '"SQUARE",' in root and '"ShapeKind",' in root + + +def test_flatten_to_root(tmp_path): + """flatten_to_root emits a top-level _generated.py re-exporting every name.""" + model = { + "package": "pychaste", + "modules": [ + { + "name": "all", + "compiled_module": "_pychaste_all", + "imports": [], + "classes": [ + _class("Node", [_inst(["2"], "Node_2")]), + _class("FileFinder", [_inst([], "FileFinder")], templated=False), + ], + "enums": ["RelativeTo"], + "free_functions": [], + } + ], + } + layout = { + "package": "chaste", + "package_root": str(tmp_path), + "compiled_module": "_pychaste_all", + "flatten_to_root": True, + "subpackages": {"core": ["FileFinder", "RelativeTo"], "mesh": ["Node"]}, + } + + genpackage.generate_shared_module_split(model, layout, overwrite=False) + + root = (tmp_path / "_generated.py").read_text() + # Re-exports the base/enum names (not the concrete Node_2) from each subpackage. + assert "from chaste.core import (" in root + assert "from chaste.mesh import (" in root + assert "FileFinder," in root and "RelativeTo," in root and "Node," in root + assert "Node_2" not in root # the stub name is flattened, not the concrete + assert '"FileFinder",' in root and '"Node",' in root # __all__ + assert "__all__ = [" in root + + +def test_flatten_to_root_warns_on_name_clash(tmp_path, capsys): + """A base name exported by two subpackages is flagged as ambiguous.""" + model = { + "package": "pkg", + "modules": [ + { + "name": "all", + "compiled_module": "_pkg_all", + "imports": [], + "classes": [_class("Dup", [_inst([], "Dup")], templated=False)], + "enums": [], + "free_functions": [], + } + ], + } + layout = { + "package": "pkg", + "package_root": str(tmp_path), + "compiled_module": "_pkg_all", + "flatten_to_root": True, + "subpackages": {"a": ["Dup"], "b": ["Dup"]}, + } + + genpackage.generate_shared_module_split(model, layout, overwrite=False) + assert "'Dup'" in capsys.readouterr().err # ambiguous top-level export + + +def test_shared_split_warns_on_unknown_and_unassigned(tmp_path, capsys): + model = { + "package": "pkg", + "modules": [ + { + "name": "all", + "compiled_module": "_pkg_all", + "imports": [], + "classes": [ + _class("Kept", [_inst([], "Kept")], templated=False), + _class("Orphan", [_inst([], "Orphan")], templated=False), + ], + "enums": [], + "free_functions": [], + } + ], + } + layout = { + "package": "pkg", + "package_root": str(tmp_path), + "compiled_module": "_pkg_all", + "subpackages": {"sub": ["Kept", "DoesNotExist"]}, + } + + genpackage.generate_shared_module_split(model, layout, overwrite=False) + + err = capsys.readouterr().err + assert "'DoesNotExist'" in err # listed but not in model + assert "'Orphan'" in err # in model but not assigned to a subpackage + + +def test_main_dispatches_genpackage_subcommand(monkeypatch): + """`cppwg genpackage ...` routes to genpackage.main with the remaining args.""" + captured = {} + + def fake_genpackage_main(argv): + captured["argv"] = argv + return 0 + + monkeypatch.setattr(genpackage, "main", fake_genpackage_main) + monkeypatch.setattr( + sys, "argv", ["cppwg", "genpackage", "--model", "m.yaml", "--layout", "l.yaml"] + ) + + with pytest.raises(SystemExit) as exc_info: + cppwg_main.main() + + assert exc_info.value.code == 0 + # The "genpackage" token is stripped; only the sub-args reach genpackage.main. + assert captured["argv"] == ["--model", "m.yaml", "--layout", "l.yaml"] + + +def test_main_end_to_end(tmp_path, capsys): + """main() reads the model + layout files, resolves a relative package_root, writes files.""" + model = { + "package": "pyshapes", + "modules": [ + { + "name": "geometry", + "compiled_module": "_pyshapes_geometry", + "imports": [], + "enums": [], + "free_functions": [], + "classes": [_class("Point", [_inst(["2"], "Point_2")])], + } + ], + } + # A relative package_root is resolved against the layout file's own directory. + layout = {"package": "pyshapes", "package_root": "pkg"} + (tmp_path / "model.yaml").write_text(json.dumps(model)) # JSON is valid YAML + (tmp_path / "package_layout.yaml").write_text(json.dumps(layout)) + argv = [ + "--model", + str(tmp_path / "model.yaml"), + "--layout", + str(tmp_path / "package_layout.yaml"), + ] + + assert genpackage.main(argv) == 0 + generated = tmp_path / "pkg" / "geometry" / "_generated.py" + assert "class Point(TemplateClass):" in generated.read_text() + assert "wrote" in capsys.readouterr().out + + # A second run leaves the file untouched (the "unchanged" branch). + assert genpackage.main(argv) == 0 + assert "unchanged" in capsys.readouterr().out + + +def test_main_dispatches_shared_split(tmp_path): + """main() takes the shared-split path when the layout has a `subpackages` key.""" + model = { + "package": "pychaste", + "modules": [ + { + "name": "all", + "compiled_module": "_pychaste_all", + "imports": [], + "enums": [], + "free_functions": [], + "classes": [_class("Node", [_inst(["2"], "Node_2")])], + } + ], + } + layout = { + "package": "chaste", + "package_root": str(tmp_path), + "compiled_module": "_pychaste_all", + "subpackages": {"mesh": ["Node"]}, + } + (tmp_path / "model.yaml").write_text(json.dumps(model)) + (tmp_path / "layout.yaml").write_text(json.dumps(layout)) + + code = genpackage.main( + ["--model", str(tmp_path / "model.yaml"), "--layout", str(tmp_path / "layout.yaml")] + ) + assert code == 0 + assert "class Node(TemplateClass):" in (tmp_path / "mesh" / "_generated.py").read_text() + + +def test_flatten_skips_subpackage_with_no_exports(tmp_path): + """generate_root_flatten omits a subpackage that exports no names.""" + genpackage.generate_root_flatten( + "pkg", str(tmp_path), {"a": ["Widget"], "b": []}, overwrite=False + ) + + root = (tmp_path / "_generated.py").read_text() + assert "from pkg.a import (" in root # a exports a name + assert "from pkg.b import (" not in root # b exports nothing -> skipped + assert '"Widget",' in root # __all__ + + +def test_render_key_escapes_special_chars(): + """A value with special characters is escaped into a valid Python literal.""" + rendered = genpackage._render_key(['a"b', "c"]) + # Round-trips: the emitted tuple literal is valid Python, not `("a"b",...`. + assert ast.literal_eval(rendered) == ('a"b', "c") + + +def test_shared_split_empty_subpackage_emits_valid_import(tmp_path): + """An empty subpackage imports the extension module, not `from ... import ()`.""" + model = { + "package": "pkg", + "modules": [ + { + "name": "all", + "compiled_module": "_pkg_all", + "imports": [], + "enums": [], + "free_functions": [], + "classes": [_class("Kept", [_inst([], "Kept")], templated=False)], + } + ], + } + layout = { + "package": "pkg", + "package_root": str(tmp_path), + "compiled_module": "_pkg_all", + "subpackages": {"a": ["Kept"], "b": []}, # b owns nothing + } + + genpackage.generate_shared_module_split(model, layout, overwrite=False) + + # b would otherwise be `from pkg._pkg_all import (\n)`, a SyntaxError; it + # must be a plain, valid module import instead, aliased to a private name so + # `from ._generated import *` does not leak the top-level `pkg` name. + b = (tmp_path / "b" / "_generated.py").read_text() + assert "import pkg._pkg_all as _extension" in b + assert "import (" not in b + tree = ast.parse(b) # the whole file parses + # The only name bound is private (underscore-prefixed) -> not star-exported. + bound = { + (alias.asname or alias.name.split(".")[0]) + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for alias in node.names + } + assert bound == {"_extension"} # `pkg` is not bound, so it cannot leak + + +def test_warn_orphans_flags_only_unwritten_banner_files(tmp_path, capsys): + """A banner-owned _generated.py that was not written this run is flagged.""" + banner = '"""Generated by cppwg genpackage - do not edit."""\n' + # A live target (in the written set) -> not flagged. + (tmp_path / "live").mkdir() + live = tmp_path / "live" / "_generated.py" + live.write_text(banner) + # An orphan: banner-owned but not written this run -> flagged. + (tmp_path / "orphan").mkdir() + orphan = tmp_path / "orphan" / "_generated.py" + orphan.write_text(banner) + # A hand-written _generated.py with no banner -> never flagged. + (tmp_path / "foreign").mkdir() + foreign = tmp_path / "foreign" / "_generated.py" + foreign.write_text("x = 1\n") + + genpackage._warn_orphans(str(tmp_path), [str(live)]) + + err = capsys.readouterr().err + assert str(orphan) in err # orphan flagged + assert str(live) not in err # live target not flagged + assert str(foreign) not in err # non-banner file left alone diff --git a/tests/test_package_model.py b/tests/test_package_model.py new file mode 100644 index 00000000..f527fcb2 --- /dev/null +++ b/tests/test_package_model.py @@ -0,0 +1,280 @@ +"""Unit tests for cppwg.utils.package_model.""" + +from types import SimpleNamespace + +from cppwg.utils.package_model import build_package_model, compiled_module_name + + +def _class(base, py_names, template_arg_lists=(), cpp_names=None, excluded=False): + return SimpleNamespace( + excluded=excluded, + py_names=list(py_names), + cpp_names=list(cpp_names if cpp_names is not None else py_names), + template_arg_lists=[list(a) for a in template_arg_lists], + py_name_base=lambda base=base: base, + ) + + +def _enum(name, name_override="", excluded=False, exported_values=None): + values = list(exported_values or []) + return SimpleNamespace( + name=name, + name_override=name_override, + excluded=excluded, + decls=[SimpleNamespace(values=[(v, i) for i, v in enumerate(values)])], + should_export_values=lambda values=values: bool(values), + ) + + +def _free_function(name, excluded=False, arg_types=(), return_type="void", excludes=None): + _ex = excludes or {} + decl = SimpleNamespace( + return_type=SimpleNamespace(decl_string=return_type), + argument_types=[SimpleNamespace(decl_string=a) for a in arg_types], + ) + return SimpleNamespace( + name=name, + excluded=excluded, + decls=[decl], + hierarchy_attribute_gather_flat=lambda key: list(_ex.get(key, [])), + ) + + +def _module(name, classes=(), enums=(), free_functions=(), imports=()): + return SimpleNamespace( + name=name, + imports=list(imports), + class_collection=list(classes), + enum_collection=list(enums), + free_function_collection=list(free_functions), + ) + + +def _package(name, modules): + return SimpleNamespace(name=name, module_collection=list(modules)) + + +def test_compiled_module_name(): + assert compiled_module_name("pyshapes", "geometry") == "_pyshapes_geometry" + assert compiled_module_name("pycells", "all") == "_pycells_all" + + +def test_build_model_templated_and_untemplated(): + package = _package( + "pyshapes", + [ + _module( + "geometry", + classes=[ + _class( + "Point", + ["Point_2", "Point_3"], + [[2], [3]], + cpp_names=["Point<2>", "Point<3>"], + ) + ], + ), + _module( + "primitives", + classes=[ + _class( + "Shape", + ["Shape_2", "Shape_3"], + [[2], [3]], + cpp_names=["Shape<2>", "Shape<3>"], + ), + _class("UnitSquare", ["UnitSquare"]), # untemplated + ], + enums=[_enum("ShapeKind")], + imports=["pyshapes.geometry._pyshapes_geometry"], + ), + ], + ) + + model = build_package_model(package) + + assert model["package"] == "pyshapes" + geometry, primitives = model["modules"] + + assert geometry["compiled_module"] == "_pyshapes_geometry" + (point,) = geometry["classes"] + assert point == { + "base": "Point", + "templated": True, + "instantiations": [ + {"args": ["2"], "cpp_name": "Point<2>", "py_name": "Point_2"}, + {"args": ["3"], "cpp_name": "Point<3>", "py_name": "Point_3"}, + ], + } + + assert primitives["imports"] == ["pyshapes.geometry._pyshapes_geometry"] + shape, unit_square = primitives["classes"] + assert shape["templated"] is True + assert unit_square == { + "base": "UnitSquare", + "templated": False, + "instantiations": [ + {"args": [], "cpp_name": "UnitSquare", "py_name": "UnitSquare"} + ], + } + assert primitives["enums"] == ["ShapeKind"] + + +def test_build_model_omits_excluded_and_pruned(): + package = _package( + "pkg", + [ + _module( + "mod", + classes=[ + _class("Kept", ["Kept"]), + _class("Gone", ["Gone"], excluded=True), # excluded + _class("Pruned", [], [[2]]), # all instantiations pruned away + ], + enums=[_enum("KeptEnum"), _enum("GoneEnum", excluded=True)], + free_functions=[ + _free_function("kept_fn"), + _free_function("gone_fn", excluded=True), + ], + ) + ], + ) + + (module,) = build_package_model(package)["modules"] + + assert [c["base"] for c in module["classes"]] == ["Kept"] + assert module["enums"] == ["KeptEnum"] + assert module["free_functions"] == ["kept_fn"] + + +def test_build_model_multi_arg_and_class_arg_keys(): + """Multi-arg and class-name-arg instantiations stringify each argument.""" + package = _package( + "pycells", + [ + _module( + "all", + classes=[ + _class( + "MacroMesh", + ["MacroMesh_2_2"], + [[2, 2]], + cpp_names=["MacroMesh<2, 2>"], + ), + _class( + "CellFactory", + ["CellFactory_Cell_2"], + [["Cell", 2]], + cpp_names=["CellFactory"], + ), + ], + ) + ], + ) + + (module,) = build_package_model(package)["modules"] + macro, factory = module["classes"] + assert macro["instantiations"] == [ + {"args": ["2", "2"], "cpp_name": "MacroMesh<2, 2>", "py_name": "MacroMesh_2_2"} + ] + assert factory["instantiations"] == [ + { + "args": ["Cell", "2"], + "cpp_name": "CellFactory", + "py_name": "CellFactory_Cell_2", + } + ] + + +def test_instantiation_carries_cpp_name_for_name_override(): + """cpp_name is the C++ type name (from cpp_names), not the overridden py name.""" + package = _package( + "pkg", + [ + _module( + "mod", + classes=[ + _class("NewName", ["NewName_2"], [[2]], cpp_names=["OldName<2>"]), + ], + ) + ], + ) + + (module,) = build_package_model(package)["modules"] + (cls,) = module["classes"] + assert cls["instantiations"] == [ + {"args": ["2"], "cpp_name": "OldName<2>", "py_name": "NewName_2"} + ] + + +def test_untemplated_name_override_carries_cpp_name(): + """A renamed untemplated class records its C++ name, distinct from py_name. + + This lets the package-layer generator resolve the class when it appears as a + template argument spelled with its C++ name (OldName -> NewName). + """ + package = _package( + "pkg", + [_module("mod", classes=[_class("NewName", ["NewName"], cpp_names=["OldName"])])], + ) + (module,) = build_package_model(package)["modules"] + (cls,) = module["classes"] + assert cls["instantiations"] == [ + {"args": [], "cpp_name": "OldName", "py_name": "NewName"} + ] + + +def test_build_model_omits_free_function_excluded_by_type(): + """A free function the writer drops for an excluded arg type is not recorded. + + Its binding is never emitted, so recording it would make a shared-module + layout import a symbol that does not exist in the compiled extension. + """ + package = _package( + "pkg", + [ + _module( + "mod", + free_functions=[ + _free_function("kept_fn"), + _free_function( + "shape_fn", + arg_types=["::Shape<2> const &"], + excludes={"arg_type_excludes": ["Shape"]}, + ), + ], + ) + ], + ) + + (module,) = build_package_model(package)["modules"] + assert module["free_functions"] == ["kept_fn"] # shape_fn dropped, not recorded + + +def test_enum_name_override_used(): + package = _package( + "pkg", [_module("mod", enums=[_enum("RawName", name_override="PyName")])] + ) + (module,) = build_package_model(package)["modules"] + assert module["enums"] == ["PyName"] + # A non-exporting enum contributes no enum_exports entry. + assert module["enum_exports"] == {} + + +def test_enum_exports_records_exported_enumerators(): + """A value-exporting enum records its enumerators, keyed by the enum py-name.""" + package = _package( + "pkg", + [ + _module( + "mod", + enums=[ + _enum("ShapeKind", exported_values=["CIRCLE", "SQUARE"]), + _enum("Scoped"), # scoped/non-exporting -> not recorded + ], + ) + ], + ) + (module,) = build_package_model(package)["modules"] + assert module["enums"] == ["ShapeKind", "Scoped"] + assert module["enum_exports"] == {"ShapeKind": ["CIRCLE", "SQUARE"]}