refactor(mail): move to core/ and delivery/ module layout (moves only)

CHANGELOG.md

@@ -17,6 +17,7 @@ All notable changes to this project are documented in this file. The format is b
 ### Changed
 
 - Scaffold new modules with `core/` and `web/` areas and mirrored nested Python tests; existing modules migrate individually. See `docs/module-layout.md`.
+- `mail` follows the module layout: `mail.core.{constants,settings,state,data_compatibility}` and `mail.delivery.{sender,runtime}`. `mail.settings` and `mail.constants` still import, with a `DeprecationWarning`, until 0.0.46.
 
 ### Fixed
 

docs/module-layout.md

@@ -131,29 +131,36 @@ Inventory old import paths used outside the module, both in this repository
 and downstream. Every such old path becomes a shim using
 `simple_module_core.deprecation.deprecated_module`, with `remove_in="0.0.46"`.
 Update internal imports to the new path; do not make new code depend on shims.
-Register every shim in `host/tests/test_deprecation_expiry.py`'s import list.
-The expiry test fails once the installed framework reaches the removal release.
+A shim built as the `sys.modules` alias below is discovered automatically by
+`host/tests/test_deprecation_expiry.py` (`_layout_shims()`), so it needs no manual
+registration. The expiry test fails once the installed framework reaches the removal release.
 See [Deprecations](framework/deprecations.md) for the API.
 
 For example, after moving `orders.service` to `orders.web.service`, leave this
 at `orders/service.py` if external callers use that old path:
 
 ```python
-"""Compatibility import for the module-layout migration."""
+"""Moved to :mod:`orders.web.service`; this alias goes in 0.0.46 (module layout)."""
 
-from simple_module_core.deprecation import deprecated_module
+from __future__ import annotations
+
+import importlib
+import sys
 
-from orders.web.service import OrderService as OrderService
+from simple_module_core.deprecation import deprecated_module
 
 deprecated_module("orders.service", remove_in="0.0.46", use="orders.web.service")
+sys.modules[__name__] = importlib.import_module("orders.web.service")
 ```
 
-Add `"orders.service"` to both the expiry test's `_SHIM_MODULES` tuple and
-`_EXPECTED` set, and pin that
-both paths expose the same class, with an import-time `DeprecationWarning`.
-Keep `service.py` in the module's frozen legacy entries until the shim is
-removed. A completely migrated module leaves `LEGACY_FLAT` only when its extra
-roots, including compatibility shims, are gone and its Python tests are nested.
+The alias makes both paths the same module object, so mocks and
+`monkeypatch` strings keep working. Files containing both `deprecated_module(`
+and `sys.modules[__name__]` are auto-discovered by the expiry test and ignored
+by the root-layout test, so the module can leave `LEGACY_FLAT` while they
+remain. A shim that is a plain re-export instead (not an alias) must be added
+by hand to the expiry test's `_SHIM_MODULES` tuple and `_EXPECTED` set and kept
+in the module's frozen legacy entries until removed. A completely migrated
+module leaves `LEGACY_FLAT` once its Python tests are nested.
 
 ## Moving an existing module
 
@@ -169,8 +176,9 @@ One module per PR; no logic changes in a move PR.
 3. Move tests to matching areas/features, keep root `conftest.py`, remove test
    package initializers and prefix each basename with the module name.
    JS tests remain in `tests-js/`.
-4. Add external-import shims with removal release `0.0.46`, register them in
-   the expiry test, shrink the exact frozen root exceptions, and update the
+4. Add external-import shims with removal release `0.0.46`, alias shims are
+   auto-discovered by the expiry test (re-export shims are registered by hand),
+   shrink the exact frozen root exceptions, and update the
    status table below. Domain areas must be named in `DOMAIN_AREAS`.
 5. Run `python -c "import <pkg>.<mod>"` in a **fresh interpreter for every
    module**, not one interpreter importing all modules in sequence.
@@ -183,7 +191,7 @@ One module per PR; no logic changes in a move PR.
 ## Migration status (2026-11-05)
 
 Counts refer to the source package root and direct `tests/test_*.py` files;
-cache directories are ignored. All modules are flat in PR 0. Update this table
+cache directories are ignored. Shim files left at old paths are counted separately. Update this table
 when a later module PR moves files.
 
 | Module | Root `.py` files | Flat `tests/test_*.py` | State |
@@ -197,7 +205,7 @@ when a later module PR moves files.
 | feature_flags | 8 | 9 | flat |
 | file_storage | 39 | 65 | flat |
 | keycloak | 13 | 19 | flat |
-| mail | 8 | 5 | flat |
+| mail | 2 (+2 shims) | 0 | follows this layout |
 | oidc | 16 | 24 | flat |
 | permissions | 12 | 14 | flat |
 | settings | 31 | 43 | flat |

framework/cli/tests/test_cli_prepare_data_migrations.py

@@ -178,7 +178,7 @@ def find_spec(self, fullname, path=None, target=None):
 
 
 def test_cli_recipe_ids_match_the_module_constants():
-    from mail.constants import UPGRADE_SETTINGS_NAMESPACE
+    from mail.core.constants import UPGRADE_SETTINGS_NAMESPACE
     from simple_module_cli.migrations_cmd import _RECIPES
     from users.constants import UPGRADE_HASHED_BEARER_TOKENS
 

host/tests/test_deprecation_expiry.py

@@ -11,6 +11,7 @@
 import importlib
 import warnings
 from importlib.metadata import version
+from pathlib import Path
 
 import pytest
 from simple_module_core.deprecation import (
@@ -48,10 +49,27 @@
 }
 
 
+_MODULES_DIR = Path(__file__).resolve().parents[2] / "modules"
+
+
+def _layout_shims() -> list[str]:
+    """Old import paths left behind by module-layout moves (docs/module-layout.md)."""
+    found = []
+    # Package sources only: ``modules/<m>/<m>/``, not tests or node_modules.
+    for path in sorted(_MODULES_DIR.glob("*/*/**/*.py")):
+        if path.relative_to(_MODULES_DIR).parts[0] != path.relative_to(_MODULES_DIR).parts[1]:
+            continue
+        text = path.read_text(encoding="utf-8")
+        if "deprecated_module(" in text and "sys.modules[__name__]" in text:
+            rel = path.relative_to(_MODULES_DIR).with_suffix("").parts[1:]
+            found.append(".".join(rel))
+    return found
+
+
 def _register_all() -> None:
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", DeprecationWarning)
-        for name in _SHIM_MODULES:
+        for name in (*_SHIM_MODULES, *_layout_shims()):
             importlib.import_module(name)
     # Overrides own event-hook calls, so omission cannot warn reliably at the base.
     from simple_module_core.module import EVENT_HANDLERS_OPTIONAL_APP_REMOVE_IN
@@ -73,6 +91,12 @@ def test_every_known_shim_is_registered() -> None:
     assert {d.name for d in registered_deprecations()} >= _EXPECTED
 
 
+def test_every_layout_shim_registers_its_old_path() -> None:
+    _register_all()
+    registered = {d.name for d in registered_deprecations()}
+    assert set(_layout_shims()) <= registered
+
+
 def test_no_shim_outlives_its_remove_in_release() -> None:
     _register_all()
     expired = expired_deprecations(version("simple_module_core"))

host/tests/test_module_layout.py

@@ -31,7 +31,9 @@
     }
 )
 # Add named domain areas here only when they contain several related features.
-DOMAIN_AREAS: dict[str, frozenset[str]] = {}
+DOMAIN_AREAS: dict[str, frozenset[str]] = {
+    "mail": frozenset({"delivery"}),
+}
 
 # Frozen inventory, not a computed baseline: moves must remove their old entries.
 LEGACY_FLAT: dict[str, frozenset[str]] = {
@@ -98,11 +100,6 @@
         renewal.py settings.py state.py tokens.py
         """.split()
     ),
-    "mail": frozenset(
-        """
-        constants.py data_compatibility.py runtime.py sender.py settings.py state.py
-        """.split()
-    ),
     "oidc": frozenset(
         """
         client.py constants.py directory.py discovery.py endpoints jwks.py persistence.py
@@ -156,14 +153,21 @@
 }
 
 
+def _is_layout_shim(path: Path) -> bool:
+    """A moved module's old path: a deprecated alias, allowed until its remove_in."""
+    text = path.read_text(encoding="utf-8")
+    return "deprecated_module(" in text and "sys.modules[__name__]" in text
+
+
 @pytest.mark.parametrize("module", sorted(p.name for p in MODULES.iterdir() if p.is_dir()))
 def test_module_root_layout(module: str) -> None:
     package = MODULES / module / module
     assert package.is_dir(), f"Missing package for {module}"
     entries = {
         p.name
         for p in package.iterdir()
-        if p.suffix == ".py" or (p.is_dir() and p.name != "__pycache__")
+        if (p.suffix == ".py" and not _is_layout_shim(p))
+        or (p.is_dir() and p.name != "__pycache__")
     }
     allowed = ALLOWED_ROOT | DOMAIN_AREAS.get(module, frozenset())
     legacy = LEGACY_FLAT.get(module, frozenset())

modules/mail/mail/constants.py

@@ -1,35 +1,11 @@
-"""Every literal the mail module depends on."""
+"""Moved to :mod:`mail.core.constants`; this alias goes in 0.0.46 (module layout)."""
 
 from __future__ import annotations
 
-MODULE_NAME = "Mail"
-MODULE_PACKAGE = "mail"
+import importlib
+import sys
 
-# Dependency names must match the other modules' ``ModuleMeta.name``.
-MODULE_SETTINGS = "Settings"
+from simple_module_core.deprecation import deprecated_module
 
-BACKEND_CONSOLE = "console"
-BACKEND_SMTP = "smtp"
-
-# On-demand health check; also drives the settings page's "Test connection".
-CHECK_MAILER = "mail.mailer"
-
-DEFAULT_FROM = "no-reply@localhost"
-DEFAULT_SMTP_PORT = 587
-MIN_PORT = 1
-MAX_PORT = 65535
-
-#: Seconds per SMTP operation (aiosmtplib defaults to 60). A send can run in a
-#: request's after-commit hook, so a hung server must not hold it for a minute.
-SMTP_TIMEOUT_SECONDS = 15.0
-
-UPGRADE_SETTINGS_NAMESPACE = "mail.settings_namespace_v1"
-LEGACY_MAIL_KEYS = (
-    "users.mailer",
-    "users.smtp_host",
-    "users.smtp_port",
-    "users.smtp_username",
-    "users.smtp_password",
-    "users.smtp_from",
-    "users.smtp_tls",
-)
+deprecated_module("mail.constants", remove_in="0.0.46", use="mail.core.constants")
+sys.modules[__name__] = importlib.import_module("mail.core.constants")

modules/mail/mail/core/__init__.py

@@ -0,0 +1 @@
+"""What every mail feature builds on: constants, settings, app state, upgrade checks."""

modules/mail/mail/core/constants.py

@@ -0,0 +1,35 @@
+"""Every literal the mail module depends on."""
+
+from __future__ import annotations
+
+MODULE_NAME = "Mail"
+MODULE_PACKAGE = "mail"
+
+# Dependency names must match the other modules' ``ModuleMeta.name``.
+MODULE_SETTINGS = "Settings"
+
+BACKEND_CONSOLE = "console"
+BACKEND_SMTP = "smtp"
+
+# On-demand health check; also drives the settings page's "Test connection".
+CHECK_MAILER = "mail.mailer"
+
+DEFAULT_FROM = "no-reply@localhost"
+DEFAULT_SMTP_PORT = 587
+MIN_PORT = 1
+MAX_PORT = 65535
+
+#: Seconds per SMTP operation (aiosmtplib defaults to 60). A send can run in a
+#: request's after-commit hook, so a hung server must not hold it for a minute.
+SMTP_TIMEOUT_SECONDS = 15.0
+
+UPGRADE_SETTINGS_NAMESPACE = "mail.settings_namespace_v1"
+LEGACY_MAIL_KEYS = (
+    "users.mailer",
+    "users.smtp_host",
+    "users.smtp_port",
+    "users.smtp_username",
+    "users.smtp_password",
+    "users.smtp_from",
+    "users.smtp_tls",
+)

modules/mail/mail/data_compatibility.py → modules/mail/mail/core/data_compatibility.py

@@ -10,7 +10,7 @@
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from mail.constants import LEGACY_MAIL_KEYS, UPGRADE_SETTINGS_NAMESPACE
+from mail.core.constants import LEGACY_MAIL_KEYS, UPGRADE_SETTINGS_NAMESPACE
 
 
 async def check_data_compatibility(db: AsyncSession) -> list[DataCompatibilityIssue]:

modules/mail/mail/core/settings.py

@@ -0,0 +1,30 @@
+"""Mail module settings — DB-backed via ``register_module_settings``.
+
+``smtp_password`` is masked by the settings UI because its name matches the
+secret-field pattern (``simple_module_core.secret_fields.is_secret_field``).
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import Field
+from pydantic_settings import SettingsConfigDict
+from simple_module_core.settings_base import DbBackedSettings
+
+from mail.core import constants as c
+
+
+class MailSettings(DbBackedSettings):
+    """Which transport sends mail, and the SMTP account it uses."""
+
+    # ``DbBackedSettings`` so the DB is the only source (GH #283).
+    model_config = SettingsConfigDict(extra="ignore")
+
+    backend: Literal["console", "smtp"] = c.BACKEND_CONSOLE
+    smtp_host: str = ""
+    smtp_port: int = Field(default=c.DEFAULT_SMTP_PORT, ge=c.MIN_PORT, le=c.MAX_PORT)
+    smtp_username: str = ""
+    smtp_password: str = ""
+    smtp_from: str = c.DEFAULT_FROM
+    smtp_tls: bool = True

modules/mail/mail/state.py → modules/mail/mail/core/state.py

@@ -4,7 +4,7 @@
 
 from dataclasses import dataclass
 
-from mail.settings import MailSettings
+from mail.core.settings import MailSettings
 
 
 @dataclass

modules/mail/mail/delivery/__init__.py

@@ -0,0 +1 @@
+"""Sending mail: the SMTP/console senders and the app-free resolver and health check."""

modules/mail/mail/runtime.py → modules/mail/mail/delivery/runtime.py

@@ -12,9 +12,9 @@
 from simple_module_core.mail import MailSender, current_mail_sender, install_mail_sender
 from simple_module_core.settings_hydrate import hydrate_settings
 
-from mail import constants as c
-from mail.sender import build_sender
-from mail.settings import MailSettings
+from mail.core import constants as c
+from mail.core.settings import MailSettings
+from mail.delivery.sender import build_sender
 
 
 async def resolve_sender_from_db(db: Any) -> MailSender:

modules/mail/mail/sender.py → modules/mail/mail/delivery/sender.py

@@ -8,8 +8,8 @@
 import aiosmtplib
 from simple_module_core.mail import ConsoleMailSender, MailMessage, MailSender
 
-from mail import constants as c
-from mail.settings import MailSettings
+from mail.core import constants as c
+from mail.core.settings import MailSettings
 
 
 class SmtpMailSender: