From 3eb8ebd6b37fd4a38737e339ec918226de248f26 Mon Sep 17 00:00:00 2001 From: Asaf Varon Date: Fri, 16 Oct 2026 14:57:34 +0300 Subject: [PATCH 1/7] test(manager): close SQL engines owned by repository tests Repository-contract fixtures and SQL conversation-creation helpers created aiosqlite engines without disposing them, causing intermittent worker-thread exceptions. Each test now owns and disposes its engines. Co-Authored-By: Claude Opus 5.5 --- tasks/0018-sql-test-resource-cleanup.md | 58 +++++++++++++++++ .../agent_manager/test_repository_contract.py | 62 ++++++++++--------- .../test_sql_conversation_creation.py | 61 ++++++++++++------ 3 files changed, 132 insertions(+), 49 deletions(-) create mode 100644 tasks/0018-sql-test-resource-cleanup.md diff --git a/tasks/0018-sql-test-resource-cleanup.md b/tasks/0018-sql-test-resource-cleanup.md new file mode 100644 index 00000000..d2d1bb43 --- /dev/null +++ b/tasks/0018-sql-test-resource-cleanup.md @@ -0,0 +1,58 @@ +# Task 0018 — Close SQL engines owned by existing repository tests + +## Goal / context + +The user's review-and-fix request exposed intermittent aiosqlite worker-thread +exceptions in the full gate. A diagnostic recording connection creation traced +unclosed connections to existing repository-contract fixtures and SQL +conversation-creation helpers. Keep this correction separate from task 0017's +WhatsApp implementation fixes. No product contract changes are involved. + +## Read first + +AGENTS.md, .ai/README.md, project-architecture, testing, +senior-python-engineering, systematic debugging, and verification skills; +the two test modules and persistence/database.py. + +## Scope / files allowed to change + +- This task and tasks/README.md. +- tests/agent_manager/test_repository_contract.py. +- tests/agent_manager/test_sql_conversation_creation.py. +- docs/WHATSAPP_REVIEW.md, to record the verification follow-up. + +## Requirements / acceptance criteria + +- [x] Demonstrate leaked live connections in these two modules before editing. +- [x] Test fixtures own and dispose every engine they create, including + failure/assertion paths and repositories used through HTTP clients. +- [x] Preserve test assertions and the separate-connection concurrency setup. +- [x] Repeat the connection diagnostic: no owned live connections remain after + the modules finish. Run the full make check and whitespace validation. + +## Out of scope + +Runtime/database behavior, dependencies, engine or manager public contracts, +other test modules, warning suppression, and WhatsApp product features. + +## Commands / expected report + +Use an in-process diagnostic around real local aiosqlite creation and the two +test modules, followed by make check and git diff --check. No live external +database, model, MCP, or Meta calls. Report this as test-resource cleanup, +separately from the three product corrections in task 0017. + +## Implementation result + +Completed on 2026-10-16. The pre-fix diagnostic ran the two modules successfully +but failed its resource assertion: 38 of 42 real local SQLite connections were +still open. Managed async fixtures/context managers now dispose each owned +engine, including test/setup failure paths. The same diagnostic then passed: +61 tests, 42 connections created, zero left open. Existing assertions and +independent-engine concurrency scenarios are preserved. + +Final `make check` passed with 1,327 tests and the 24 prior dependency-warning +instances; formatting, lint, mypy, generated-stub checks, whitespace and doc +links passed. The earlier aiosqlite closed-loop thread warnings were absent. +No production source or contracts changed in this follow-up; no warnings were +suppressed, and no dependency or external service was changed. diff --git a/tests/agent_manager/test_repository_contract.py b/tests/agent_manager/test_repository_contract.py index 6c161349..0a8ef58e 100644 --- a/tests/agent_manager/test_repository_contract.py +++ b/tests/agent_manager/test_repository_contract.py @@ -7,10 +7,13 @@ from __future__ import annotations import asyncio +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager from datetime import UTC, datetime, timedelta from uuid import uuid4 import pytest +from sqlalchemy.ext.asyncio import AsyncEngine from sqlmodel import SQLModel, select import agent_manager.infrastructure.persistence.tables # noqa: F401 (register tables) @@ -30,20 +33,24 @@ ) -async def _memory() -> Repository: - return MemoryRepository() - - -async def _sql() -> Repository: +@asynccontextmanager +async def _sql_database() -> AsyncIterator[AsyncEngine]: engine = create_db_engine("sqlite+aiosqlite:///:memory:") - async with engine.begin() as conn: - await conn.run_sync(SQLModel.metadata.create_all) - return SqlRepository(session_factory(engine)) + try: + async with engine.begin() as conn: + await conn.run_sync(SQLModel.metadata.create_all) + yield engine + finally: + await engine.dispose() -@pytest.fixture(params=[_memory, _sql], ids=["memory", "sql"]) -async def repo(request: pytest.FixtureRequest) -> Repository: - return await request.param() +@pytest.fixture(params=["memory", "sql"]) +async def repo(request: pytest.FixtureRequest) -> AsyncIterator[Repository]: + if request.param == "memory": + yield MemoryRepository() + else: + async with _sql_database() as engine: + yield SqlRepository(session_factory(engine)) async def test_create_and_exists(repo: Repository) -> None: @@ -469,17 +476,15 @@ async def test_different_run_ids_can_share_one_session(repo: Repository) -> None async def test_sql_schema_has_cold_and_snapshot_rows() -> None: - engine = create_db_engine("sqlite+aiosqlite:///:memory:") - async with engine.begin() as conn: - await conn.run_sync(SQLModel.metadata.create_all) - sessions = session_factory(engine) - repo = SqlRepository(sessions) - await repo.create_session("sess-1") - await repo.append_message(_message("sess-1", Role.USER, "hello")) + async with _sql_database() as engine: + sessions = session_factory(engine) + repo = SqlRepository(sessions) + await repo.create_session("sess-1") + await repo.append_message(_message("sess-1", Role.USER, "hello")) - async with sessions() as session: - cold = (await session.exec(select(ConversationMessageRow))).all() - snapshots = (await session.exec(select(ConversationSnapshotRow))).all() + async with sessions() as session: + cold = (await session.exec(select(ConversationMessageRow))).all() + snapshots = (await session.exec(select(ConversationSnapshotRow))).all() assert len(cold) == 1 assert len(snapshots) == 1 @@ -503,17 +508,14 @@ def test_legacy_conversations_and_messages_tables_are_gone() -> None: async def test_sql_repository_does_not_write_legacy_tables() -> None: - engine = create_db_engine("sqlite+aiosqlite:///:memory:") - async with engine.begin() as conn: - await conn.run_sync(SQLModel.metadata.create_all) - sessions = session_factory(engine) - repo = SqlRepository(sessions) + async with _sql_database() as engine: + repo = SqlRepository(session_factory(engine)) - cid = await repo.create_conversation() - await repo.add_message(cid, Role.USER, "hi") + cid = await repo.create_conversation() + await repo.add_message(cid, Role.USER, "hi") - assert await repo.conversation_exists(cid) - assert [m.content for m in await repo.list_messages(cid)] == ["hi"] + assert await repo.conversation_exists(cid) + assert [m.content for m in await repo.list_messages(cid)] == ["hi"] def _message( diff --git a/tests/agent_manager/test_sql_conversation_creation.py b/tests/agent_manager/test_sql_conversation_creation.py index 074b8e17..ea640f34 100644 --- a/tests/agent_manager/test_sql_conversation_creation.py +++ b/tests/agent_manager/test_sql_conversation_creation.py @@ -10,13 +10,14 @@ from __future__ import annotations import asyncio -from collections.abc import Coroutine +from collections.abc import AsyncIterator, Callable, Coroutine from pathlib import Path from typing import Any import pytest from httpx import ASGITransport, AsyncClient, Response from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import AsyncEngine from sqlmodel import SQLModel from sqlmodel.ext.asyncio.session import AsyncSession @@ -26,24 +27,42 @@ from agent_manager.infrastructure.persistence.sql_repository import SqlRepository from tests.agent_manager.conftest import RecordingEngine, bearer, build_test_app +RepositoryFactory = Callable[[], SqlRepository] + @pytest.fixture async def db_url(tmp_path: Path) -> str: url = f"sqlite+aiosqlite:///{tmp_path / 'race.db'}" engine = create_db_engine(url) - async with engine.begin() as conn: - await conn.run_sync(SQLModel.metadata.create_all) - await engine.dispose() + try: + async with engine.begin() as conn: + await conn.run_sync(SQLModel.metadata.create_all) + finally: + await engine.dispose() return url -def _repository(db_url: str) -> SqlRepository: - """A repository on its own engine, so two of them can race for real.""" - return SqlRepository(session_factory(create_db_engine(db_url))) +@pytest.fixture +async def repository_factory(db_url: str) -> AsyncIterator[RepositoryFactory]: + """Separate engines for real races, all owned by this test's lifetime.""" + engines: list[AsyncEngine] = [] + + def create() -> SqlRepository: + engine = create_db_engine(db_url) + engines.append(engine) + return SqlRepository(session_factory(engine)) + try: + yield create + finally: + for engine in engines: + await engine.dispose() -async def test_concurrent_create_session_settles_on_one_owner(db_url: str) -> None: - alice, bob = _repository(db_url), _repository(db_url) + +async def test_concurrent_create_session_settles_on_one_owner( + repository_factory: RepositoryFactory, +) -> None: + alice, bob = repository_factory(), repository_factory() results = await asyncio.gather( alice.create_session("shared-id", user_id="alice"), @@ -63,11 +82,11 @@ async def test_concurrent_create_session_settles_on_one_owner(db_url: str) -> No async def test_an_integrity_error_that_is_not_a_duplicate_id_is_not_swallowed( - db_url: str, monkeypatch: pytest.MonkeyPatch + repository_factory: RepositoryFactory, monkeypatch: pytest.MonkeyPatch ) -> None: """Only a taken id is absorbed. Reading back nothing means the insert failed for another reason — a foreign key, a not-null — and that belongs upstream.""" - repo = _repository(db_url) + repo = repository_factory() await repo.create_session("shared-id", user_id="alice") # Blind the reads, so the insert both collides and finds no winner. @@ -80,12 +99,14 @@ async def blind(*_args: object, **_kwargs: object) -> None: await repo.create_session("shared-id", user_id="bob") -async def test_the_api_answers_409_to_the_losing_caller(db_url: str) -> None: +async def test_the_api_answers_409_to_the_losing_caller( + repository_factory: RepositoryFactory, +) -> None: """Two app instances on one database, the way two replicas would be — a single app serialises the requests and never reaches the race.""" - def client(db_url: str) -> AsyncClient: - app = build_test_app(ConversationService(RecordingEngine(), _repository(db_url))) + def client() -> AsyncClient: + app = build_test_app(ConversationService(RecordingEngine(), repository_factory())) return AsyncClient(transport=ASGITransport(app=app), base_url="http://test") def create(client: AsyncClient, user: str) -> Coroutine[Any, Any, Response]: @@ -95,13 +116,15 @@ def create(client: AsyncClient, user: str) -> Coroutine[Any, Any, Response]: headers=bearer(user), ) - async with client(db_url) as alice, client(db_url) as bob: + async with client() as alice, client() as bob: responses = await asyncio.gather(create(alice, "alice"), create(bob, "bob")) assert sorted(r.status_code for r in responses) == [200, 409] -async def test_a_rejected_create_leaves_the_other_system_s_session_alone(db_url: str) -> None: +async def test_a_rejected_create_leaves_the_other_system_s_session_alone( + repository_factory: RepositoryFactory, +) -> None: """Two systems sharing a conversation database, which is when this bites: naming a live id must not rebind it to the caller's system.""" @@ -109,7 +132,7 @@ def client(system: str) -> AsyncClient: app = build_test_app( ConversationService( RecordingEngine(), - _repository(db_url), + repository_factory(), system_name=system, config_path=f"/{system}/agents.yml", ) @@ -123,7 +146,7 @@ def client(system: str) -> AsyncClient: headers=bearer("alice"), ) assert created.status_code == 200 - before = await _repository(db_url).get_session("shared-id") + before = await repository_factory().get_session("shared-id") rejected = await bob.post( "/conversations", @@ -132,4 +155,4 @@ def client(system: str) -> AsyncClient: ) assert rejected.status_code == 409 - assert await _repository(db_url).get_session("shared-id") == before + assert await repository_factory().get_session("shared-id") == before From 53ad5845b76100576d16e934f607c8d7f702289e Mon Sep 17 00:00:00 2001 From: Asaf Varon Date: Fri, 16 Oct 2026 14:57:41 +0300 Subject: [PATCH 2/7] feat(manager): prepare conversation turns idempotently An opt-in idempotency key claims a durable receipt together with the user message, so a retried preparation recovers the same turn instead of appending another. Adds saved-response recovery for channel workers and migration 0007. Co-Authored-By: Claude Opus 5.5 --- ...dempotent-conversation-turn-preparation.md | 55 +++ .../application/conversation_service.py | 137 ++++++- src/agent_manager/domain/turn_preparation.py | 53 +++ .../persistence/memory_repository.py | 41 +++ .../0007_turn_preparation_receipts.py | 29 ++ .../persistence/sql_repository.py | 73 ++++ .../infrastructure/persistence/tables.py | 11 + tasks/0013-idempotent-conversation-turns.md | 58 +++ tests/agent_manager/test_turn_preparation.py | 337 ++++++++++++++++++ 9 files changed, 776 insertions(+), 18 deletions(-) create mode 100644 docs/adr/0006-idempotent-conversation-turn-preparation.md create mode 100644 src/agent_manager/domain/turn_preparation.py create mode 100644 src/agent_manager/infrastructure/persistence/migrations/versions/0007_turn_preparation_receipts.py create mode 100644 tasks/0013-idempotent-conversation-turns.md create mode 100644 tests/agent_manager/test_turn_preparation.py diff --git a/docs/adr/0006-idempotent-conversation-turn-preparation.md b/docs/adr/0006-idempotent-conversation-turn-preparation.md new file mode 100644 index 00000000..54a99707 --- /dev/null +++ b/docs/adr/0006-idempotent-conversation-turn-preparation.md @@ -0,0 +1,55 @@ +# ADR 0006 — Prepare retried conversation turns once + +- **Status:** Accepted for implementation following the user's approval of the WhatsApp plan on 2026-10-15. +- **Date:** 2026-10-15. + +## Context + +A durable channel can receive one provider message repeatedly. Deduplicating +the webhook does not close the crash between committing a manager user message +and storing its generated run/message IDs in the channel queue. Calling the +existing `prepare_turn()` again creates another turn and changes its history. + +## Decision + +Add an optional `idempotency_key` to `ConversationService.prepare_turn()`. +Preparation remains separate from execution: recovering a prepared turn never +authorizes executing it twice. Existing HTTP, CLI, and streaming contracts and +callers retain their current behavior. + +An optional repository capability atomically commits a hashed-key receipt, +the user message, the selected branch head, and the snapshot. A receipt owns +one conversation, user, exact content, message ID, and run ID. Reusing its key +with another owner/conversation/content fails without changing history. The +receipt survives head changes and restarts; history is reconstructed from the +original user message's parent rather than the current head. Idempotent edits +are deliberately unsupported. + +Register the durable run after the prepared message commits, then reconcile +that registration on a retry. A failure between those commits leaves a +recoverable receipt, not another user turn. Recovery does not reset run status +or token usage. A restored receipt bypasses new-turn budget checks because it +does not authorize additional work. New keys still obey the existing budget. + +The memory and SQL repository adapters implement the capability. Custom +repositories may omit it; requesting idempotence then fails explicitly before +writing anything. SQL adds one receipt table through an explicit migration; +no schema is created during application startup. + +## Consequences and alternatives + +Channel workers must claim execution durably and reconcile ambiguous runs; +the preparation key is not an exactly-once execution guarantee. The engine, +YAML, prompts, and client auth contracts stay unchanged. Keys must be bounded +and are stored only as SHA-256 digests, not logged. + +Deriving message IDs from a provider ID alone was rejected: it couples generic +conversation persistence to a provider and does not bind retries to original +content/ownership. A separate receipt write after appending was rejected +because it leaves the same crash gap. An in-memory cache cannot survive restart. + +## Enforcement + +Repository/service tests cover replay, conflicting input, branch changes, +concurrent same-key requests, failed appends, failed run registration, and SQL +restart. The full `make check` gate remains required. diff --git a/src/agent_manager/application/conversation_service.py b/src/agent_manager/application/conversation_service.py index f7bcea7a..f49ae259 100644 --- a/src/agent_manager/application/conversation_service.py +++ b/src/agent_manager/application/conversation_service.py @@ -52,12 +52,21 @@ TokenBudgetUsage, thread_title, ) +from agent_manager.domain.turn_preparation import ( + IdempotentTurnRepository, + preparation_key_digest, + validate_prepared_message, +) logger = logging.getLogger(__name__) T = TypeVar("T") +def _assistant_message_id(session_id: str, run_id: str) -> str: + return uuid.uuid5(uuid.NAMESPACE_URL, f"agent-manager:{session_id}:{run_id}:assistant").hex + + def _run_context(turn: PreparedConversationTurn) -> RunContext: """The per-run context one turn executes under, carrying the caller's own host credential so tools act as this user rather than share a privileged key.""" @@ -238,10 +247,28 @@ async def prepare_turn( principal: Principal, *, edit_message_id: str | None = None, + idempotency_key: str | None = None, ) -> PreparedConversationTurn: - """Persist a user message and return its isolated prior model context.""" + """Persist a user message and return its isolated prior model context. + + An opt-in key recovers preparation only. Its caller must claim execution + separately; preparing the same turn twice does not authorize two runs. + """ await self._authorize(conversation_id, principal) user_id = principal.user_id + preparation_repository: IdempotentTurnRepository | None = None + key_digest: str | None = None + if idempotency_key is not None: + key_digest = preparation_key_digest(idempotency_key) + if edit_message_id is not None: + raise ValueError("idempotent preparation cannot be combined with an edit") + if not isinstance(self._repository, IdempotentTurnRepository): + raise RuntimeError("repository does not support idempotent turn preparation") + preparation_repository = self._repository + existing = await preparation_repository.get_prepared_message(key_digest) + if existing is not None: + validate_prepared_message(existing, conversation_id, user_id, text) + return await self._restore_prepared_turn(existing, principal) await self._register(principal) if self._max_tokens is not None: @@ -253,7 +280,17 @@ async def prepare_turn( session = await self._require(conversation_id) expected_head = session.head_message_id parent_message_id = expected_head - if edit_message_id is None: + if preparation_repository is not None: + # Read the immutable parent directly. The normal hot-context path + # can rebuild a missing snapshot, which must not race a retry's + # atomic receipt/message/snapshot transaction. + prior_context = await self._repository.get_context_at( + conversation_id, + parent_message_id, + max_messages=self._window, + max_chars=self._max_chars, + ) + elif edit_message_id is None: prior_context = await self._repository.get_context( conversation_id, max_messages=self._window, @@ -281,20 +318,36 @@ async def prepare_turn( run_id = uuid.uuid4().hex message_id = uuid.uuid4().hex now = datetime.now(UTC) + message = ConversationMessage( + message_id=message_id, + session_id=conversation_id, + run_id=run_id, + user_id=user_id, + role=Role.USER, + content=text, + created_at=now, + parent_message_id=parent_message_id, + status="running", + ) + if preparation_repository is not None and key_digest is not None: + persisted = await preparation_repository.append_prepared_message( + key_digest, + message, + expected_head, + snapshot_ttl_seconds=self._snapshot_ttl_seconds, + ) + if persisted is None: + raise ConversationBranchConflict(conversation_id) + turn = await self._restore_prepared_turn(persisted, principal) + if persisted.message_id == message_id and expected_head is None: + turn = dataclasses.replace( + turn, title_task=await self._name_conversation(conversation_id, text) + ) + return turn await self._register_run(run_id) try: appended = await self._repository.append_message_if_head( - ConversationMessage( - message_id=message_id, - session_id=conversation_id, - run_id=run_id, - user_id=user_id, - role=Role.USER, - content=text, - created_at=now, - parent_message_id=parent_message_id, - status="running", - ), + message, expected_head, snapshot_ttl_seconds=self._snapshot_ttl_seconds, ) @@ -322,6 +375,27 @@ async def prepare_turn( title_task=title_task, ) + async def _restore_prepared_turn( + self, message: ConversationMessage, principal: Principal + ) -> PreparedConversationTurn: + assert message.run_id is not None + await self._register_run(message.run_id, reuse=True) + prior = await self._repository.get_context_at( + message.session_id, + message.parent_message_id, + max_messages=self._window, + max_chars=self._max_chars, + ) + return PreparedConversationTurn( + session_id=message.session_id, + run_id=message.run_id, + message_id=message.message_id, + user_id=principal.user_id, + message=message.content, + history=build_history(prior.messages, self._window), + principal=principal, + ) + async def complete_turn( self, turn: PreparedConversationTurn, @@ -342,6 +416,28 @@ async def complete_turn( used_tools=result.used_tools, ) + async def get_turn_response( + self, conversation_id: str, run_id: str, principal: Principal + ) -> ConversationMessage | None: + """Recover one already-persisted response, after normal owner authorization.""" + await self._authorize(conversation_id, principal) + user = await self._repository.get_user_message_for_run(conversation_id, run_id) + if user is None: + return None + response = await self._repository.get_message( + _assistant_message_id(conversation_id, run_id) + ) + if response is not None and ( + response.role is not Role.ASSISTANT + or response.session_id != conversation_id + or response.run_id != run_id + or response.parent_message_id != user.message_id + or response.user_id != principal.user_id + or response.status != "completed" + ): + raise ValueError("persisted response does not match its owned turn") + return response + async def decide_approval( self, conversation_id: str, @@ -696,10 +792,7 @@ async def _persist_assistant_turn( used_tools: Sequence[ToolUsageRecord], ) -> None: """Persist one normalized final assistant response; its tokens are the run's charge.""" - message_id = uuid.uuid5( - uuid.NAMESPACE_URL, - f"agent-manager:{session_id}:{run_id}:assistant", - ).hex + message_id = _assistant_message_id(session_id, run_id) await self._repository.append_message_if_absent( ConversationMessage( message_id=message_id, @@ -753,7 +846,7 @@ async def _with_run_statuses( for message in messages ] - async def _register_run(self, run_id: str) -> None: + async def _register_run(self, run_id: str, *, reuse: bool = False) -> None: if self._run_repository is None: return created = await self._run_repository.create_if_absent( @@ -764,6 +857,14 @@ async def _register_run(self, run_id: str) -> None: ) ) if not created: + if reuse: + existing = await self._run_repository.get(run_id) + if ( + existing is not None + and existing.thread_id == run_id + and existing.system_name == (self._system_name or "") + ): + return raise RuntimeError(f"generated run id already exists: {run_id}") async def _transition_run(self, run_id: str, target: RunStatus) -> None: diff --git a/src/agent_manager/domain/turn_preparation.py b/src/agent_manager/domain/turn_preparation.py new file mode 100644 index 00000000..e1e25e5b --- /dev/null +++ b/src/agent_manager/domain/turn_preparation.py @@ -0,0 +1,53 @@ +"""Optional atomic preparation capability, independent of any message provider.""" + +from __future__ import annotations + +import hashlib +from typing import Protocol, runtime_checkable + +from agent_manager.domain.models import ConversationMessage, Role + + +class TurnPreparationConflict(ValueError): + """An idempotency key already belongs to different preparation input.""" + + +def preparation_key_digest(key: str) -> str: + if not key.strip() or len(key.encode("utf-8")) > 512: + raise ValueError("Idempotency key must contain 1 to 512 UTF-8 bytes") + return hashlib.sha256(key.encode("utf-8")).hexdigest() + + +def validate_prepared_message( + message: ConversationMessage, conversation_id: str, user_id: str, content: str +) -> None: + if ( + message.session_id != conversation_id + or message.user_id != user_id + or message.content != content + or message.role is not Role.USER + or message.run_id is None + ): + raise TurnPreparationConflict("Idempotency key is bound to different turn input") + + +@runtime_checkable +class IdempotentTurnRepository(Protocol): + """A receipt and its user append share one atomic commit. + + An existing receipt returns the original message regardless of the current + branch head, after validating owner/conversation/content. A new key whose + expected head is stale returns None and writes nothing. Implementations + must not use a separate receipt transaction after appending the message. + """ + + async def get_prepared_message(self, key_digest: str) -> ConversationMessage | None: ... + + async def append_prepared_message( + self, + key_digest: str, + message: ConversationMessage, + expected_head_message_id: str | None, + *, + snapshot_ttl_seconds: int | None = None, + ) -> ConversationMessage | None: ... diff --git a/src/agent_manager/infrastructure/persistence/memory_repository.py b/src/agent_manager/infrastructure/persistence/memory_repository.py index cc5a8f2f..5d066ec5 100644 --- a/src/agent_manager/infrastructure/persistence/memory_repository.py +++ b/src/agent_manager/infrastructure/persistence/memory_repository.py @@ -26,6 +26,7 @@ encode_cursor, ensure_utc, ) +from agent_manager.domain.turn_preparation import validate_prepared_message _EPOCH = datetime(1970, 1, 1, tzinfo=UTC) @@ -53,6 +54,7 @@ def __init__(self) -> None: self._messages: dict[str, list[ConversationMessage]] = {} self._snapshots: dict[str, ConversationSnapshot] = {} self._run_token_usage: dict[str, dict[str, int]] = {} + self._preparations: dict[str, ConversationMessage] = {} async def upsert_user( self, @@ -263,6 +265,45 @@ async def get_message(self, message_id: str) -> ConversationMessage | None: None, ) + async def get_prepared_message(self, key_digest: str) -> ConversationMessage | None: + return self._preparations.get(key_digest) + + async def append_prepared_message( + self, + key_digest: str, + message: ConversationMessage, + expected_head_message_id: str | None, + *, + snapshot_ttl_seconds: int | None = None, + ) -> ConversationMessage | None: + existing = self._preparations.get(key_digest) + if existing is not None: + validate_prepared_message( + existing, message.session_id, message.user_id or "", message.content + ) + return existing + original_session = self._sessions.get(message.session_id) + if original_session is None or original_session.head_message_id != expected_head_message_id: + return None + original_messages = list(self._messages.get(message.session_id, [])) + original_snapshot = self._snapshots.get(message.session_id) + # No suspension points occur inside the memory append. Restore all + # projections if snapshot construction fails before the receipt commits. + try: + await self.append_message_if_head( + message, expected_head_message_id, snapshot_ttl_seconds=snapshot_ttl_seconds + ) + except BaseException: + self._sessions[message.session_id] = original_session + self._messages[message.session_id] = original_messages + if original_snapshot is None: + self._snapshots.pop(message.session_id, None) + else: + self._snapshots[message.session_id] = original_snapshot + raise + self._preparations[key_digest] = message + return message + async def get_user_message_for_run( self, session_id: str, run_id: str ) -> ConversationMessage | None: diff --git a/src/agent_manager/infrastructure/persistence/migrations/versions/0007_turn_preparation_receipts.py b/src/agent_manager/infrastructure/persistence/migrations/versions/0007_turn_preparation_receipts.py new file mode 100644 index 00000000..35ff787e --- /dev/null +++ b/src/agent_manager/infrastructure/persistence/migrations/versions/0007_turn_preparation_receipts.py @@ -0,0 +1,29 @@ +"""Atomically bind preparation keys to their original user messages. + +Revision ID: 0007 +Revises: 0006 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision: str = "0007" +down_revision: str | None = "0006" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "conversation_turn_preparations", + sa.Column("key_digest", sa.String(length=64), nullable=False), + sa.Column("message_id", sa.String(length=64), nullable=False), + sa.PrimaryKeyConstraint("key_digest"), + sa.UniqueConstraint("message_id"), + ) + + +def downgrade() -> None: + op.drop_table("conversation_turn_preparations") diff --git a/src/agent_manager/infrastructure/persistence/sql_repository.py b/src/agent_manager/infrastructure/persistence/sql_repository.py index 34e5df50..256c5df5 100644 --- a/src/agent_manager/infrastructure/persistence/sql_repository.py +++ b/src/agent_manager/infrastructure/persistence/sql_repository.py @@ -37,15 +37,21 @@ encode_cursor, ensure_utc, ) +from agent_manager.domain.turn_preparation import validate_prepared_message from agent_manager.infrastructure.persistence.tables import ( ConversationMessageRow, ConversationRunTokenUsageRow, ConversationSessionRow, ConversationSnapshotRow, ConversationUserRow, + TurnPreparationReceiptRow, ) +class _PreparationHeadChanged(Exception): + """Roll back the key claim together with a rejected branch append.""" + + class SqlRepository(Repository): def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: self._sessions = sessions @@ -359,6 +365,73 @@ async def get_message(self, message_id: str) -> ConversationMessage | None: row = await session.get(ConversationMessageRow, message_id) return _message(row) if row is not None else None + async def get_prepared_message(self, key_digest: str) -> ConversationMessage | None: + async with self._sessions() as session: + receipt = await session.get(TurnPreparationReceiptRow, key_digest) + if receipt is None: + return None + row = await session.get(ConversationMessageRow, receipt.message_id) + if row is None: + raise RuntimeError("turn preparation receipt has no message") + return _message(row) + + async def append_prepared_message( + self, + key_digest: str, + message: ConversationMessage, + expected_head_message_id: str | None, + *, + snapshot_ttl_seconds: int | None = None, + ) -> ConversationMessage | None: + try: + async with self._sessions() as session, session.begin(): + insert = ( + sqlite_insert(TurnPreparationReceiptRow) + if session.bind is not None and session.bind.dialect.name == "sqlite" + else postgres_insert(TurnPreparationReceiptRow) + ) + claimed = await session.exec( + insert.values(key_digest=key_digest, message_id=message.message_id) + .on_conflict_do_nothing(index_elements=["key_digest"]) + .returning(col(TurnPreparationReceiptRow.key_digest)) + ) + if claimed.scalar_one_or_none() is None: + receipt = await session.get(TurnPreparationReceiptRow, key_digest) + assert receipt is not None + row = await session.get(ConversationMessageRow, receipt.message_id) + if row is None: + raise RuntimeError("turn preparation receipt has no message") + existing = _message(row) + validate_prepared_message( + existing, message.session_id, message.user_id or "", message.content + ) + return existing + head = col(ConversationSessionRow.head_message_id) + expected = ( + head.is_(None) + if expected_head_message_id is None + else head == expected_head_message_id + ) + changed = await session.exec( + update(ConversationSessionRow) + .where(col(ConversationSessionRow.session_id) == message.session_id, expected) + .values( + head_message_id=message.message_id, + updated_at=message.created_at, + last_message_at=message.created_at, + ) + ) + if _rows_affected(changed) == 0: + raise _PreparationHeadChanged + session.add(_message_row(message)) + await session.flush() + await self._rebuild_snapshot_in_session( + session, message.session_id, snapshot_ttl_seconds=snapshot_ttl_seconds + ) + except _PreparationHeadChanged: + return None + return message + async def get_user_message_for_run( self, session_id: str, run_id: str ) -> ConversationMessage | None: diff --git a/src/agent_manager/infrastructure/persistence/tables.py b/src/agent_manager/infrastructure/persistence/tables.py index 7e09cecd..e0cb200f 100644 --- a/src/agent_manager/infrastructure/persistence/tables.py +++ b/src/agent_manager/infrastructure/persistence/tables.py @@ -107,6 +107,17 @@ class ConversationSnapshotRow(SQLModel, table=True): ) +class TurnPreparationReceiptRow(SQLModel, table=True): + """Committed with the user message; the receipt owns its original input.""" + + __tablename__ = "conversation_turn_preparations" + + key_digest: str = Field(primary_key=True, max_length=64) + # No immediate message FK: this row claims the key before the user append + # in the same transaction, so competing preparations serialize on the key. + message_id: str = Field(max_length=64, unique=True) + + class ConversationRunTokenUsageRow(SQLModel, table=True): """Lifetime run charges, retained independently of answers and run TTLs.""" diff --git a/tasks/0013-idempotent-conversation-turns.md b/tasks/0013-idempotent-conversation-turns.md new file mode 100644 index 00000000..7cd741f2 --- /dev/null +++ b/tasks/0013-idempotent-conversation-turns.md @@ -0,0 +1,58 @@ +# Task 0013 — Idempotent conversation turn preparation (WA-01) + +## Goal and context + +Implement the provider-independent preparation prerequisite in the approved +[WhatsApp plan](../docs/WHATSAPP_INTEGRATION_PLAN.md), following +[ADR 0006](../docs/adr/0006-idempotent-conversation-turn-preparation.md). + +## Read first + +`AGENTS.md`, `.ai/README.md`, project-architecture, architecture-review, +senior-python-engineering, testing, runtime-engine, documentation, and code-review +skills; the current manager service and both persistence adapters. + +## Scope / files allowed to change + +- This task, `tasks/README.md`, `docs/README.md`, the WhatsApp plan, and ADR 0006. +- New `src/agent_manager/domain/turn_preparation.py`. +- `src/agent_manager/application/conversation_service.py`. +- `src/agent_manager/infrastructure/persistence/{memory_repository,sql_repository,tables}.py`. +- New migration `0007_turn_preparation_receipts.py`. +- New `tests/agent_manager/test_turn_preparation.py` and, if needed, existing + manager database/repository/service tests. + +## Requirements / acceptance criteria + +- [x] An opt-in preparation key returns the original message/run/history. +- [x] Conflicting user, conversation, or content is rejected without mutation. +- [x] Receipt, append, head selection, and snapshot commit atomically. +- [x] Concurrent retries prepare one turn; branch conflicts retain no receipt. +- [x] Recovery reconciles missing run registration without resetting a run. +- [x] SQL restart preserves replay; normal callers and budget rules still work. +- [x] Keys are bounded/digested; unsupported capabilities and edits fail clearly. +- [x] Behavior-focused tests and final `make check` pass. + +## Out of scope + +Executing a recovered turn, webhook/provider APIs, channel queues, account +linking, changes to engine/YAML/HTTP contracts, and production deployment. + +## Commands / final report + +Run focused tests during development and `make check` before finishing. +Report using `AGENTS.md` section 9; identify remaining WhatsApp stages honestly. + +## Implementation result + +Completed WA-01 on 2026-10-15. The optional port is implemented by both memory +and SQL adapters; migration 0007 is required before SQL callers opt into keys. +The engine, YAML, and HTTP request shapes were not changed. + +Validation: `make check` passed (1,159 tests; 17 existing warnings), including +33 new preparation cases. `git diff --check` and documentation-link checks +also passed. No live Meta, LLM, MCP, or production database was accessed. + +WhatsApp transport, durable execution claims/outbox, and account linking are +subsequent tasks. The user confirmed an internal-information pilot, so trusted +organization identity resolution is required before channel activation. diff --git a/tests/agent_manager/test_turn_preparation.py b/tests/agent_manager/test_turn_preparation.py new file mode 100644 index 00000000..67bd0285 --- /dev/null +++ b/tests/agent_manager/test_turn_preparation.py @@ -0,0 +1,337 @@ +"""Retrying preparation recovers one immutable turn, never a second execution.""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from pathlib import Path +from typing import cast + +import pytest +from sqlmodel import SQLModel + +from agent_engine.approvals.models import RunRecord, RunStatus +from agent_engine.engine.types import ChatMessage, ChatRole, RunResult +from agent_engine.runs.in_memory import InMemoryRunRepository +from agent_engine.runs.repository import RunRepository +from agent_manager.application.conversation_service import ConversationService +from agent_manager.application.errors import ( + ConversationAccessDenied, + ConversationTokenBudgetExceeded, +) +from agent_manager.domain import ConversationMessage, Principal, Repository, Role +from agent_manager.domain.turn_preparation import ( + IdempotentTurnRepository, + TurnPreparationConflict, + preparation_key_digest, +) +from agent_manager.infrastructure.persistence.database import create_db_engine, session_factory +from agent_manager.infrastructure.persistence.memory_repository import MemoryRepository +from agent_manager.infrastructure.persistence.run_repository import SqlRunRepository +from agent_manager.infrastructure.persistence.sql_repository import SqlRepository +from tests.agent_manager.conftest import RecordingEngine + +ALICE = Principal.external("alice") +BOB = Principal.external("bob") + + +@pytest.fixture(params=["memory", "sql"]) +async def storage( + request: pytest.FixtureRequest, tmp_path: Path +) -> AsyncIterator[tuple[Repository, RunRepository]]: + if request.param == "memory": + yield MemoryRepository(), InMemoryRunRepository() + return + database = create_db_engine(f"sqlite+aiosqlite:///{tmp_path / 'turns.db'}") + async with database.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + sessions = session_factory(database) + try: + yield SqlRepository(sessions), SqlRunRepository(sessions) + finally: + await database.dispose() + + +def service_for( + storage: tuple[Repository, RunRepository], *, max_tokens: int | None = None +) -> ConversationService: + return ConversationService( + RecordingEngine(), + storage[0], + run_repository=storage[1], + system_name="test", + max_tokens=max_tokens, + ) + + +async def test_retry_recovers_original_ids_and_parent_history_after_head_moves( + storage: tuple[Repository, RunRepository], +) -> None: + service = service_for(storage) + conversation = await service.create(ALICE) + await service.send(conversation, "earlier", ALICE) + first = await service.prepare_turn(conversation, "שלום 👋", ALICE, idempotency_key="event-1") + await service.complete_turn(first, RunResult("test", [], "תשובה")) + await service.send(conversation, "later", ALICE) + + recovered = await service.prepare_turn( + conversation, "שלום 👋", ALICE, idempotency_key="event-1" + ) + + assert (recovered.message_id, recovered.run_id) == (first.message_id, first.run_id) + assert recovered.history == ( + ChatMessage(ChatRole.USER, "earlier"), + ChatMessage(ChatRole.ASSISTANT, "answer:earlier"), + ) + assert [m.content for m in await service.history(conversation, ALICE)] == [ + "earlier", + "answer:earlier", + "שלום 👋", + "תשובה", + "later", + "answer:later", + ] + + +@pytest.mark.parametrize("change", ["content", "conversation", "owner"]) +async def test_key_cannot_be_rebound_to_different_input( + storage: tuple[Repository, RunRepository], change: str +) -> None: + service = service_for(storage) + conversation = await service.create(ALICE) + await service.prepare_turn(conversation, "original", ALICE, idempotency_key="one-key") + principal = BOB if change == "owner" else ALICE + target = await service.create(principal) if change != "content" else conversation + + with pytest.raises(TurnPreparationConflict, match=r"[Ii]dempotency"): + await service.prepare_turn( + target, + "changed" if change == "content" else "original", + principal, + idempotency_key="one-key", + ) + + assert [m.content for m in await service.history(conversation, ALICE)] == ["original"] + if target != conversation: + assert await service.history(target, principal) == [] + + +async def test_receipt_does_not_bypass_conversation_ownership( + storage: tuple[Repository, RunRepository], +) -> None: + service = service_for(storage) + conversation = await service.create(ALICE) + await service.prepare_turn(conversation, "private", ALICE, idempotency_key="one-key") + with pytest.raises(ConversationAccessDenied): + await service.prepare_turn(conversation, "private", BOB, idempotency_key="one-key") + + +async def test_concurrent_retries_prepare_exactly_one_message_and_run( + storage: tuple[Repository, RunRepository], +) -> None: + service = service_for(storage) + conversation = await service.create(ALICE) + turns = await asyncio.gather( + *[ + service.prepare_turn(conversation, "one", ALICE, idempotency_key="concurrent") + for _ in range(8) + ] + ) + assert len({turn.message_id for turn in turns}) == 1 + assert len({turn.run_id for turn in turns}) == 1 + assert [m.content for m in await service.history(conversation, ALICE)] == ["one"] + assert all(turn.history == () for turn in turns) + + +async def test_receipt_replay_preserves_terminal_run_and_does_not_spend_new_budget( + storage: tuple[Repository, RunRepository], +) -> None: + service = service_for(storage, max_tokens=10) + conversation = await service.create(ALICE) + first = await service.prepare_turn(conversation, "one", ALICE, idempotency_key="event") + await storage[1].transition_if_allowed(first.run_id, RunStatus.COMPLETED) + await storage[0].record_run_token_usage(conversation, first.run_id, 10) + + replay = await service.prepare_turn(conversation, "one", ALICE, idempotency_key="event") + assert replay.run_id == first.run_id + run = await storage[1].get(first.run_id) + assert run is not None and run.status == RunStatus.COMPLETED + with pytest.raises(ConversationTokenBudgetExceeded): + await service.prepare_turn(conversation, "two", ALICE, idempotency_key="different") + assert [m.content for m in await service.history(conversation, ALICE)] == ["one"] + + +@pytest.mark.parametrize("key", ["", " ", "x" * 513, "ש" * 257]) +async def test_invalid_key_writes_nothing( + storage: tuple[Repository, RunRepository], key: str +) -> None: + service = service_for(storage) + conversation = await service.create(ALICE) + with pytest.raises(ValueError, match=r"[Ii]dempotency"): + await service.prepare_turn(conversation, "one", ALICE, idempotency_key=key) + assert await service.history(conversation, ALICE) == [] + + +async def test_edits_and_idempotency_cannot_be_combined( + storage: tuple[Repository, RunRepository], +) -> None: + service = service_for(storage) + conversation = await service.create(ALICE) + turn = await service.prepare_turn(conversation, "one", ALICE) + with pytest.raises(ValueError, match="edit"): + await service.prepare_turn( + conversation, + "replacement", + ALICE, + idempotency_key="edit", + edit_message_id=turn.message_id, + ) + assert [m.content for m in await service.history(conversation, ALICE)] == ["one"] + + +class UnavailableRunRepository(InMemoryRunRepository): + async def create_if_absent(self, record: RunRecord) -> bool: + raise OSError("registration unavailable") + + +async def test_registration_failure_leaves_a_recoverable_prepared_message() -> None: + repository = MemoryRepository() + service = service_for((repository, UnavailableRunRepository())) + conversation = await service.create(ALICE) + with pytest.raises(OSError, match="registration unavailable"): + await service.prepare_turn(conversation, "one", ALICE, idempotency_key="event") + original = (await repository.list_conversation_messages(conversation))[0] + + restarted = service_for((repository, InMemoryRunRepository())) + recovered = await restarted.prepare_turn(conversation, "one", ALICE, idempotency_key="event") + assert (recovered.message_id, recovered.run_id) == (original.message_id, original.run_id) + assert len(await restarted.history(conversation, ALICE)) == 1 + + +async def test_sql_receipt_survives_a_new_process_storage_handle(tmp_path: Path) -> None: + url = f"sqlite+aiosqlite:///{tmp_path / 'restart.db'}" + database = create_db_engine(url) + async with database.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + sessions = session_factory(database) + first_service = service_for((SqlRepository(sessions), SqlRunRepository(sessions))) + conversation = await first_service.create(ALICE) + original = await first_service.prepare_turn(conversation, "one", ALICE, idempotency_key="event") + await database.dispose() + + reopened = create_db_engine(url) + try: + sessions = session_factory(reopened) + restarted = service_for((SqlRepository(sessions), SqlRunRepository(sessions))) + recovered = await restarted.prepare_turn( + conversation, "one", ALICE, idempotency_key="event" + ) + assert (recovered.message_id, recovered.run_id) == (original.message_id, original.run_id) + assert [m.role for m in await restarted.history(conversation, ALICE)] == [Role.USER] + finally: + await reopened.dispose() + + +async def test_stale_head_rolls_back_receipt_so_the_key_can_be_prepared_again( + storage: tuple[Repository, RunRepository], +) -> None: + from datetime import UTC, datetime + + repository = storage[0] + assert isinstance(repository, IdempotentTurnRepository) + service = service_for(storage) + conversation = await service.create(ALICE) + first = await service.prepare_turn(conversation, "first", ALICE) + key = preparation_key_digest("retry-after-conflict") + stale = ConversationMessage( + "stale", + conversation, + Role.USER, + "second", + datetime.now(UTC), + run_id="stale-run", + user_id=ALICE.user_id, + ) + + assert await repository.append_prepared_message(key, stale, None) is None + assert await repository.get_prepared_message(key) is None + recovered = await service.prepare_turn( + conversation, "second", ALICE, idempotency_key="retry-after-conflict" + ) + assert recovered.message_id != "stale" + assert recovered.history == (ChatMessage(ChatRole.USER, "first"),) + messages = await service.history(conversation, ALICE) + assert [m.message_id for m in messages] == [first.message_id, recovered.message_id] + + +async def test_snapshot_failure_rolls_back_message_head_and_receipt( + storage: tuple[Repository, RunRepository], + monkeypatch: pytest.MonkeyPatch, +) -> None: + repository = storage[0] + service = service_for(storage) + conversation = await service.create(ALICE) + + async def fail_snapshot(*args: object, **kwargs: object) -> None: + raise OSError("snapshot unavailable") + + def fail_memory_snapshot(*args: object, **kwargs: object) -> None: + raise OSError("snapshot unavailable") + + if isinstance(repository, SqlRepository): + original_rebuild = repository._rebuild_snapshot_in_session + monkeypatch.setattr(repository, "_rebuild_snapshot_in_session", fail_snapshot) + else: + assert isinstance(repository, MemoryRepository) + original_memory_rebuild = repository._build_snapshot + monkeypatch.setattr(repository, "_build_snapshot", fail_memory_snapshot) + with pytest.raises(OSError, match="snapshot unavailable"): + await service.prepare_turn(conversation, "one", ALICE, idempotency_key="event") + + assert isinstance(repository, IdempotentTurnRepository) + assert await repository.get_prepared_message(preparation_key_digest("event")) is None + assert await repository.list_conversation_messages(conversation) == [] + session = await repository.get_session(conversation) + assert session is not None and session.head_message_id is None + if isinstance(repository, SqlRepository): + monkeypatch.setattr(repository, "_rebuild_snapshot_in_session", original_rebuild) + else: + monkeypatch.setattr(repository, "_build_snapshot", original_memory_rebuild) + prepared = await service.prepare_turn(conversation, "one", ALICE, idempotency_key="event") + assert prepared.history == () + + +async def test_missing_optional_capability_fails_before_writing() -> None: + class LegacyRepository: + """A third-party adapter implementing only the original repository port.""" + + def __init__(self) -> None: + self.delegate = MemoryRepository() + + def __getattr__(self, name: str) -> object: + if name in {"get_prepared_message", "append_prepared_message"}: + raise AttributeError(name) + return getattr(self.delegate, name) + + service = service_for((cast(Repository, LegacyRepository()), InMemoryRunRepository())) + conversation = await service.create(ALICE) + + with pytest.raises(RuntimeError, match="does not support idempotent"): + await service.prepare_turn(conversation, "one", ALICE, idempotency_key="event") + assert await service.history(conversation, ALICE) == [] + await service.send(conversation, "normal", ALICE) + assert [m.content for m in await service.history(conversation, ALICE)] == [ + "normal", + "answer:normal", + ] + + +async def test_different_keys_preserve_distinct_identical_messages( + storage: tuple[Repository, RunRepository], +) -> None: + service = service_for(storage) + conversation = await service.create(ALICE) + first = await service.prepare_turn(conversation, "same", ALICE, idempotency_key="one") + second = await service.prepare_turn(conversation, "same", ALICE, idempotency_key="two") + assert first.run_id != second.run_id + assert [m.content for m in await service.history(conversation, ALICE)] == ["same", "same"] From b2c467d8947b9d62d184ba5726a78232ed5a9b9e Mon Sep 17 00:00:00 2001 From: Asaf Varon Date: Fri, 16 Oct 2026 15:00:47 +0300 Subject: [PATCH 3/7] feat(manager): add durable channel storage Provider-independent inbox, sender bindings, execution claims, webhook receipts, ordered outbox and delivery events, worker ownership, and the channel identity port, with migrations 0008-0010. Co-Authored-By: Claude Opus 5.5 --- src/agent_manager/domain/channels/__init__.py | 1 + src/agent_manager/domain/channels/controls.py | 15 + src/agent_manager/domain/channels/delivery.py | 81 ++++ src/agent_manager/domain/channels/identity.py | 28 ++ src/agent_manager/domain/channels/models.py | 123 ++++++ .../domain/channels/ownership.py | 21 + .../domain/channels/repository.py | 60 +++ src/agent_manager/domain/channels/webhooks.py | 68 +++ .../persistence/channels/__init__.py | 1 + .../persistence/channels/controls.py | 117 ++++++ .../persistence/channels/delivery.py | 341 ++++++++++++++++ .../channels/delivery_projection.py | 90 ++++ .../persistence/channels/intake.py | 117 ++++++ .../persistence/channels/ownership.py | 127 ++++++ .../persistence/channels/replies.py | 125 ++++++ .../persistence/channels/repository.py | 386 ++++++++++++++++++ .../persistence/channels/tables.py | 134 ++++++ .../persistence/channels/webhooks.py | 128 ++++++ .../versions/0008_channel_storage.py | 74 ++++ .../versions/0009_webhook_receipts.py | 51 +++ .../versions/0010_channel_delivery.py | 93 +++++ .../infrastructure/persistence/tables.py | 9 + tests/agent_manager/test_channel_migration.py | 116 ++++++ .../test_channel_worker_ownership.py | 158 +++++++ .../agent_manager/test_delivery_migration.py | 93 +++++ tests/agent_manager/test_webhook_migration.py | 114 ++++++ tests/agent_manager/test_webhook_storage.py | 155 +++++++ 27 files changed, 2826 insertions(+) create mode 100644 src/agent_manager/domain/channels/__init__.py create mode 100644 src/agent_manager/domain/channels/controls.py create mode 100644 src/agent_manager/domain/channels/delivery.py create mode 100644 src/agent_manager/domain/channels/identity.py create mode 100644 src/agent_manager/domain/channels/models.py create mode 100644 src/agent_manager/domain/channels/ownership.py create mode 100644 src/agent_manager/domain/channels/repository.py create mode 100644 src/agent_manager/domain/channels/webhooks.py create mode 100644 src/agent_manager/infrastructure/persistence/channels/__init__.py create mode 100644 src/agent_manager/infrastructure/persistence/channels/controls.py create mode 100644 src/agent_manager/infrastructure/persistence/channels/delivery.py create mode 100644 src/agent_manager/infrastructure/persistence/channels/delivery_projection.py create mode 100644 src/agent_manager/infrastructure/persistence/channels/intake.py create mode 100644 src/agent_manager/infrastructure/persistence/channels/ownership.py create mode 100644 src/agent_manager/infrastructure/persistence/channels/replies.py create mode 100644 src/agent_manager/infrastructure/persistence/channels/repository.py create mode 100644 src/agent_manager/infrastructure/persistence/channels/tables.py create mode 100644 src/agent_manager/infrastructure/persistence/channels/webhooks.py create mode 100644 src/agent_manager/infrastructure/persistence/migrations/versions/0008_channel_storage.py create mode 100644 src/agent_manager/infrastructure/persistence/migrations/versions/0009_webhook_receipts.py create mode 100644 src/agent_manager/infrastructure/persistence/migrations/versions/0010_channel_delivery.py create mode 100644 tests/agent_manager/test_channel_migration.py create mode 100644 tests/agent_manager/test_channel_worker_ownership.py create mode 100644 tests/agent_manager/test_delivery_migration.py create mode 100644 tests/agent_manager/test_webhook_migration.py create mode 100644 tests/agent_manager/test_webhook_storage.py diff --git a/src/agent_manager/domain/channels/__init__.py b/src/agent_manager/domain/channels/__init__.py new file mode 100644 index 00000000..da9d241f --- /dev/null +++ b/src/agent_manager/domain/channels/__init__.py @@ -0,0 +1 @@ +"""Provider-independent channel work and identity ports.""" diff --git a/src/agent_manager/domain/channels/controls.py b/src/agent_manager/domain/channels/controls.py new file mode 100644 index 00000000..1595e976 --- /dev/null +++ b/src/agent_manager/domain/channels/controls.py @@ -0,0 +1,15 @@ +"""Durable control operations, with no model or organization data access.""" + +from abc import ABC, abstractmethod + +from agent_manager.domain.channels.models import InboxItem + + +class ControlRepository(ABC): + @abstractmethod + async def suppressed(self, item: InboxItem) -> bool: ... + + @abstractmethod + async def complete( + self, item: InboxItem, reply: str | None, *, reset: bool = False + ) -> None: ... diff --git a/src/agent_manager/domain/channels/delivery.py b/src/agent_manager/domain/channels/delivery.py new file mode 100644 index 00000000..c02b6117 --- /dev/null +++ b/src/agent_manager/domain/channels/delivery.py @@ -0,0 +1,81 @@ +"""Provider-neutral delivery evidence; uncertain submission is never a retry.""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from datetime import datetime +from typing import Literal + +from agent_manager.domain.channels.models import ChannelAddress + +SendKind = Literal["accepted", "retry", "failed", "unknown"] + + +@dataclass(frozen=True) +class SendOutcome: + kind: SendKind + provider_message_id: str | None = None + retry_after: float = 0 + + +class ChannelSender(ABC): + @abstractmethod + async def send(self, recipient: str, text: str) -> SendOutcome: ... + + +@dataclass(frozen=True) +class ReplySource: + event_id: str + texts: tuple[str, ...] = field(repr=False) + + +@dataclass(frozen=True) +class DeliveryPart: + delivery_id: str + event_id: str + address: ChannelAddress + user_id: str | None + organization_id: str | None + part_index: int + text: str = field(repr=False) + state: str + attempts: int + token: str | None = field(default=None, repr=False) + provider_message_id: str | None = None + next_attempt_at: datetime | None = None + reason: str | None = None + + +class DeliveryRepository(ABC): + @abstractmethod + async def sources(self, connection_id: str, system_id: str) -> tuple[ReplySource, ...]: ... + + @abstractmethod + async def materialize(self, event_id: str, parts: tuple[str, ...]) -> None: ... + + @abstractmethod + async def recover_sending(self, connection_id: str, system_id: str) -> None: ... + + @abstractmethod + async def claim_next( + self, connection_id: str, system_id: str, now: datetime + ) -> DeliveryPart | None: ... + + @abstractmethod + async def guard(self, part: DeliveryPart, now: datetime) -> str | None: ... + + @abstractmethod + async def complete( + self, + part: DeliveryPart, + state: str, + *, + provider_id: str | None = None, + next_attempt_at: datetime | None = None, + reason: str | None = None, + ) -> None: ... + + @abstractmethod + async def project(self, connection_id: str, system_id: str) -> None: ... + + @abstractmethod + async def parts(self, event_id: str) -> tuple[DeliveryPart, ...]: ... diff --git a/src/agent_manager/domain/channels/identity.py b/src/agent_manager/domain/channels/identity.py new file mode 100644 index 00000000..ef0a4c7c --- /dev/null +++ b/src/agent_manager/domain/channels/identity.py @@ -0,0 +1,28 @@ +"""Client-owned identity lookup; transport authentication grants no privileges.""" + +from abc import ABC, abstractmethod + +from agent_manager.domain.channels.models import ChannelAddress +from agent_manager.domain.identity import Principal + + +class ChannelIdentityDenied(PermissionError): + """No authorized organization identity is linked to this sender.""" + + +class ChannelIdentityUnavailable(RuntimeError): + """Identity could not be verified; no privileged work may proceed.""" + + +class ChannelIdentityResolver(ABC): + async def close(self) -> None: # noqa: B027 - optional, backwards-compatible cleanup + """Release any client-owned lookup resources at application shutdown.""" + + @abstractmethod + async def resolve(self, address: ChannelAddress) -> Principal | None: + """Resolve a current, revocable account link and its current permissions. + + Implemented by the client integration, never from message text. Return + None for an unlinked/revoked sender. Credentials are request-scoped and + must not be logged, cached on the resolver, or persisted in queue rows. + """ diff --git a/src/agent_manager/domain/channels/models.py b/src/agent_manager/domain/channels/models.py new file mode 100644 index 00000000..a9dbc92f --- /dev/null +++ b/src/agent_manager/domain/channels/models.py @@ -0,0 +1,123 @@ +"""Bounded normalized messages and durable work state, never provider payloads.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from datetime import UTC, datetime +from enum import StrEnum +from typing import Literal + +MessageKind = Literal["text", "reset", "unsupported", "help", "stop", "start"] + + +class ChannelMessageConflict(ValueError): + """A stable message or reply key was reused for different input.""" + + +class ChannelClaimConflict(RuntimeError): + """The caller does not own this work in the required state.""" + + +class ChannelOwnerConflict(ValueError): + """A sender binding cannot implicitly change organization, user, or chat.""" + + +def bounded_identifier(value: str, *, limit: int = 512) -> None: + if "\x00" in value: + raise ValueError("channel identifiers must not contain NUL") + if not value.strip() or len(value.encode("utf-8")) > limit: + raise ValueError(f"channel identifier must contain 1 to {limit} UTF-8 bytes") + + +def bounded_text(value: str) -> None: + if "\x00" in value: + raise ValueError("channel text must not contain NUL") + if len(value.encode("utf-8")) > 65_536: + raise ValueError("normalized channel text exceeds 65536 UTF-8 bytes") + + +def stable_id(*parts: str) -> str: + encoded = json.dumps(parts, ensure_ascii=True, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +@dataclass(frozen=True) +class ChannelAddress: + connection_id: str + system_id: str + sender_id: str = field(repr=False) + + def __post_init__(self) -> None: + bounded_identifier(self.connection_id, limit=128) + bounded_identifier(self.system_id, limit=256) + bounded_identifier(self.sender_id) + + +@dataclass(frozen=True) +class InboundMessage: + address: ChannelAddress + provider_message_id: str + content: str = field(repr=False) + received_at: datetime + sent_at: datetime | None + kind: MessageKind = "text" + + def __post_init__(self) -> None: + bounded_identifier(self.provider_message_id) + bounded_text(self.content) + if self.kind not in {"text", "reset", "unsupported", "help", "stop", "start"}: + raise ValueError("unsupported normalized channel message kind") + if self.kind in {"text", "reset"} and not self.content.strip(): + raise ValueError("channel text must not be empty") + for name in ("received_at", "sent_at"): + value = getattr(self, name) + if value is not None: + if value.utcoffset() is None: + raise ValueError("channel timestamps must include a timezone") + object.__setattr__(self, name, value.astimezone(UTC)) + + +class InboxState(StrEnum): + QUEUED = "queued" + PREPARING = "preparing" + PREPARED = "prepared" + RUNNING = "running" + ANSWERED = "answered" + DONE = "done" + DENIED = "denied" + FAILED = "failed" + NEEDS_REVIEW = "needs_review" + + +@dataclass(frozen=True) +class InboxItem: + event_id: str + sequence: int + message: InboundMessage + state: InboxState + claim_token: str | None = field(default=None, repr=False) + conversation_id: str | None = None + run_id: str | None = None + message_id: str | None = None + + +@dataclass(frozen=True) +class ChannelBinding: + address: ChannelAddress + generation: int + user_id: str | None = None + organization_id: str | None = None + conversation_id: str | None = None + active_event_id: str | None = None + + +@dataclass(frozen=True) +class StagedReply: + reply_id: str + event_id: str + response_id: str + part_index: int + text: str = field(repr=False) + created_at: datetime diff --git a/src/agent_manager/domain/channels/ownership.py b/src/agent_manager/domain/channels/ownership.py new file mode 100644 index 00000000..a9e35882 --- /dev/null +++ b/src/agent_manager/domain/channels/ownership.py @@ -0,0 +1,21 @@ +"""Exclusive supervisor lifetime, separate from durable per-event execution claims.""" + +from abc import ABC, abstractmethod +from contextlib import AbstractAsyncContextManager + + +class ChannelWorkerAlreadyRunning(RuntimeError): + """Another supervisor owns the configured connection.""" + + +class ChannelWorkerOwnershipLost(RuntimeError): + """Claims and child work must stop because exclusive ownership is uncertain.""" + + +class ChannelWorkerOwnership(ABC): + @abstractmethod + def hold(self) -> AbstractAsyncContextManager[None]: ... + + @abstractmethod + async def check(self) -> None: + """Raise rather than silently reconnect/reacquire a lost ownership session.""" diff --git a/src/agent_manager/domain/channels/repository.py b/src/agent_manager/domain/channels/repository.py new file mode 100644 index 00000000..9d43f9b1 --- /dev/null +++ b/src/agent_manager/domain/channels/repository.py @@ -0,0 +1,60 @@ +"""Durable channel handoff and ownership, separate from agent execution.""" + +from abc import ABC, abstractmethod +from collections.abc import Sequence + +from agent_manager.domain.channels.models import ( + ChannelAddress, + ChannelBinding, + InboundMessage, + InboxItem, + InboxState, + StagedReply, +) + + +class ChannelRepository(ABC): + @abstractmethod + async def ingest(self, message: InboundMessage) -> InboxItem: ... + + @abstractmethod + async def get_inbound(self, event_id: str) -> InboxItem | None: ... + + @abstractmethod + async def get_binding(self, address: ChannelAddress) -> ChannelBinding | None: ... + + @abstractmethod + async def claim_next(self, connection_id: str, system_id: str) -> InboxItem | None: ... + + @abstractmethod + async def list_active( + self, connection_id: str, system_id: str, *, after_sequence: int = 0, limit: int = 32 + ) -> tuple[InboxItem, ...]: + """Bounded recovery scan; only the exclusive supervisor may schedule these.""" + + @abstractmethod + async def bind_conversation( + self, event_id: str, token: str, user_id: str, organization_id: str, conversation_id: str + ) -> ChannelBinding: ... + + @abstractmethod + async def record_prepared( + self, event_id: str, token: str, conversation_id: str, run_id: str, message_id: str + ) -> InboxItem: ... + + @abstractmethod + async def claim_execution(self, event_id: str, token: str) -> bool: ... + + @abstractmethod + async def finish(self, event_id: str, token: str, state: InboxState) -> None: ... + + @abstractmethod + async def reset_conversation(self, event_id: str, token: str) -> int: ... + + @abstractmethod + async def stage_reply( + self, event_id: str, token: str, response_id: str, parts: Sequence[str] + ) -> tuple[StagedReply, ...]: ... + + @abstractmethod + async def list_replies(self, event_id: str) -> tuple[StagedReply, ...]: ... diff --git a/src/agent_manager/domain/channels/webhooks.py b/src/agent_manager/domain/channels/webhooks.py new file mode 100644 index 00000000..8185f80f --- /dev/null +++ b/src/agent_manager/domain/channels/webhooks.py @@ -0,0 +1,68 @@ +"""Normalized authenticated batch receipts; provider payloads stay at the edge.""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from datetime import datetime +from typing import Literal + +from agent_manager.domain.channels.models import InboundMessage + +IssueReason = Literal[ + "invalid_envelope", + "invalid_entry", + "invalid_change", + "invalid_value", + "invalid_message", + "invalid_status", + "conflicting_message", + "issue_limit", +] + + +@dataclass(frozen=True) +class WebhookIssue: + location: str + reason: IssueReason + + +@dataclass(frozen=True) +class DeliveryEvent: + provider_message_id: str + recipient_id: str = field(repr=False) + status: str + observed_at: datetime + + +@dataclass(frozen=True) +class WebhookBatch: + messages: tuple[InboundMessage, ...] = () + delivery_events: tuple[DeliveryEvent, ...] = () + issues: tuple[WebhookIssue, ...] = () + + +@dataclass(frozen=True) +class WebhookReceipt: + receipt_id: str + connection_id: str + system_id: str + body_digest: str + received_at: datetime + event_ids: tuple[str, ...] + delivery_event_ids: tuple[str, ...] + issues: tuple[WebhookIssue, ...] + + +class WebhookRepository(ABC): + @abstractmethod + async def accept( + self, + connection_id: str, + system_id: str, + body_digest: str, + received_at: datetime, + batch: WebhookBatch, + ) -> WebhookReceipt: + """Commit the complete verified batch atomically or leave it unrecorded.""" + + @abstractmethod + async def get_receipt(self, receipt_id: str) -> WebhookReceipt | None: ... diff --git a/src/agent_manager/infrastructure/persistence/channels/__init__.py b/src/agent_manager/infrastructure/persistence/channels/__init__.py new file mode 100644 index 00000000..25c78fcf --- /dev/null +++ b/src/agent_manager/infrastructure/persistence/channels/__init__.py @@ -0,0 +1 @@ +"""SQL storage for normalized channel work, without provider dependencies.""" diff --git a/src/agent_manager/infrastructure/persistence/channels/controls.py b/src/agent_manager/infrastructure/persistence/channels/controls.py new file mode 100644 index 00000000..426889c1 --- /dev/null +++ b/src/agent_manager/infrastructure/persistence/channels/controls.py @@ -0,0 +1,117 @@ +"""Commit control completion, optional static reply and reset in one transaction.""" + +from datetime import UTC, datetime + +from sqlalchemy import update +from sqlalchemy.ext.asyncio import async_sessionmaker +from sqlmodel import col, select +from sqlmodel.ext.asyncio.session import AsyncSession + +from agent_manager.domain.channels.controls import ControlRepository +from agent_manager.domain.channels.models import ( + ChannelClaimConflict, + InboxItem, + bounded_text, + stable_id, +) +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelBindingRow, + ChannelInboxRow, + ChannelOutboxRow, +) + + +class SqlControlRepository(ControlRepository): + def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: + self._sessions = sessions + + async def suppressed(self, item: InboxItem) -> bool: + address = item.message.address + async with self._sessions() as session: + binding = await session.get( + ChannelBindingRow, (address.connection_id, address.system_id, address.sender_id) + ) + return ( + binding is None + or binding.opted_out + or item.sequence <= binding.suppressed_through_sequence + ) + + async def complete(self, item: InboxItem, reply: str | None, *, reset: bool = False) -> None: + if reply is not None: + bounded_text(reply) + if not reply.strip(): + raise ValueError("control reply must not be empty") + async with self._sessions() as session, session.begin(): + row = ( + await session.exec( + select(ChannelInboxRow) + .where(col(ChannelInboxRow.event_id) == item.event_id) + .with_for_update() + ) + ).first() + if row is None or not item.claim_token or row.claim_token != item.claim_token: + raise ChannelClaimConflict("control requires its durable claim") + if row.state in {"answered", "done"}: + return + if row.state != "preparing" or (reset and row.kind != "reset"): + raise ChannelClaimConflict("control is not preparing") + binding = await session.get( + ChannelBindingRow, (row.connection_id, row.system_id, row.sender_id) + ) + if binding is None or binding.active_event_id != item.event_id: + await session.refresh(row) + if row.state in {"answered", "done"}: + return + raise ChannelClaimConflict("control does not own its sender") + response_id = ( + stable_id("control-response", item.event_id) if reply is not None else None + ) + changed = await session.exec( + update(ChannelInboxRow) + .where( + col(ChannelInboxRow.event_id) == item.event_id, + col(ChannelInboxRow.claim_token) == item.claim_token, + col(ChannelInboxRow.state) == "preparing", + ) + .values( + state="answered" if reply is not None else "done", + response_id=response_id, + reset_generation=binding.generation + 1 if reset else None, + ) + .returning(col(ChannelInboxRow.event_id)) + ) + if changed.scalar_one_or_none() is None: + await session.refresh(row) + if row.state in {"answered", "done"}: + return + raise ChannelClaimConflict("control changed before completion") + values: dict[str, object] = {"active_event_id": None} + if reset: + values.update(generation=binding.generation + 1, conversation_id=None) + released = await session.exec( + update(ChannelBindingRow) + .where( + col(ChannelBindingRow.connection_id) == row.connection_id, + col(ChannelBindingRow.system_id) == row.system_id, + col(ChannelBindingRow.sender_id) == row.sender_id, + col(ChannelBindingRow.active_event_id) == item.event_id, + col(ChannelBindingRow.generation) == binding.generation, + ) + .values(**values) + .returning(col(ChannelBindingRow.connection_id)) + ) + if released.scalar_one_or_none() is None: + raise ChannelClaimConflict("control lost its sender generation") + if reply is not None: + assert response_id is not None + session.add( + ChannelOutboxRow( + reply_id=stable_id("reply", item.event_id, response_id, "0"), + event_id=item.event_id, + response_id=response_id, + part_index=0, + text=reply, + created_at=datetime.now(UTC), + ) + ) diff --git a/src/agent_manager/infrastructure/persistence/channels/delivery.py b/src/agent_manager/infrastructure/persistence/channels/delivery.py new file mode 100644 index 00000000..b733703e --- /dev/null +++ b/src/agent_manager/infrastructure/persistence/channels/delivery.py @@ -0,0 +1,341 @@ +"""Durable provider rendering and CAS attempt claims, separate from answers.""" + +import secrets +from datetime import datetime, timedelta + +from sqlalchemy import and_, exists, or_, update +from sqlalchemy.ext.asyncio import async_sessionmaker +from sqlalchemy.orm import aliased +from sqlmodel import col, select +from sqlmodel.ext.asyncio.session import AsyncSession + +from agent_manager.domain.channels.delivery import DeliveryPart, DeliveryRepository, ReplySource +from agent_manager.domain.channels.models import ( + ChannelAddress, + ChannelClaimConflict, + ChannelMessageConflict, + bounded_identifier, + stable_id, +) +from agent_manager.domain.pagination import ensure_utc +from agent_manager.infrastructure.persistence.channels.intake import sql_insert +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelBindingRow, + ChannelInboxRow, + ChannelOutboxRow, +) +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelDeliveryBatchRow as Batch, +) +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelDeliveryPartRow as Part, +) +from agent_manager.infrastructure.persistence.tables import ConversationSessionRow + +ADVANCED = ("accepted", "delivered", "read", "suppressed") +POSITIVE = ("accepted", "delivered", "read") + + +class SqlDeliveryRepository(DeliveryRepository): + def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: + self._sessions = sessions + + async def sources(self, connection_id: str, system_id: str) -> tuple[ReplySource, ...]: + async with self._sessions() as session: + rows = ( + await session.exec( + select(ChannelInboxRow) + .where( + col(ChannelInboxRow.connection_id) == connection_id, + col(ChannelInboxRow.system_id) == system_id, + col(ChannelInboxRow.state) == "answered", + ~exists().where(col(Batch.event_id) == col(ChannelInboxRow.event_id)), + ) + .order_by(col(ChannelInboxRow.sequence)) + .limit(32) + ) + ).all() + result = [] + for row in rows: + texts = ( + await session.exec( + select(ChannelOutboxRow) + .where(col(ChannelOutboxRow.event_id) == row.event_id) + .order_by(col(ChannelOutboxRow.part_index)) + .limit(128) + ) + ).all() + if texts: + result.append(ReplySource(row.event_id, tuple(part.text for part in texts))) + return tuple(result) + + async def materialize(self, event_id: str, parts: tuple[str, ...]) -> None: + if len(parts) > 128 or any( + not text or len(text.encode("utf-16-le")) // 2 > 4096 or "\x00" in text + for text in parts + ): + raise ValueError("invalid provider part plan") + async with self._sessions() as session, session.begin(): + inbox = ( + await session.exec( + select(ChannelInboxRow).where(col(ChannelInboxRow.event_id) == event_id) + ) + ).first() + if inbox is None or inbox.state != "answered": + raise ChannelClaimConflict("delivery source is not answered") + binding = await session.get( + ChannelBindingRow, (inbox.connection_id, inbox.system_id, inbox.sender_id) + ) + owner = ( + await session.get(ConversationSessionRow, inbox.conversation_id) + if inbox.conversation_id + else None + ) + state = "ready" if parts else "blocked_format" + if inbox.kind not in {"reset", "help", "start", "unsupported"} and ( + owner is None + or not owner.user_id + or binding is None + or binding.user_id != owner.user_id + or not binding.organization_id + ): + state = "blocked_identity" + inserted = await session.exec( + sql_insert(session, Batch) + .values( + event_id=event_id, + connection_id=inbox.connection_id, + system_id=inbox.system_id, + sender_id=inbox.sender_id, + sequence=inbox.sequence, + user_id=owner.user_id if owner else None, + organization_id=binding.organization_id if binding else None, + state=state, + reason=None if state == "ready" else state, + ) + .on_conflict_do_nothing(index_elements=["event_id"]) + .returning(col(Batch.event_id)) + ) + if inserted.scalar_one_or_none() is None: + existing = ( + await session.exec( + select(Part) + .where(col(Part.event_id) == event_id) + .order_by(col(Part.part_index)) + ) + ).all() + if tuple(row.text for row in existing) != parts: + raise ChannelMessageConflict("provider part plan is immutable") + return + for index, text in enumerate(parts): + session.add( + Part( + delivery_id=stable_id("delivery-part", event_id, str(index)), + event_id=event_id, + connection_id=inbox.connection_id, + part_index=index, + text=text, + ) + ) + + async def recover_sending(self, connection_id: str, system_id: str) -> None: + async with self._sessions() as session, session.begin(): + ids = select(Batch.event_id).where( + col(Batch.connection_id) == connection_id, + col(Batch.system_id) == system_id, + col(Batch.state) == "sending", + ) + await session.exec( + update(Part) + .where(col(Part.event_id).in_(ids), col(Part.state) == "sending") + .values(state="delivery_unknown", reason="interrupted_send") + ) + await session.exec( + update(Batch) + .where(col(Batch.event_id).in_(ids)) + .values(state="delivery_unknown", reason="interrupted_send") + ) + + async def claim_next( + self, connection_id: str, system_id: str, now: datetime + ) -> DeliveryPart | None: + older = aliased(Batch) + previous = aliased(Part) + unplanned = aliased(ChannelInboxRow) + planned = aliased(Batch) + async with self._sessions() as session, session.begin(): + rows = ( + await session.exec( + select(Part, Batch) + .join(Batch, col(Batch.event_id) == col(Part.event_id)) + .where( + col(Batch.connection_id) == connection_id, + col(Batch.system_id) == system_id, + col(Batch.state).in_(("ready", "retry")), + col(Part.state).in_(("ready", "retry")), + or_(col(Part.next_attempt_at).is_(None), col(Part.next_attempt_at) <= now), + ~exists().where( + col(previous.event_id) == col(Part.event_id), + col(previous.part_index) < col(Part.part_index), + col(previous.state).not_in(POSITIVE), + ), + ~exists().where( + and_( + col(older.connection_id) == col(Batch.connection_id), + col(older.system_id) == col(Batch.system_id), + col(older.sender_id) == col(Batch.sender_id), + col(older.sequence) < col(Batch.sequence), + col(older.state).not_in(ADVANCED), + ) + ), + ~exists().where( + col(unplanned.connection_id) == col(Batch.connection_id), + col(unplanned.system_id) == col(Batch.system_id), + col(unplanned.sender_id) == col(Batch.sender_id), + col(unplanned.sequence) < col(Batch.sequence), + col(unplanned.state) == "answered", + ~exists() + .where(col(planned.event_id) == col(unplanned.event_id)) + .correlate(unplanned), + ), + ) + .order_by(col(Batch.sequence), col(Part.part_index)) + .limit(32) + ) + ).all() + for part, batch in rows: + token = secrets.token_hex(16) + changed = await session.exec( + update(Part) + .where( + col(Part.delivery_id) == part.delivery_id, + col(Part.state) == part.state, + col(Part.attempts) == part.attempts, + ) + .values(state="sending", attempts=part.attempts + 1, token=token) + .returning(Part) + ) + claimed = changed.scalar_one_or_none() + if claimed is not None: + batch.state = "sending" + session.add(batch) + return _part(claimed, batch) + return None + + async def guard(self, part: DeliveryPart, now: datetime) -> str | None: + async with self._sessions() as session: + batch = await session.get(Batch, part.event_id) + binding = await session.get( + ChannelBindingRow, + (part.address.connection_id, part.address.system_id, part.address.sender_id), + ) + if batch is None or binding is None: + return "blocked_identity" + if binding.opted_out or batch.sequence <= binding.suppressed_through_sequence: + return "suppressed" + if part.user_id is not None and ( + binding.user_id != part.user_id or binding.organization_id != part.organization_id + ): + return "blocked_identity" + latest = ensure_utc(binding.latest_inbound_at) + if latest is None or latest > now or now >= latest + timedelta(hours=24): + return "blocked_window" + return None + + async def complete( + self, + part: DeliveryPart, + state: str, + *, + provider_id: str | None = None, + next_attempt_at: datetime | None = None, + reason: str | None = None, + ) -> None: + if state not in { + "accepted", + "retry", + "failed", + "delivery_unknown", + "blocked_identity", + "blocked_window", + "suppressed", + }: + raise ValueError("invalid send completion state") + if provider_id is not None: + bounded_identifier(provider_id) + if state == "accepted" and not provider_id: + raise ValueError("acceptance requires a provider message ID") + async with self._sessions() as session, session.begin(): + changed = await session.exec( + update(Part) + .where( + col(Part.delivery_id) == part.delivery_id, + col(Part.state) == "sending", + col(Part.token) == part.token, + ) + .values( + state=state, + provider_message_id=provider_id, + next_attempt_at=next_attempt_at, + reason=reason, + ) + .returning(col(Part.delivery_id)) + ) + if changed.scalar_one_or_none() is None: + raise ChannelClaimConflict("send attempt is no longer owned") + batch = await session.get(Batch, part.event_id) + assert batch is not None + pending = ( + await session.exec( + select(Part.delivery_id).where( + col(Part.event_id) == part.event_id, col(Part.state).not_in(POSITIVE) + ) + ) + ).first() + batch.state = ("ready" if pending else "accepted") if state == "accepted" else state + batch.reason = reason + session.add(batch) + if batch.state == "accepted": + await session.exec( + update(ChannelInboxRow) + .where( + col(ChannelInboxRow.event_id) == part.event_id, + col(ChannelInboxRow.state) == "answered", + ) + .values(state="done") + ) + + async def project(self, connection_id: str, system_id: str) -> None: + from agent_manager.infrastructure.persistence.channels.delivery_projection import project + + await project(self._sessions, connection_id, system_id) + + async def parts(self, event_id: str) -> tuple[DeliveryPart, ...]: + async with self._sessions() as session: + rows = ( + await session.exec( + select(Part, Batch) + .join(Batch, col(Batch.event_id) == col(Part.event_id)) + .where(col(Part.event_id) == event_id) + .order_by(col(Part.part_index)) + ) + ).all() + return tuple(_part(part, batch) for part, batch in rows) + + +def _part(part: Part, batch: Batch) -> DeliveryPart: + return DeliveryPart( + part.delivery_id, + part.event_id, + ChannelAddress(batch.connection_id, batch.system_id, batch.sender_id), + batch.user_id, + batch.organization_id, + part.part_index, + part.text, + part.state, + part.attempts, + part.token, + part.provider_message_id, + ensure_utc(part.next_attempt_at), + part.reason, + ) diff --git a/src/agent_manager/infrastructure/persistence/channels/delivery_projection.py b/src/agent_manager/infrastructure/persistence/channels/delivery_projection.py new file mode 100644 index 00000000..2ca9575e --- /dev/null +++ b/src/agent_manager/infrastructure/persistence/channels/delivery_projection.py @@ -0,0 +1,90 @@ +"""Project authenticated observations by exact recipient and provider ID.""" + +from sqlalchemy.ext.asyncio import async_sessionmaker +from sqlmodel import col, select +from sqlmodel.ext.asyncio.session import AsyncSession + +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelDeliveryBatchRow as Batch, +) +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelDeliveryEventRow as Event, +) +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelDeliveryPartRow as Part, +) + + +async def project( + sessions: async_sessionmaker[AsyncSession], connection_id: str, system_id: str +) -> None: + async with sessions() as session, session.begin(): + rows = ( + await session.exec( + select(Event, Part, Batch) + .join(Part, col(Part.provider_message_id) == col(Event.provider_message_id)) + .join(Batch, col(Batch.event_id) == col(Part.event_id)) + .where( + col(Event.connection_id) == connection_id, + col(Event.system_id) == system_id, + col(Batch.connection_id) == col(Event.connection_id), + col(Batch.system_id) == col(Event.system_id), + col(Batch.sender_id) == col(Event.recipient_id), + col(Event.projected).is_(False), + ) + .order_by(col(Event.received_at), col(Event.event_id)) + .limit(100) + .with_for_update() + ) + ).all() + touched: set[str] = set() + for event, part, batch in rows: + if event.status == "read": + part.state = "read" + part.reason = None + elif event.status == "delivered" and part.state != "read": + part.state = "delivered" + part.reason = None + elif event.status == "failed" and part.state not in {"delivered", "read"}: + part.state = "failed" + part.reason = "provider_delivery_failed" + event.projected = True + session.add(event) + session.add(part) + touched.add(batch.event_id) + await session.flush() + for event_id in touched: + aggregate = await session.get(Batch, event_id) + assert aggregate is not None + parts = (await session.exec(select(Part).where(col(Part.event_id) == event_id))).all() + states = {part.state for part in parts} + aggregate.state = _batch_state(states) + aggregate.reason = ( + "provider_delivery_failed" + if aggregate.state == "failed" + else aggregate.state + if aggregate.state.startswith("blocked_") + or aggregate.state in {"delivery_unknown", "suppressed"} + else None + ) + session.add(aggregate) + + +def _batch_state(states: set[str]) -> str: + for blocking in ( + "delivery_unknown", + "blocked_identity", + "blocked_window", + "suppressed", + "failed", + "sending", + "retry", + "ready", + ): + if blocking in states: + return blocking + if states == {"read"}: + return "read" + if states <= {"delivered", "read"}: + return "delivered" + return "accepted" diff --git a/src/agent_manager/infrastructure/persistence/channels/intake.py b/src/agent_manager/infrastructure/persistence/channels/intake.py new file mode 100644 index 00000000..f46a95dc --- /dev/null +++ b/src/agent_manager/infrastructure/persistence/channels/intake.py @@ -0,0 +1,117 @@ +"""Shared inbox insert within a caller-owned transaction.""" + +from sqlalchemy import case, or_, update +from sqlalchemy.dialects.postgresql import Insert as PostgresInsert +from sqlalchemy.dialects.postgresql import insert as postgres_insert +from sqlalchemy.dialects.sqlite import Insert as SqliteInsert +from sqlalchemy.dialects.sqlite import insert as sqlite_insert +from sqlmodel import SQLModel, col, select +from sqlmodel.ext.asyncio.session import AsyncSession + +from agent_manager.domain.channels.models import ( + ChannelMessageConflict, + InboundMessage, + InboxState, + stable_id, +) +from agent_manager.domain.pagination import ensure_utc +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelBindingRow, + ChannelInboxRow, +) + + +def sql_insert(session: AsyncSession, model: type[SQLModel]) -> SqliteInsert | PostgresInsert: + if session.bind is not None and session.bind.dialect.name == "sqlite": + return sqlite_insert(model) + return postgres_insert(model) + + +async def ingest_in_session(session: AsyncSession, message: InboundMessage) -> ChannelInboxRow: + address = message.address + event_id = stable_id("inbox", address.connection_id, message.provider_message_id) + inserted = await session.exec( + sql_insert(session, ChannelInboxRow) + .values( + event_id=event_id, + connection_id=address.connection_id, + system_id=address.system_id, + sender_id=address.sender_id, + provider_message_id=message.provider_message_id, + content=message.content, + kind=message.kind, + received_at=message.received_at, + sent_at=message.sent_at, + state=InboxState.QUEUED.value, + ) + .on_conflict_do_nothing(index_elements=["connection_id", "provider_message_id"]) + .returning(col(ChannelInboxRow.event_id)) + ) + created = inserted.scalar_one_or_none() is not None + rows = await session.exec( + select(ChannelInboxRow).where(col(ChannelInboxRow.event_id) == event_id) + ) + row = rows.one() + if not created: + if ( + row.system_id != address.system_id + or row.sender_id != address.sender_id + or row.kind != message.kind + or row.content != message.content + or ensure_utc(row.sent_at) != message.sent_at + ): + raise ChannelMessageConflict("provider message is bound to different input") + return row + await session.exec( + sql_insert(session, ChannelBindingRow) + .values( + connection_id=address.connection_id, + system_id=address.system_id, + sender_id=address.sender_id, + generation=0, + ) + .on_conflict_do_nothing(index_elements=["connection_id", "system_id", "sender_id"]) + ) + await _inbound_policy(session, row, message) + return row + + +async def _inbound_policy( + session: AsyncSession, row: ChannelInboxRow, message: InboundMessage +) -> None: + binding = ChannelBindingRow + observed = min(message.sent_at, message.received_at) if message.sent_at else None + values: dict[str, object] = {} + if observed is not None: + values["latest_inbound_at"] = case( + ( + or_( + col(binding.latest_inbound_at).is_(None), + col(binding.latest_inbound_at) < observed, + ), + observed, + ), + else_=binding.latest_inbound_at, + ) + if message.kind in {"stop", "start"}: + preference = observed or message.received_at + fresh = or_(col(binding.preference_at).is_(None), col(binding.preference_at) <= preference) + cutoff = row.sequence if message.kind == "stop" else (row.sequence or 1) - 1 + values["suppressed_through_sequence"] = case( + (col(binding.suppressed_through_sequence) < cutoff, cutoff), + else_=binding.suppressed_through_sequence, + ) + values["opted_out"] = ( + True if message.kind == "stop" else case((fresh, False), else_=binding.opted_out) + ) + values["preference_at"] = case((fresh, preference), else_=binding.preference_at) + if values: + await session.exec( + update(binding) + .where( + col(binding.connection_id) == row.connection_id, + col(binding.system_id) == row.system_id, + col(binding.sender_id) == row.sender_id, + ) + .values(**values) + ) diff --git a/src/agent_manager/infrastructure/persistence/channels/ownership.py b/src/agent_manager/infrastructure/persistence/channels/ownership.py new file mode 100644 index 00000000..7f60f862 --- /dev/null +++ b/src/agent_manager/infrastructure/persistence/channels/ownership.py @@ -0,0 +1,127 @@ +"""Exclusive connection supervisor: local OS lock or dedicated PostgreSQL session.""" + +from __future__ import annotations + +import asyncio +import errno +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager, suppress +from pathlib import Path +from urllib.parse import unquote, urlsplit + +from sqlalchemy import text +from sqlalchemy.engine import make_url +from sqlalchemy.ext.asyncio import AsyncConnection, create_async_engine +from sqlalchemy.pool import NullPool + +from agent_manager.domain.channels.models import stable_id +from agent_manager.domain.channels.ownership import ( + ChannelWorkerAlreadyRunning, + ChannelWorkerOwnership, + ChannelWorkerOwnershipLost, +) + + +class SqlChannelWorkerOwnership(ChannelWorkerOwnership): + def __init__(self, database_url: str, connection_id: str) -> None: + self._url = database_url + self._scope = stable_id("channel-worker", connection_id) + self._entered = False + self._held = False + self._lost = False + self._connection: AsyncConnection | None = None + self._backend_pid: int | None = None + + @asynccontextmanager + async def hold(self) -> AsyncIterator[None]: + if self._entered: + raise ChannelWorkerAlreadyRunning("channel supervisor already entered") + self._entered = True + self._lost = False + try: + backend = make_url(self._url).get_backend_name() + if backend == "sqlite": + async with self._sqlite(): + yield + elif backend == "postgresql": + async with self._postgres(): + yield + else: + raise ValueError("unsupported channel ownership backend") + finally: + self._entered = False + self._held = False + + @asynccontextmanager + async def _sqlite(self) -> AsyncIterator[None]: + try: + import fcntl + except ImportError: + raise ValueError("SQLite channel ownership requires local POSIX storage") from None + database = make_url(self._url).database or "" + if database.startswith("file:"): + database = unquote(urlsplit(database).path) + path = Path(database).resolve() + lock_path = path.with_name(f".{path.name}.extra-worker-{self._scope[:16]}.lock") + with lock_path.open("a+b") as handle: + try: + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as exc: + if exc.errno not in (errno.EACCES, errno.EAGAIN): + raise + raise ChannelWorkerAlreadyRunning( + "another worker owns this channel connection" + ) from None + self._held = True + try: + yield + finally: + self._held = False + fcntl.flock(handle.fileno(), fcntl.LOCK_UN) + # Do not unlink: another process may already be waiting on this inode. + + @asynccontextmanager + async def _postgres(self) -> AsyncIterator[None]: + engine = create_async_engine(self._url, poolclass=NullPool, isolation_level="AUTOCOMMIT") + key = int.from_bytes(bytes.fromhex(self._scope)[:8], "big", signed=True) + try: + async with engine.connect() as connection: + acquired = ( + await connection.execute( + text("SELECT pg_try_advisory_lock(:key)"), {"key": key} + ) + ).scalar_one() + if not acquired: + raise ChannelWorkerAlreadyRunning("another worker owns this channel connection") + self._connection = connection + self._backend_pid = ( + await connection.execute(text("SELECT pg_backend_pid()")) + ).scalar_one() + self._held = True + try: + yield + finally: + self._held = False + self._connection = None + if not self._lost: + with suppress(Exception): + await connection.execute( + text("SELECT pg_advisory_unlock(:key)"), {"key": key} + ) + finally: + await engine.dispose() + + async def check(self) -> None: + if not self._held or self._lost: + raise ChannelWorkerOwnershipLost("channel supervisor ownership is unavailable") + if self._connection is not None: + try: + async with asyncio.timeout(5): + pid = ( + await self._connection.execute(text("SELECT pg_backend_pid()")) + ).scalar_one() + if pid != self._backend_pid: + raise ChannelWorkerOwnershipLost("channel ownership session was replaced") + except Exception: + self._lost = True + raise ChannelWorkerOwnershipLost("channel supervisor ownership was lost") from None diff --git a/src/agent_manager/infrastructure/persistence/channels/replies.py b/src/agent_manager/infrastructure/persistence/channels/replies.py new file mode 100644 index 00000000..b14fe0a6 --- /dev/null +++ b/src/agent_manager/infrastructure/persistence/channels/replies.py @@ -0,0 +1,125 @@ +"""Atomically stage one immutable reply; provider sending is a separate concern.""" + +from __future__ import annotations + +from collections.abc import Sequence +from datetime import UTC, datetime + +from sqlalchemy import and_, update +from sqlalchemy.ext.asyncio import async_sessionmaker +from sqlmodel import col, select +from sqlmodel.ext.asyncio.session import AsyncSession + +from agent_manager.domain.channels.models import ( + ChannelClaimConflict, + ChannelMessageConflict, + InboxState, + StagedReply, + bounded_identifier, + bounded_text, + stable_id, +) +from agent_manager.domain.pagination import ensure_utc +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelBindingRow, + ChannelInboxRow, + ChannelOutboxRow, +) + + +async def stage_reply( + sessions: async_sessionmaker[AsyncSession], + event_id: str, + token: str, + response_id: str, + parts: Sequence[str], +) -> tuple[StagedReply, ...]: + bounded_identifier(response_id, limit=64) + parts = tuple(parts) + if not 1 <= len(parts) <= 128: + raise ValueError("reply requires 1 to 128 parts") + for text in parts: + bounded_text(text) + if not text.strip(): + raise ValueError("reply parts must not be empty") + async with sessions() as session, session.begin(): + rows = await session.exec( + select(ChannelInboxRow) + .where(col(ChannelInboxRow.event_id) == event_id) + .with_for_update() + ) + inbox = rows.first() + if inbox is None or not token or inbox.claim_token != token: + raise ChannelClaimConflict("invalid reply work claim") + changed = await session.exec( + update(ChannelInboxRow) + .where( + col(ChannelInboxRow.event_id) == event_id, + col(ChannelInboxRow.state) == InboxState.RUNNING.value, + col(ChannelInboxRow.claim_token) == token, + ) + .values(state=InboxState.ANSWERED.value, response_id=response_id) + .returning(col(ChannelInboxRow.event_id)) + ) + if changed.scalar_one_or_none() is None: + await session.refresh(inbox) + if inbox.state not in {InboxState.ANSWERED, InboxState.DONE}: + raise ChannelClaimConflict("reply requires running or answered work") + existing = await _list(session, event_id) + if inbox.response_id != response_id or tuple(part.text for part in existing) != parts: + raise ChannelMessageConflict("staged reply cannot be replaced") + return existing + now = datetime.now(UTC) + for index, text in enumerate(parts): + session.add( + ChannelOutboxRow( + reply_id=stable_id("reply", event_id, response_id, str(index)), + event_id=event_id, + response_id=response_id, + part_index=index, + text=text, + created_at=now, + ) + ) + await session.flush() + await session.exec( + update(ChannelBindingRow) + .where( + and_( + col(ChannelBindingRow.connection_id) == inbox.connection_id, + col(ChannelBindingRow.system_id) == inbox.system_id, + col(ChannelBindingRow.sender_id) == inbox.sender_id, + col(ChannelBindingRow.active_event_id) == event_id, + ) + ) + .values(active_event_id=None) + ) + return await _list(session, event_id) + + +async def list_replies( + sessions: async_sessionmaker[AsyncSession], + event_id: str, +) -> tuple[StagedReply, ...]: + async with sessions() as session: + return await _list(session, event_id) + + +async def _list(session: AsyncSession, event_id: str) -> tuple[StagedReply, ...]: + rows = await session.exec( + select(ChannelOutboxRow) + .where(col(ChannelOutboxRow.event_id) == event_id) + .order_by(col(ChannelOutboxRow.part_index)) + .limit(128) + ) + return tuple( + StagedReply( + row.reply_id, + row.event_id, + row.response_id, + row.part_index, + row.text, + ensure_utc(row.created_at) or row.created_at, + ) + for row in rows.all() + ) diff --git a/src/agent_manager/infrastructure/persistence/channels/repository.py b/src/agent_manager/infrastructure/persistence/channels/repository.py new file mode 100644 index 00000000..97e8219b --- /dev/null +++ b/src/agent_manager/infrastructure/persistence/channels/repository.py @@ -0,0 +1,386 @@ +"""SQL inbox/binding transactions; claims never expire into duplicate work.""" + +from __future__ import annotations + +import secrets +from collections.abc import Sequence +from typing import cast + +from sqlalchemy import and_, or_, update +from sqlalchemy.ext.asyncio import async_sessionmaker +from sqlalchemy.sql.elements import ColumnElement +from sqlmodel import col, select +from sqlmodel.ext.asyncio.session import AsyncSession + +from agent_manager.domain.channels.models import ( + ChannelAddress, + ChannelBinding, + ChannelClaimConflict, + ChannelOwnerConflict, + InboundMessage, + InboxItem, + InboxState, + MessageKind, + StagedReply, + bounded_identifier, +) +from agent_manager.domain.channels.repository import ChannelRepository +from agent_manager.domain.pagination import ensure_utc +from agent_manager.infrastructure.persistence.channels.intake import ingest_in_session +from agent_manager.infrastructure.persistence.channels.replies import list_replies, stage_reply +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelBindingRow, + ChannelInboxRow, +) + +_TERMINAL = frozenset({InboxState.DONE, InboxState.FAILED, InboxState.DENIED}) + + +class SqlChannelRepository(ChannelRepository): + def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: + self._sessions = sessions + + async def ingest(self, message: InboundMessage) -> InboxItem: + async with self._sessions() as session, session.begin(): + return _item(await ingest_in_session(session, message)) + + async def get_inbound(self, event_id: str) -> InboxItem | None: + async with self._sessions() as session: + row = await _inbound(session, event_id) + return _item(row) if row is not None else None + + async def get_binding(self, address: ChannelAddress) -> ChannelBinding | None: + async with self._sessions() as session: + row = await session.get( + ChannelBindingRow, (address.connection_id, address.system_id, address.sender_id) + ) + return _binding(row) if row is not None else None + + async def list_active( + self, connection_id: str, system_id: str, *, after_sequence: int = 0, limit: int = 32 + ) -> tuple[InboxItem, ...]: + if after_sequence < 0 or not 1 <= limit <= 100: + raise ValueError("invalid channel recovery page bounds") + async with self._sessions() as session: + rows = await session.exec( + select(ChannelInboxRow) + .join(ChannelBindingRow, _binding_condition(ChannelInboxRow)) + .where( + col(ChannelInboxRow.connection_id) == connection_id, + col(ChannelInboxRow.system_id) == system_id, + col(ChannelInboxRow.sequence) > after_sequence, + col(ChannelBindingRow.active_event_id) == col(ChannelInboxRow.event_id), + col(ChannelInboxRow.state).in_( + ( + InboxState.PREPARING.value, + InboxState.PREPARED.value, + InboxState.RUNNING.value, + ) + ), + ) + .order_by(col(ChannelInboxRow.sequence)) + .limit(limit) + ) + return tuple(_item(row) for row in rows.all()) + + async def claim_next(self, connection_id: str, system_id: str) -> InboxItem | None: + async with self._sessions() as session, session.begin(): + pending = await session.exec( + select(ChannelInboxRow) + .join(ChannelBindingRow, _binding_condition(ChannelInboxRow)) + .where( + col(ChannelInboxRow.connection_id) == connection_id, + col(ChannelInboxRow.system_id) == system_id, + col(ChannelInboxRow.state) == InboxState.QUEUED.value, + col(ChannelBindingRow.active_event_id).is_(None), + ) + .order_by(col(ChannelInboxRow.sequence)) + .limit(32) + ) + for row in pending.all(): + claimed = await session.exec( + update(ChannelBindingRow) + .where( + _binding_condition(row), col(ChannelBindingRow.active_event_id).is_(None) + ) + .values(active_event_id=row.event_id) + .returning(col(ChannelBindingRow.connection_id)) + ) + if claimed.scalar_one_or_none() is None: + continue + changed = await session.exec( + update(ChannelInboxRow) + .where( + col(ChannelInboxRow.event_id) == row.event_id, + col(ChannelInboxRow.state) == InboxState.QUEUED.value, + ) + .values(state=InboxState.PREPARING.value, claim_token=secrets.token_hex(16)) + .returning(ChannelInboxRow) + ) + claimed_row = changed.scalar_one_or_none() + if claimed_row is None: + raise ChannelClaimConflict("inbox changed while claiming its sender") + return _item(claimed_row) + return None + + async def bind_conversation( + self, event_id: str, token: str, user_id: str, organization_id: str, conversation_id: str + ) -> ChannelBinding: + for value in (user_id, conversation_id): + bounded_identifier(value, limit=64) + bounded_identifier(organization_id) + async with self._sessions() as session, session.begin(): + row = await _claimed(session, event_id, token) + if row.state not in {InboxState.PREPARING, InboxState.PREPARED}: + raise ChannelClaimConflict("binding requires a preparing or prepared turn") + changed = await session.exec( + update(ChannelBindingRow) + .where( + _binding_condition(row), + col(ChannelBindingRow.active_event_id) == event_id, + or_( + col(ChannelBindingRow.user_id).is_(None), + col(ChannelBindingRow.user_id) == user_id, + ), + or_( + col(ChannelBindingRow.organization_id).is_(None), + col(ChannelBindingRow.organization_id) == organization_id, + ), + or_( + col(ChannelBindingRow.conversation_id).is_(None), + col(ChannelBindingRow.conversation_id) == conversation_id, + ), + ) + .values( + user_id=user_id, + organization_id=organization_id, + conversation_id=conversation_id, + ) + .returning(ChannelBindingRow) + ) + binding = changed.scalar_one_or_none() + if binding is None: + raise ChannelOwnerConflict("channel binding cannot change owners or conversation") + return _binding(binding) + + async def record_prepared( + self, event_id: str, token: str, conversation_id: str, run_id: str, message_id: str + ) -> InboxItem: + for value in (conversation_id, run_id, message_id): + bounded_identifier(value, limit=64) + async with self._sessions() as session, session.begin(): + row = await _claimed(session, event_id, token) + binding = await session.get( + ChannelBindingRow, (row.connection_id, row.system_id, row.sender_id) + ) + if binding is None or binding.conversation_id != conversation_id: + raise ChannelClaimConflict("prepared conversation does not match sender binding") + if row.state == InboxState.PREPARED: + if (row.conversation_id, row.run_id, row.message_id) != ( + conversation_id, + run_id, + message_id, + ): + raise ChannelClaimConflict("prepared work cannot change turn identifiers") + return _item(row) + if row.state != InboxState.PREPARING: + raise ChannelClaimConflict("work is not being prepared") + changed = await session.exec( + update(ChannelInboxRow) + .where( + col(ChannelInboxRow.event_id) == event_id, + col(ChannelInboxRow.state) == InboxState.PREPARING.value, + col(ChannelInboxRow.claim_token) == token, + ) + .values( + state=InboxState.PREPARED.value, + conversation_id=conversation_id, + run_id=run_id, + message_id=message_id, + ) + .returning(ChannelInboxRow) + ) + prepared = changed.scalar_one_or_none() + if prepared is None: + # Another caller can commit this same preparation between the + # read and CAS. Reuse its receipt without overwriting its IDs. + await session.refresh(row) + if row.state == InboxState.PREPARED and ( + row.conversation_id, + row.run_id, + row.message_id, + ) == (conversation_id, run_id, message_id): + return _item(row) + raise ChannelClaimConflict("work changed during preparation") + return _item(prepared) + + async def claim_execution(self, event_id: str, token: str) -> bool: + async with self._sessions() as session, session.begin(): + changed = await session.exec( + update(ChannelInboxRow) + .where( + col(ChannelInboxRow.event_id) == event_id, + col(ChannelInboxRow.claim_token) == token, + col(ChannelInboxRow.state) == InboxState.PREPARED.value, + ) + .values(state=InboxState.RUNNING.value) + .returning(col(ChannelInboxRow.event_id)) + ) + return changed.scalar_one_or_none() is not None + + async def finish(self, event_id: str, token: str, state: InboxState) -> None: + if state not in _TERMINAL | {InboxState.NEEDS_REVIEW}: + raise ValueError("finish requires a terminal or review state") + async with self._sessions() as session, session.begin(): + row = await _claimed(session, event_id, token, require_active=False) + if row.state == state: + return + if row.state in _TERMINAL | {InboxState.NEEDS_REVIEW}: + raise ChannelClaimConflict("finished or review work cannot be reopened") + if state is InboxState.DENIED and row.state != InboxState.PREPARING: + raise ChannelClaimConflict("prepared or running identity denial requires review") + if row.state == InboxState.ANSWERED and state is not InboxState.DONE: + raise ChannelClaimConflict("staged replies must be reconciled by delivery") + changed = await session.exec( + update(ChannelInboxRow) + .where( + col(ChannelInboxRow.event_id) == event_id, + col(ChannelInboxRow.state) == row.state, + ) + .values(state=state.value) + .returning(col(ChannelInboxRow.event_id)) + ) + if changed.scalar_one_or_none() is None: + raise ChannelClaimConflict("work changed before completion") + if state is not InboxState.NEEDS_REVIEW: + await _release(session, row) + + async def reset_conversation(self, event_id: str, token: str) -> int: + async with self._sessions() as session, session.begin(): + row = await _claimed(session, event_id, token, require_active=False) + if row.kind != "reset": + raise ChannelClaimConflict("only a claimed reset event can reset a conversation") + if row.state == InboxState.DONE and row.reset_generation is not None: + return row.reset_generation + if row.state != InboxState.PREPARING: + raise ChannelClaimConflict("reset is not being prepared") + binding = await session.get( + ChannelBindingRow, (row.connection_id, row.system_id, row.sender_id) + ) + if binding is None or binding.active_event_id != event_id: + await session.refresh(row) + if row.state == InboxState.DONE and row.reset_generation is not None: + return row.reset_generation + raise ChannelClaimConflict("reset does not own its sender") + generation = binding.generation + 1 + changed = await session.exec( + update(ChannelInboxRow) + .where( + col(ChannelInboxRow.event_id) == event_id, + col(ChannelInboxRow.state) == InboxState.PREPARING.value, + col(ChannelInboxRow.claim_token) == token, + ) + .values(state=InboxState.DONE.value, reset_generation=generation) + .returning(col(ChannelInboxRow.event_id)) + ) + if changed.scalar_one_or_none() is None: + await session.refresh(row) + if row.state == InboxState.DONE and row.reset_generation is not None: + return row.reset_generation + raise ChannelClaimConflict("reset changed before its commit") + reset_binding = await session.exec( + update(ChannelBindingRow) + .where( + _binding_condition(row), + col(ChannelBindingRow.active_event_id) == event_id, + col(ChannelBindingRow.generation) == binding.generation, + ) + .values(conversation_id=None, generation=generation, active_event_id=None) + .returning(col(ChannelBindingRow.connection_id)) + ) + if reset_binding.scalar_one_or_none() is None: + raise ChannelClaimConflict("reset lost its sender generation") + return generation + + async def stage_reply( + self, event_id: str, token: str, response_id: str, parts: Sequence[str] + ) -> tuple[StagedReply, ...]: + return await stage_reply(self._sessions, event_id, token, response_id, parts) + + async def list_replies(self, event_id: str) -> tuple[StagedReply, ...]: + return await list_replies(self._sessions, event_id) + + +def _binding_condition(row: ChannelInboxRow | type[ChannelInboxRow]) -> ColumnElement[bool]: + return and_( + col(ChannelBindingRow.connection_id) == row.connection_id, + col(ChannelBindingRow.system_id) == row.system_id, + col(ChannelBindingRow.sender_id) == row.sender_id, + ) + + +async def _inbound(session: AsyncSession, event_id: str) -> ChannelInboxRow | None: + rows = await session.exec( + select(ChannelInboxRow) + .where(col(ChannelInboxRow.event_id) == event_id) + .execution_options(populate_existing=True) + .with_for_update() + ) + return rows.first() + + +async def _claimed( + session: AsyncSession, event_id: str, token: str, *, require_active: bool = True +) -> ChannelInboxRow: + row = await _inbound(session, event_id) + if row is None or not token or row.claim_token != token: + raise ChannelClaimConflict("invalid channel work claim") + if require_active: + binding = await session.get( + ChannelBindingRow, (row.connection_id, row.system_id, row.sender_id) + ) + if binding is None or binding.active_event_id != event_id: + raise ChannelClaimConflict("work no longer owns its sender") + return row + + +async def _release(session: AsyncSession, row: ChannelInboxRow) -> None: + await session.exec( + update(ChannelBindingRow) + .where(_binding_condition(row), col(ChannelBindingRow.active_event_id) == row.event_id) + .values(active_event_id=None) + ) + + +def _binding(row: ChannelBindingRow) -> ChannelBinding: + return ChannelBinding( + ChannelAddress(row.connection_id, row.system_id, row.sender_id), + row.generation, + row.user_id, + row.organization_id, + row.conversation_id, + row.active_event_id, + ) + + +def _item(row: ChannelInboxRow) -> InboxItem: + assert row.sequence is not None + received_at = ensure_utc(row.received_at) + assert received_at is not None + return InboxItem( + row.event_id, + row.sequence, + InboundMessage( + ChannelAddress(row.connection_id, row.system_id, row.sender_id), + row.provider_message_id, + row.content, + received_at, + ensure_utc(row.sent_at), + cast(MessageKind, row.kind), + ), + InboxState(row.state), + row.claim_token, + row.conversation_id, + row.run_id, + row.message_id, + ) diff --git a/src/agent_manager/infrastructure/persistence/channels/tables.py b/src/agent_manager/infrastructure/persistence/channels/tables.py new file mode 100644 index 00000000..e7b2c2f5 --- /dev/null +++ b/src/agent_manager/infrastructure/persistence/channels/tables.py @@ -0,0 +1,134 @@ +"""Durable sender ownership, ordered inbox, and immutable reply parts.""" + +from datetime import datetime + +from sqlalchemy import JSON, Column, DateTime, Index, Text, UniqueConstraint +from sqlmodel import Field, SQLModel + + +class ChannelBindingRow(SQLModel, table=True): + __tablename__ = "channel_bindings" + + connection_id: str = Field(primary_key=True, max_length=128) + system_id: str = Field(primary_key=True, max_length=256) + sender_id: str = Field(primary_key=True, max_length=512, repr=False) + generation: int = 0 + user_id: str | None = Field(default=None, max_length=64) + organization_id: str | None = Field(default=None, max_length=512) + conversation_id: str | None = Field(default=None, max_length=64) + active_event_id: str | None = Field(default=None, max_length=64) + opted_out: bool = False + suppressed_through_sequence: int = 0 + preference_at: datetime | None = Field(default=None, sa_column=Column(DateTime(timezone=True))) + latest_inbound_at: datetime | None = Field( + default=None, sa_column=Column(DateTime(timezone=True)) + ) + + +class ChannelInboxRow(SQLModel, table=True): + __tablename__ = "channel_inbox" + __table_args__ = ( + UniqueConstraint("connection_id", "provider_message_id"), + Index("idx_channel_inbox_pending", "connection_id", "system_id", "state", "sequence"), + ) + + sequence: int | None = Field(default=None, primary_key=True) + event_id: str = Field(unique=True, max_length=64) + connection_id: str = Field(max_length=128) + system_id: str = Field(max_length=256) + sender_id: str = Field(max_length=512, repr=False) + provider_message_id: str = Field(max_length=512) + kind: str = Field(max_length=32) + content: str = Field(sa_column=Column(Text, nullable=False), repr=False) + received_at: datetime = Field(sa_column=Column(DateTime(timezone=True), nullable=False)) + sent_at: datetime | None = Field(default=None, sa_column=Column(DateTime(timezone=True))) + state: str = Field(default="queued", max_length=32) + claim_token: str | None = Field(default=None, max_length=64, repr=False) + conversation_id: str | None = Field(default=None, max_length=64) + run_id: str | None = Field(default=None, max_length=64) + message_id: str | None = Field(default=None, max_length=64) + response_id: str | None = Field(default=None, max_length=64) + reset_generation: int | None = None + + +class ChannelOutboxRow(SQLModel, table=True): + __tablename__ = "channel_outbox" + __table_args__ = (UniqueConstraint("event_id", "response_id", "part_index"),) + + reply_id: str = Field(primary_key=True, max_length=64) + event_id: str = Field(foreign_key="channel_inbox.event_id", index=True, max_length=64) + response_id: str = Field(max_length=64) + part_index: int + text: str = Field(sa_column=Column(Text, nullable=False), repr=False) + created_at: datetime = Field(sa_column=Column(DateTime(timezone=True), nullable=False)) + + +class ChannelWebhookReceiptRow(SQLModel, table=True): + __tablename__ = "channel_webhook_receipts" + __table_args__ = (UniqueConstraint("connection_id", "body_digest"),) + + receipt_id: str = Field(primary_key=True, max_length=64) + connection_id: str = Field(max_length=128) + system_id: str = Field(max_length=256) + body_digest: str = Field(max_length=64) + received_at: datetime = Field(sa_column=Column(DateTime(timezone=True), nullable=False)) + event_ids: list[str] = Field(sa_column=Column(JSON, nullable=False)) + delivery_event_ids: list[str] = Field(sa_column=Column(JSON, nullable=False)) + issues: list[dict[str, str]] = Field(sa_column=Column(JSON, nullable=False)) + + +class ChannelDeliveryEventRow(SQLModel, table=True): + __tablename__ = "channel_delivery_events" + __table_args__ = ( + Index("idx_channel_delivery_message", "connection_id", "provider_message_id"), + ) + + event_id: str = Field(primary_key=True, max_length=64) + connection_id: str = Field(max_length=128) + system_id: str = Field(max_length=256) + provider_message_id: str = Field(max_length=512) + recipient_id: str = Field(max_length=512, repr=False) + status: str = Field(max_length=64) + observed_at: datetime = Field(sa_column=Column(DateTime(timezone=True), nullable=False)) + received_at: datetime = Field(sa_column=Column(DateTime(timezone=True), nullable=False)) + projected: bool = False + + +class ChannelDeliveryBatchRow(SQLModel, table=True): + __tablename__ = "channel_delivery_batches" + __table_args__ = ( + Index("idx_delivery_recipient", "connection_id", "system_id", "sender_id", "sequence"), + ) + + event_id: str = Field(primary_key=True, foreign_key="channel_inbox.event_id", max_length=64) + connection_id: str = Field(max_length=128) + system_id: str = Field(max_length=256) + sender_id: str = Field(max_length=512, repr=False) + sequence: int + user_id: str | None = Field(default=None, max_length=64) + organization_id: str | None = Field(default=None, max_length=512) + state: str = Field(default="ready", max_length=32) + reason: str | None = Field(default=None, max_length=64) + + +class ChannelDeliveryPartRow(SQLModel, table=True): + __tablename__ = "channel_delivery_parts" + __table_args__ = ( + UniqueConstraint("event_id", "part_index"), + UniqueConstraint("connection_id", "provider_message_id"), + Index("idx_delivery_pending", "state", "next_attempt_at"), + ) + + delivery_id: str = Field(primary_key=True, max_length=64) + event_id: str = Field(foreign_key="channel_delivery_batches.event_id", max_length=64) + connection_id: str = Field(max_length=128) + part_index: int + text: str = Field(sa_column=Column(Text, nullable=False), repr=False) + state: str = Field(default="ready", max_length=32) + attempts: int = 0 + token: str | None = Field(default=None, max_length=64, repr=False) + provider_message_id: str | None = Field(default=None, max_length=512) + next_attempt_at: datetime | None = Field( + default=None, sa_column=Column(DateTime(timezone=True)) + ) + reason: str | None = Field(default=None, max_length=64) diff --git a/src/agent_manager/infrastructure/persistence/channels/webhooks.py b/src/agent_manager/infrastructure/persistence/channels/webhooks.py new file mode 100644 index 00000000..fd2e9b54 --- /dev/null +++ b/src/agent_manager/infrastructure/persistence/channels/webhooks.py @@ -0,0 +1,128 @@ +"""One SQL transaction accepts messages, delivery events, and safe quarantine.""" + +from __future__ import annotations + +from datetime import datetime +from typing import cast + +from sqlalchemy.ext.asyncio import async_sessionmaker +from sqlmodel import col +from sqlmodel.ext.asyncio.session import AsyncSession + +from agent_manager.domain.channels.models import ChannelMessageConflict, stable_id +from agent_manager.domain.channels.webhooks import ( + IssueReason, + WebhookBatch, + WebhookIssue, + WebhookReceipt, + WebhookRepository, +) +from agent_manager.domain.pagination import ensure_utc +from agent_manager.infrastructure.persistence.channels.intake import ingest_in_session, sql_insert +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelDeliveryEventRow, + ChannelWebhookReceiptRow, +) + + +class SqlWebhookRepository(WebhookRepository): + def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: + self._sessions = sessions + + async def accept( + self, + connection_id: str, + system_id: str, + body_digest: str, + received_at: datetime, + batch: WebhookBatch, + ) -> WebhookReceipt: + if any( + message.address.connection_id != connection_id or message.address.system_id != system_id + for message in batch.messages + ): + raise ValueError("webhook messages must match the configured target") + receipt_id = stable_id("webhook", connection_id, body_digest) + async with self._sessions() as session, session.begin(): + inserted = await session.exec( + sql_insert(session, ChannelWebhookReceiptRow) + .values( + receipt_id=receipt_id, + connection_id=connection_id, + system_id=system_id, + body_digest=body_digest, + received_at=received_at, + event_ids=[], + delivery_event_ids=[], + issues=[], + ) + .on_conflict_do_nothing(index_elements=["connection_id", "body_digest"]) + .returning(col(ChannelWebhookReceiptRow.receipt_id)) + ) + created = inserted.scalar_one_or_none() is not None + row = await session.get(ChannelWebhookReceiptRow, receipt_id) + assert row is not None + if not created: + return _receipt(row) + event_ids: list[str] = [] + issues = list(batch.issues) + for index, message in enumerate(batch.messages): + try: + inbox = await ingest_in_session(session, message) + event_ids.append(inbox.event_id) + except ChannelMessageConflict: + issues.append(WebhookIssue(f"messages.{index}", "conflicting_message")) + delivery_ids: list[str] = [] + for event in batch.delivery_events: + event_id = stable_id( + "delivery", + connection_id, + event.provider_message_id, + event.recipient_id, + event.status, + event.observed_at.isoformat(), + ) + await session.exec( + sql_insert(session, ChannelDeliveryEventRow) + .values( + event_id=event_id, + connection_id=connection_id, + system_id=system_id, + provider_message_id=event.provider_message_id, + recipient_id=event.recipient_id, + status=event.status, + observed_at=event.observed_at, + received_at=received_at, + ) + .on_conflict_do_nothing(index_elements=["event_id"]) + ) + delivery_ids.append(event_id) + if len(issues) > 256: + issues = [*issues[:255], WebhookIssue("body", "issue_limit")] + row.event_ids = list(dict.fromkeys(event_ids)) + row.delivery_event_ids = list(dict.fromkeys(delivery_ids)) + row.issues = [{"location": issue.location, "reason": issue.reason} for issue in issues] + session.add(row) + await session.flush() + return _receipt(row) + + async def get_receipt(self, receipt_id: str) -> WebhookReceipt | None: + async with self._sessions() as session: + row = await session.get(ChannelWebhookReceiptRow, receipt_id) + return _receipt(row) if row is not None else None + + +def _receipt(row: ChannelWebhookReceiptRow) -> WebhookReceipt: + return WebhookReceipt( + row.receipt_id, + row.connection_id, + row.system_id, + row.body_digest, + ensure_utc(row.received_at) or row.received_at, + tuple(row.event_ids), + tuple(row.delivery_event_ids), + tuple( + WebhookIssue(issue["location"], cast(IssueReason, issue["reason"])) + for issue in row.issues + ), + ) diff --git a/src/agent_manager/infrastructure/persistence/migrations/versions/0008_channel_storage.py b/src/agent_manager/infrastructure/persistence/migrations/versions/0008_channel_storage.py new file mode 100644 index 00000000..fc515427 --- /dev/null +++ b/src/agent_manager/infrastructure/persistence/migrations/versions/0008_channel_storage.py @@ -0,0 +1,74 @@ +"""Durable normalized channel inbox, sender bindings, and reply parts. + +Revision ID: 0008 +Revises: 0007 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision: str = "0008" +down_revision: str | None = "0007" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "channel_bindings", + sa.Column("connection_id", sa.String(128), nullable=False), + sa.Column("system_id", sa.String(256), nullable=False), + sa.Column("sender_id", sa.String(512), nullable=False), + sa.Column("generation", sa.Integer(), nullable=False), + sa.Column("user_id", sa.String(64)), + sa.Column("organization_id", sa.String(512)), + sa.Column("conversation_id", sa.String(64)), + sa.Column("active_event_id", sa.String(64)), + sa.PrimaryKeyConstraint("connection_id", "system_id", "sender_id"), + ) + op.create_table( + "channel_inbox", + sa.Column("sequence", sa.Integer(), primary_key=True, autoincrement=True), + sa.Column("event_id", sa.String(64), nullable=False, unique=True), + sa.Column("connection_id", sa.String(128), nullable=False), + sa.Column("system_id", sa.String(256), nullable=False), + sa.Column("sender_id", sa.String(512), nullable=False), + sa.Column("provider_message_id", sa.String(512), nullable=False), + sa.Column("kind", sa.String(32), nullable=False), + sa.Column("content", sa.Text(), nullable=False), + sa.Column("received_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("sent_at", sa.DateTime(timezone=True)), + sa.Column("state", sa.String(32), nullable=False), + sa.Column("claim_token", sa.String(64)), + sa.Column("conversation_id", sa.String(64)), + sa.Column("run_id", sa.String(64)), + sa.Column("message_id", sa.String(64)), + sa.Column("response_id", sa.String(64)), + sa.Column("reset_generation", sa.Integer()), + sa.UniqueConstraint("connection_id", "provider_message_id"), + ) + op.create_index( + "idx_channel_inbox_pending", + "channel_inbox", + ["connection_id", "system_id", "state", "sequence"], + ) + op.create_table( + "channel_outbox", + sa.Column("reply_id", sa.String(64), primary_key=True), + sa.Column("event_id", sa.String(64), nullable=False), + sa.Column("response_id", sa.String(64), nullable=False), + sa.Column("part_index", sa.Integer(), nullable=False), + sa.Column("text", sa.Text(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False), + sa.ForeignKeyConstraint(["event_id"], ["channel_inbox.event_id"]), + sa.UniqueConstraint("event_id", "response_id", "part_index"), + ) + op.create_index("ix_channel_outbox_event_id", "channel_outbox", ["event_id"]) + + +def downgrade() -> None: + op.drop_table("channel_outbox") + op.drop_table("channel_inbox") + op.drop_table("channel_bindings") diff --git a/src/agent_manager/infrastructure/persistence/migrations/versions/0009_webhook_receipts.py b/src/agent_manager/infrastructure/persistence/migrations/versions/0009_webhook_receipts.py new file mode 100644 index 00000000..1bdcb2f6 --- /dev/null +++ b/src/agent_manager/infrastructure/persistence/migrations/versions/0009_webhook_receipts.py @@ -0,0 +1,51 @@ +"""Authenticated webhook receipts and normalized delivery observations. + +Revision ID: 0009 +Revises: 0008 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision: str = "0009" +down_revision: str | None = "0008" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "channel_webhook_receipts", + sa.Column("receipt_id", sa.String(64), primary_key=True), + sa.Column("connection_id", sa.String(128), nullable=False), + sa.Column("system_id", sa.String(256), nullable=False), + sa.Column("body_digest", sa.String(64), nullable=False), + sa.Column("received_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("event_ids", sa.JSON(), nullable=False), + sa.Column("delivery_event_ids", sa.JSON(), nullable=False), + sa.Column("issues", sa.JSON(), nullable=False), + sa.UniqueConstraint("connection_id", "body_digest"), + ) + op.create_table( + "channel_delivery_events", + sa.Column("event_id", sa.String(64), primary_key=True), + sa.Column("connection_id", sa.String(128), nullable=False), + sa.Column("system_id", sa.String(256), nullable=False), + sa.Column("provider_message_id", sa.String(512), nullable=False), + sa.Column("recipient_id", sa.String(512), nullable=False), + sa.Column("status", sa.String(64), nullable=False), + sa.Column("observed_at", sa.DateTime(timezone=True), nullable=False), + sa.Column("received_at", sa.DateTime(timezone=True), nullable=False), + ) + op.create_index( + "idx_channel_delivery_message", + "channel_delivery_events", + ["connection_id", "provider_message_id"], + ) + + +def downgrade() -> None: + op.drop_table("channel_delivery_events") + op.drop_table("channel_webhook_receipts") diff --git a/src/agent_manager/infrastructure/persistence/migrations/versions/0010_channel_delivery.py b/src/agent_manager/infrastructure/persistence/migrations/versions/0010_channel_delivery.py new file mode 100644 index 00000000..e9247488 --- /dev/null +++ b/src/agent_manager/infrastructure/persistence/migrations/versions/0010_channel_delivery.py @@ -0,0 +1,93 @@ +"""Provider delivery parts and durable sender suppression/window state. + +Revision ID: 0010 +Revises: 0009 +""" + +import sqlalchemy as sa +from alembic import op + +revision = "0010" +down_revision = "0009" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "channel_bindings", + sa.Column("opted_out", sa.Boolean(), server_default=sa.false(), nullable=False), + ) + op.add_column( + "channel_bindings", + sa.Column("suppressed_through_sequence", sa.Integer(), server_default="0", nullable=False), + ) + op.add_column("channel_bindings", sa.Column("preference_at", sa.DateTime(timezone=True))) + op.add_column("channel_bindings", sa.Column("latest_inbound_at", sa.DateTime(timezone=True))) + op.add_column( + "channel_delivery_events", + sa.Column("projected", sa.Boolean(), server_default=sa.false(), nullable=False), + ) + op.execute( + sa.text("""UPDATE channel_bindings SET latest_inbound_at = ( + SELECT MAX(CASE WHEN sent_at > received_at THEN received_at ELSE sent_at END) + FROM channel_inbox WHERE channel_inbox.connection_id = channel_bindings.connection_id + AND channel_inbox.system_id = channel_bindings.system_id + AND channel_inbox.sender_id = channel_bindings.sender_id AND sent_at IS NOT NULL + AND kind IN ('text', 'reset', 'help', 'stop', 'start') + )""") + ) + op.create_table( + "channel_delivery_batches", + sa.Column( + "event_id", sa.String(64), sa.ForeignKey("channel_inbox.event_id"), primary_key=True + ), + sa.Column("connection_id", sa.String(128), nullable=False), + sa.Column("system_id", sa.String(256), nullable=False), + sa.Column("sender_id", sa.String(512), nullable=False), + sa.Column("sequence", sa.Integer(), nullable=False), + sa.Column("user_id", sa.String(64)), + sa.Column("organization_id", sa.String(512)), + sa.Column("state", sa.String(32), nullable=False), + sa.Column("reason", sa.String(64)), + ) + op.create_index( + "idx_delivery_recipient", + "channel_delivery_batches", + ["connection_id", "system_id", "sender_id", "sequence"], + ) + op.create_table( + "channel_delivery_parts", + sa.Column("delivery_id", sa.String(64), primary_key=True), + sa.Column( + "event_id", + sa.String(64), + sa.ForeignKey("channel_delivery_batches.event_id"), + nullable=False, + ), + sa.Column("connection_id", sa.String(128), nullable=False), + sa.Column("part_index", sa.Integer(), nullable=False), + sa.Column("text", sa.Text(), nullable=False), + sa.Column("state", sa.String(32), nullable=False), + sa.Column("attempts", sa.Integer(), nullable=False), + sa.Column("token", sa.String(64)), + sa.Column("provider_message_id", sa.String(512)), + sa.Column("next_attempt_at", sa.DateTime(timezone=True)), + sa.Column("reason", sa.String(64)), + sa.UniqueConstraint("event_id", "part_index"), + sa.UniqueConstraint("connection_id", "provider_message_id"), + ) + op.create_index("idx_delivery_pending", "channel_delivery_parts", ["state", "next_attempt_at"]) + + +def downgrade() -> None: + op.drop_table("channel_delivery_parts") + op.drop_table("channel_delivery_batches") + op.drop_column("channel_delivery_events", "projected") + for column in ( + "latest_inbound_at", + "preference_at", + "suppressed_through_sequence", + "opted_out", + ): + op.drop_column("channel_bindings", column) diff --git a/src/agent_manager/infrastructure/persistence/tables.py b/src/agent_manager/infrastructure/persistence/tables.py index e0cb200f..e122e68a 100644 --- a/src/agent_manager/infrastructure/persistence/tables.py +++ b/src/agent_manager/infrastructure/persistence/tables.py @@ -17,6 +17,15 @@ from sqlalchemy import JSON, Column, DateTime, Float, Index, Integer, Text, text from sqlmodel import Field, SQLModel +# Register channel tables with the same metadata used by Alembic and tests. +from agent_manager.infrastructure.persistence.channels.tables import ( # noqa: F401 + ChannelBindingRow, + ChannelDeliveryEventRow, + ChannelInboxRow, + ChannelOutboxRow, + ChannelWebhookReceiptRow, +) + class ConversationUserRow(SQLModel, table=True): __tablename__ = "conversation_users" diff --git a/tests/agent_manager/test_channel_migration.py b/tests/agent_manager/test_channel_migration.py new file mode 100644 index 00000000..106d41ad --- /dev/null +++ b/tests/agent_manager/test_channel_migration.py @@ -0,0 +1,116 @@ +"""The channel schema is explicitly migrated and usable after reopening SQL.""" + +from __future__ import annotations + +import asyncio +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from alembic import command +from alembic.autogenerate import compare_metadata +from alembic.config import Config +from alembic.migration import MigrationContext +from sqlalchemy import MetaData, create_engine +from sqlmodel import SQLModel + +from agent_manager.config import Settings +from agent_manager.domain.channels.models import ChannelAddress, InboundMessage +from agent_manager.infrastructure.persistence.channels.repository import SqlChannelRepository +from agent_manager.infrastructure.persistence.database import ( + create_db_engine, + session_factory, + upgrade_database, +) +from agent_manager.infrastructure.persistence.sql_repository import SqlRepository + + +def test_upgrade_and_downgrade_preserve_existing_conversations( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + path = tmp_path / "migration.db" + url = f"sqlite+aiosqlite:///{path}" + monkeypatch.setattr( + "agent_manager.config.Settings", lambda: Settings.from_values(database_url=url) + ) + migrations = ( + Path(__file__).resolve().parents[2] + / "src/agent_manager/infrastructure/persistence/migrations" + ) + config = Config(str(migrations / "alembic.ini")) + config.set_main_option("script_location", str(migrations)) + command.upgrade(config, "0007") + + async def existing_conversation(*, create: bool = False) -> bool: + engine = create_db_engine(url) + try: + repository = SqlRepository(session_factory(engine)) + if create: + await repository.create_session("existing") + return await repository.conversation_exists("existing") + finally: + await engine.dispose() + + assert asyncio.run(existing_conversation(create=True)) + assert upgrade_database() + + async def use_channels() -> str: + engine = create_db_engine(url) + try: + repository = SqlChannelRepository(session_factory(engine)) + timestamp = datetime(2026, 10, 15, tzinfo=UTC) + item = await repository.ingest( + InboundMessage( + ChannelAddress("connection", "system", "sender"), + "event", + "hello", + timestamp, + timestamp, + ) + ) + claimed = await repository.claim_next("connection", "system") + assert claimed is not None and claimed.claim_token is not None + await repository.bind_conversation( + item.event_id, claimed.claim_token, "user", "org", "existing" + ) + await repository.record_prepared( + item.event_id, claimed.claim_token, "existing", "run", "message" + ) + assert await repository.claim_execution(item.event_id, claimed.claim_token) + replies = await repository.stage_reply( + item.event_id, claimed.claim_token, "answer", ("saved",) + ) + return replies[0].reply_id + finally: + await engine.dispose() + + reply_id = asyncio.run(use_channels()) + database = create_engine(f"sqlite:///{path}") + try: + with database.connect() as connection: + channel_schema = MetaData() + for table in SQLModel.metadata.sorted_tables: + if table.name.startswith("channel_"): + table.to_metadata(channel_schema) + differences = compare_metadata( + MigrationContext.configure( + connection, + opts={ + "include_name": lambda name, kind, parents: ( + kind != "table" or name.startswith("channel_") + ) + }, + ), + channel_schema, + ) + assert differences == [] + assert ( + connection.exec_driver_sql("SELECT reply_id FROM channel_outbox").scalar_one() + == reply_id + ) + finally: + database.dispose() + command.downgrade(config, "0007") + assert asyncio.run(existing_conversation()) + assert upgrade_database() + assert asyncio.run(existing_conversation()) diff --git a/tests/agent_manager/test_channel_worker_ownership.py b/tests/agent_manager/test_channel_worker_ownership.py new file mode 100644 index 00000000..11d1839a --- /dev/null +++ b/tests/agent_manager/test_channel_worker_ownership.py @@ -0,0 +1,158 @@ +"""Exclusive local ownership and dedicated PostgreSQL-session contract.""" + +from pathlib import Path + +import pytest + +from agent_manager.domain.channels.ownership import ( + ChannelWorkerAlreadyRunning, + ChannelWorkerOwnershipLost, +) + + +async def test_sqlite_connection_has_one_owner_and_releases_after_an_exception(tmp_path: Path): + from agent_manager.infrastructure.persistence.channels.ownership import ( + SqlChannelWorkerOwnership, + ) + + url = f"sqlite+aiosqlite:///{tmp_path / 'queue.db'}" + first = SqlChannelWorkerOwnership(url, "connection") + second = SqlChannelWorkerOwnership(url, "connection") + with pytest.raises(RuntimeError, match="injected"): + async with first.hold(): + await first.check() + with pytest.raises(ChannelWorkerAlreadyRunning): + async with second.hold(): + pytest.fail("second owner entered the same connection") + async with SqlChannelWorkerOwnership(url, "other").hold(): + pass + raise RuntimeError("injected") + with pytest.raises(ChannelWorkerOwnershipLost): + await first.check() + async with second.hold(): + await second.check() + + +async def test_sqlite_uri_and_path_aliases_share_one_lock(tmp_path: Path): + from agent_manager.infrastructure.persistence.channels.ownership import ( + SqlChannelWorkerOwnership, + ) + + database = tmp_path / "queue.db" + url = f"sqlite+aiosqlite:///{database}" + alias = f"sqlite+aiosqlite:///file:{database}?uri=true" + async with SqlChannelWorkerOwnership(url, "connection").hold(): + with pytest.raises(ChannelWorkerAlreadyRunning): + async with SqlChannelWorkerOwnership(alias, "connection").hold(): + pytest.fail("URI alias bypassed connection ownership") + + +class PgResult: + def __init__(self, value): + self.value = value + + def scalar_one(self): + return self.value + + +class PgSession: + def __init__(self): + self.pid = 100 + self.locked = False + self.unlocks = 0 + self.closed = False + + async def __aenter__(self): + self.closed = False + return self + + async def __aexit__(self, *args): + self.closed = True + self.locked = False + + async def execute(self, statement, parameters=None): + query = str(statement) + if "pg_try_advisory_lock" in query: + granted = not self.locked + self.locked = True + return PgResult(granted) + if "pg_advisory_unlock" in query: + self.locked = False + self.unlocks += 1 + return PgResult(True) + if "pg_backend_pid" in query: + return PgResult(self.pid) + raise AssertionError(f"unexpected ownership SQL: {query}") + + +class PgEngine: + def __init__(self, session): + self.session = session + self.disposed = False + + def connect(self): + return self.session + + async def dispose(self): + self.disposed = True + + +async def test_postgres_ownership_detects_session_replacement_and_closes_its_connection( + monkeypatch, +): + from agent_manager.infrastructure.persistence.channels.ownership import ( + SqlChannelWorkerOwnership, + ) + + session = PgSession() + engine = PgEngine(session) + monkeypatch.setattr( + "agent_manager.infrastructure.persistence.channels.ownership.create_async_engine", + lambda *a, **kw: engine, + ) + owner = SqlChannelWorkerOwnership("postgresql+asyncpg://unused.invalid/test", "connection") + async with owner.hold(): + await owner.check() + session.pid = 101 + with pytest.raises(ChannelWorkerOwnershipLost): + await owner.check() + assert session.closed and engine.disposed + + +async def test_postgres_busy_lock_fails_without_polling_or_stealing(monkeypatch): + from agent_manager.infrastructure.persistence.channels.ownership import ( + SqlChannelWorkerOwnership, + ) + + session = PgSession() + session.locked = True + engine = PgEngine(session) + monkeypatch.setattr( + "agent_manager.infrastructure.persistence.channels.ownership.create_async_engine", + lambda *a, **kw: engine, + ) + with pytest.raises(ChannelWorkerAlreadyRunning): + async with SqlChannelWorkerOwnership( + "postgresql+asyncpg://unused.invalid/test", "connection" + ).hold(): + pytest.fail("busy PostgreSQL lock was acquired") + assert session.unlocks == 0 and engine.disposed + + +async def test_posix_permission_busy_errno_is_reported_as_another_owner(tmp_path, monkeypatch): + import errno + import fcntl + + from agent_manager.infrastructure.persistence.channels.ownership import ( + SqlChannelWorkerOwnership, + ) + + def busy(*args): + raise OSError(errno.EACCES, "injected busy lock") + + monkeypatch.setattr(fcntl, "flock", busy) + with pytest.raises(ChannelWorkerAlreadyRunning): + async with SqlChannelWorkerOwnership( + f"sqlite+aiosqlite:///{tmp_path / 'queue.db'}", "connection" + ).hold(): + pytest.fail("busy lock entered") diff --git a/tests/agent_manager/test_delivery_migration.py b/tests/agent_manager/test_delivery_migration.py new file mode 100644 index 00000000..9d542189 --- /dev/null +++ b/tests/agent_manager/test_delivery_migration.py @@ -0,0 +1,93 @@ +"""Upgrade an existing staged reply and preserve it through downgrade.""" + +import asyncio +from datetime import UTC, datetime +from pathlib import Path + +from alembic import command +from alembic.config import Config +from sqlalchemy import create_engine, text + +from agent_manager.config import Settings +from agent_manager.infrastructure.persistence.channels.delivery import SqlDeliveryRepository +from agent_manager.infrastructure.persistence.database import create_db_engine, session_factory +from agent_manager.infrastructure.persistence.sql_repository import SqlRepository + + +def test_delivery_migration_preserves_old_outbox_and_backfills_verified_window( + tmp_path, monkeypatch +): + path = tmp_path / "delivery.db" + url = f"sqlite+aiosqlite:///{path}" + monkeypatch.setattr( + "agent_manager.config.Settings", lambda: Settings.from_values(database_url=url) + ) + migrations = ( + Path(__file__).parents[2] / "src/agent_manager/infrastructure/persistence/migrations" + ) + config = Config(str(migrations / "alembic.ini")) + config.set_main_option("script_location", str(migrations)) + command.upgrade(config, "0009") + now = datetime(2026, 10, 16, tzinfo=UTC) + + async def session(): + database = create_db_engine(url) + try: + await SqlRepository(session_factory(database)).create_session( + "conversation", user_id="owner" + ) + finally: + await database.dispose() + + asyncio.run(session()) + sync = create_engine(f"sqlite:///{path}") + try: + with sync.begin() as connection: + connection.execute( + text( + "INSERT INTO channel_bindings (connection_id, system_id, sender_id, " + "generation, user_id, organization_id, conversation_id) " + "VALUES ('connection','system','sender',0,'owner','org','conversation')" + ) + ) + connection.execute( + text( + "INSERT INTO channel_inbox (event_id, connection_id, system_id, sender_id, " + "provider_message_id, kind, content, received_at, sent_at, state, " + "conversation_id) VALUES ('event','connection','system','sender'," + "'inbound','text','question',:now,:now,'answered','conversation')" + ), + {"now": now.isoformat()}, + ) + connection.execute( + text( + "INSERT INTO channel_outbox (reply_id, event_id, response_id, part_index, " + "text, created_at) VALUES ('reply','event','response',0,'saved answer',:now)" + ), + {"now": now.isoformat()}, + ) + command.upgrade(config, "head") + + async def recover(): + database = create_db_engine(url) + try: + delivery = SqlDeliveryRepository(session_factory(database)) + assert (await delivery.sources("connection", "system"))[0].texts == ( + "saved answer", + ) + await delivery.materialize("event", ("saved answer",)) + part = await delivery.claim_next("connection", "system", now) + assert part is not None + assert part.user_id == "owner" and await delivery.guard(part, now) is None + finally: + await database.dispose() + + asyncio.run(recover()) + command.downgrade(config, "0009") + with sync.connect() as connection: + assert ( + connection.exec_driver_sql("SELECT text FROM channel_outbox").scalar_one() + == "saved answer" + ) + finally: + sync.dispose() diff --git a/tests/agent_manager/test_webhook_migration.py b/tests/agent_manager/test_webhook_migration.py new file mode 100644 index 00000000..fec83694 --- /dev/null +++ b/tests/agent_manager/test_webhook_migration.py @@ -0,0 +1,114 @@ +"""Upgrade/downgrade preserves WA-02 input while adding durable callbacks.""" + +import asyncio +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from alembic import command +from alembic.config import Config +from sqlalchemy import create_engine, inspect, text + +from agent_manager.config import Settings +from agent_manager.domain.channels.models import ChannelAddress, InboundMessage, stable_id +from agent_manager.domain.channels.webhooks import DeliveryEvent, WebhookBatch +from agent_manager.infrastructure.persistence.channels.repository import SqlChannelRepository +from agent_manager.infrastructure.persistence.channels.webhooks import SqlWebhookRepository +from agent_manager.infrastructure.persistence.database import create_db_engine, session_factory + + +def test_webhook_migration_preserves_existing_channel_input( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +): + path = tmp_path / "webhook-migration.db" + url = f"sqlite+aiosqlite:///{path}" + monkeypatch.setattr( + "agent_manager.config.Settings", lambda: Settings.from_values(database_url=url) + ) + migrations = ( + Path(__file__).resolve().parents[2] + / "src/agent_manager/infrastructure/persistence/migrations" + ) + config = Config(str(migrations / "alembic.ini")) + config.set_main_option("script_location", str(migrations)) + command.upgrade(config, "0008") + + async def intake(*, webhook=False, legacy=False): + database = create_db_engine(url) + try: + sessions = session_factory(database) + timestamp = datetime(2026, 10, 15, tzinfo=UTC) + if legacy: + # Freeze the historical fixture at 0008 rather than invoking + # today's adapter before its required migrations exist. + event_id = stable_id("inbox", "connection", "existing") + async with database.begin() as connection: + await connection.execute( + text( + "INSERT INTO channel_bindings " + "(connection_id, system_id, sender_id, generation) " + "VALUES ('connection','system','sender',0) ON CONFLICT DO NOTHING" + ) + ) + await connection.execute( + text( + "INSERT INTO channel_inbox (event_id, connection_id, system_id, " + "sender_id, provider_message_id, kind, content, received_at, " + "sent_at, state) VALUES (:event,'connection','system'," + "'sender','existing','text','hello'," + ":now,:now,'queued') ON CONFLICT DO NOTHING" + ), + {"event": event_id, "now": timestamp.isoformat()}, + ) + return event_id + item = await SqlChannelRepository(sessions).ingest( + InboundMessage( + ChannelAddress("connection", "system", "sender"), + "existing", + "hello", + timestamp, + timestamp, + ) + ) + if webhook: + result = await SqlWebhookRepository(sessions).accept( + "connection", + "system", + "a" * 64, + timestamp, + WebhookBatch( + delivery_events=(DeliveryEvent("outbound", "sender", "sent", timestamp),) + ), + ) + assert len(result.delivery_event_ids) == 1 + return item.event_id + finally: + await database.dispose() + + original = asyncio.run(intake(legacy=True)) + command.upgrade(config, "head") + assert asyncio.run(intake(webhook=True)) == original + sync_database = create_engine(f"sqlite:///{path}") + try: + with sync_database.connect() as connection: + assert ( + connection.exec_driver_sql( + "SELECT count(*) FROM channel_webhook_receipts" + ).scalar_one() + == 1 + ) + assert ( + connection.exec_driver_sql( + "SELECT count(*) FROM channel_delivery_events" + ).scalar_one() + == 1 + ) + finally: + sync_database.dispose() + command.downgrade(config, "0008") + assert asyncio.run(intake(legacy=True)) == original + sync_database = create_engine(f"sqlite:///{path}") + try: + assert "channel_webhook_receipts" not in inspect(sync_database).get_table_names() + finally: + sync_database.dispose() diff --git a/tests/agent_manager/test_webhook_storage.py b/tests/agent_manager/test_webhook_storage.py new file mode 100644 index 00000000..fb19f2fa --- /dev/null +++ b/tests/agent_manager/test_webhook_storage.py @@ -0,0 +1,155 @@ +"""Atomic batch intake, callback deduplication, and durable receipt recovery.""" + +import asyncio +from collections.abc import AsyncIterator +from dataclasses import replace +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from sqlalchemy import event +from sqlalchemy.ext.asyncio import AsyncEngine +from sqlmodel import SQLModel, select + +from agent_manager.domain.channels.models import ChannelAddress, InboundMessage +from agent_manager.domain.channels.webhooks import DeliveryEvent, WebhookBatch, WebhookIssue +from agent_manager.infrastructure.persistence.channels.repository import SqlChannelRepository +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelBindingRow, + ChannelDeliveryEventRow, + ChannelInboxRow, + ChannelWebhookReceiptRow, +) +from agent_manager.infrastructure.persistence.channels.webhooks import SqlWebhookRepository +from agent_manager.infrastructure.persistence.database import create_db_engine, session_factory + +NOW = datetime(2026, 10, 15, 12, tzinfo=UTC) +MESSAGE = InboundMessage( + ChannelAddress("connection", "system", "sender"), "message", "hello", NOW, NOW +) +BATCH = WebhookBatch( + (MESSAGE,), + (DeliveryEvent("outbound", "recipient", "delivered", NOW),), + (WebhookIssue("entry.1", "invalid_entry"),), +) + + +@pytest.fixture +async def storage(tmp_path: Path) -> AsyncIterator[tuple[SqlWebhookRepository, AsyncEngine]]: + database = create_db_engine(f"sqlite+aiosqlite:///{tmp_path / 'receipts.db'}") + async with database.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + try: + yield SqlWebhookRepository(session_factory(database)), database + finally: + await database.dispose() + + +async def rows(database, model): + async with session_factory(database)() as session: + return list((await session.exec(select(model))).all()) + + +async def accept(repository, *, digest="a" * 64, batch=BATCH, connection="connection"): + return await repository.accept(connection, "system", digest, NOW, batch) + + +async def test_concurrent_callbacks_share_one_receipt_and_one_set_of_events(storage): + repository, database = storage + receipts = await asyncio.gather(*(accept(repository) for _ in range(8))) + assert len({receipt.receipt_id for receipt in receipts}) == 1 + assert all(receipt == receipts[0] for receipt in receipts) + for model in ( + ChannelWebhookReceiptRow, + ChannelInboxRow, + ChannelBindingRow, + ChannelDeliveryEventRow, + ): + assert len(await rows(database, model)) == 1 + + +async def test_differently_serialized_callbacks_deduplicate_normalized_messages_and_statuses( + storage, +): + repository, database = storage + first = await accept(repository) + second = await accept(repository, digest="b" * 64) + assert first.receipt_id != second.receipt_id + assert first.event_ids == second.event_ids + assert first.delivery_event_ids == second.delivery_event_ids + assert len(await rows(database, ChannelInboxRow)) == 1 + assert len(await rows(database, ChannelDeliveryEventRow)) == 1 + + +async def test_delivery_write_failure_rolls_back_the_entire_callback(storage): + repository, database = storage + + def fail_delivery(connection, cursor, statement, parameters, context, executemany): + if statement.startswith("INSERT INTO channel_delivery_events"): + raise OSError("injected delivery storage failure") + + event.listen(database.sync_engine, "after_cursor_execute", fail_delivery) + try: + with pytest.raises(OSError): + await accept(repository) + finally: + event.remove(database.sync_engine, "after_cursor_execute", fail_delivery) + for model in ( + ChannelWebhookReceiptRow, + ChannelInboxRow, + ChannelBindingRow, + ChannelDeliveryEventRow, + ): + assert await rows(database, model) == [] + committed = await accept(repository) + assert len(committed.event_ids) == len(committed.delivery_event_ids) == 1 + + +async def test_receipt_and_delivery_identity_survive_reopening(storage): + repository, database = storage + original = await accept(repository) + url = database.url.render_as_string(hide_password=False) + await database.dispose() + reopened = create_db_engine(url) + try: + recovered_repository = SqlWebhookRepository(session_factory(reopened)) + assert await recovered_repository.get_receipt(original.receipt_id) == original + assert await accept(recovered_repository) == original + assert len(await rows(reopened, ChannelDeliveryEventRow)) == 1 + finally: + await reopened.dispose() + + +async def test_separate_connections_never_share_callback_or_message_identity(storage): + repository, database = storage + first = await accept(repository) + other_message = replace(MESSAGE, address=replace(MESSAGE.address, connection_id="other")) + other = await accept( + repository, connection="other", batch=replace(BATCH, messages=(other_message,)) + ) + assert first.receipt_id != other.receipt_id + assert first.event_ids != other.event_ids + assert first.delivery_event_ids != other.delivery_event_ids + assert len(await rows(database, ChannelInboxRow)) == 2 + + +async def test_conflicting_item_keeps_original_and_commits_valid_siblings(storage): + repository, database = storage + original = await accept(repository) + conflict = replace(MESSAGE, address=replace(MESSAGE.address, sender_id="attacker")) + another = replace(MESSAGE, provider_message_id="two") + receipt = await accept(repository, digest="b" * 64, batch=WebhookBatch((conflict, another))) + assert receipt.event_ids != original.event_ids + assert receipt.issues[0].reason == "conflicting_message" + assert [row.sender_id for row in await rows(database, ChannelBindingRow)] == ["sender"] + assert len(await rows(database, ChannelInboxRow)) == 2 + + +async def test_mismatched_batch_scope_is_rejected_without_writes(storage): + repository, database = storage + with pytest.raises(ValueError): + await accept(repository, connection="other") + assert await rows(database, ChannelWebhookReceiptRow) == [] + assert ( + await SqlChannelRepository(session_factory(database)).get_binding(MESSAGE.address) is None + ) From 63033b80dd38a83d5ef2a1810c6f96e593a9dfcc Mon Sep 17 00:00:00 2001 From: Asaf Varon Date: Fri, 16 Oct 2026 15:00:54 +0300 Subject: [PATCH 4/7] feat(manager): connect WhatsApp webhook, worker, and delivery Opt-in authenticated Cloud API intake, a supervised channel turn worker with conservative recovery, provider delivery with text controls, and health reporting for a stopped worker. Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 1 + src/agent_manager/api/app.py | 30 +- src/agent_manager/api/routes/whatsapp.py | 87 +++ .../application/channels/__init__.py | 1 + .../application/channels/controls.py | 41 ++ .../application/channels/delivery.py | 151 ++++ .../application/channels/preparation.py | 146 ++++ .../application/channels/processing.py | 210 ++++++ .../application/channels/worker.py | 191 +++++ .../application/channels/worker_options.py | 29 + src/agent_manager/composition.py | 161 +++++ src/agent_manager/config.py | 85 ++- .../infrastructure/channels/__init__.py | 1 + .../channels/identity_loader.py | 21 + .../channels/whatsapp/__init__.py | 1 + .../channels/whatsapp/config.py | 24 + .../channels/whatsapp/normalization.py | 156 ++++ .../channels/whatsapp/payloads.py | 64 ++ .../channels/whatsapp/rendering.py | 33 + .../channels/whatsapp/security.py | 33 + .../channels/whatsapp/sender.py | 106 +++ .../channels/whatsapp/webhook.py | 28 + tests/agent_manager/test_channel_controls.py | 136 ++++ tests/agent_manager/test_channel_delivery.py | 185 +++++ .../test_channel_delivery_storage.py | 225 ++++++ tests/agent_manager/test_channel_identity.py | 268 +++++++ tests/agent_manager/test_channel_storage.py | 481 +++++++++++++ tests/agent_manager/test_channel_worker.py | 681 ++++++++++++++++++ .../test_channel_worker_lifecycle.py | 236 ++++++ .../test_whatsapp_delivery_lifecycle.py | 256 +++++++ .../test_whatsapp_normalization.py | 163 +++++ .../agent_manager/test_whatsapp_rendering.py | 43 ++ tests/agent_manager/test_whatsapp_sender.py | 108 +++ tests/agent_manager/test_whatsapp_webhook.py | 383 ++++++++++ 34 files changed, 4758 insertions(+), 7 deletions(-) create mode 100644 src/agent_manager/api/routes/whatsapp.py create mode 100644 src/agent_manager/application/channels/__init__.py create mode 100644 src/agent_manager/application/channels/controls.py create mode 100644 src/agent_manager/application/channels/delivery.py create mode 100644 src/agent_manager/application/channels/preparation.py create mode 100644 src/agent_manager/application/channels/processing.py create mode 100644 src/agent_manager/application/channels/worker.py create mode 100644 src/agent_manager/application/channels/worker_options.py create mode 100644 src/agent_manager/infrastructure/channels/__init__.py create mode 100644 src/agent_manager/infrastructure/channels/identity_loader.py create mode 100644 src/agent_manager/infrastructure/channels/whatsapp/__init__.py create mode 100644 src/agent_manager/infrastructure/channels/whatsapp/config.py create mode 100644 src/agent_manager/infrastructure/channels/whatsapp/normalization.py create mode 100644 src/agent_manager/infrastructure/channels/whatsapp/payloads.py create mode 100644 src/agent_manager/infrastructure/channels/whatsapp/rendering.py create mode 100644 src/agent_manager/infrastructure/channels/whatsapp/security.py create mode 100644 src/agent_manager/infrastructure/channels/whatsapp/sender.py create mode 100644 src/agent_manager/infrastructure/channels/whatsapp/webhook.py create mode 100644 tests/agent_manager/test_channel_controls.py create mode 100644 tests/agent_manager/test_channel_delivery.py create mode 100644 tests/agent_manager/test_channel_delivery_storage.py create mode 100644 tests/agent_manager/test_channel_identity.py create mode 100644 tests/agent_manager/test_channel_storage.py create mode 100644 tests/agent_manager/test_channel_worker.py create mode 100644 tests/agent_manager/test_channel_worker_lifecycle.py create mode 100644 tests/agent_manager/test_whatsapp_delivery_lifecycle.py create mode 100644 tests/agent_manager/test_whatsapp_normalization.py create mode 100644 tests/agent_manager/test_whatsapp_rendering.py create mode 100644 tests/agent_manager/test_whatsapp_sender.py create mode 100644 tests/agent_manager/test_whatsapp_webhook.py diff --git a/pyproject.toml b/pyproject.toml index 1507270e..9abbe60d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ dependencies = [ "fastapi>=0.111", "uvicorn>=0.30", "httpx>=0.27", + "regex>=2024.5.15", "jinja2>=3.1", # HITL continuation uses functional tasks and the public runtime accessor. "langgraph>=1.0,<2.0", diff --git a/src/agent_manager/api/app.py b/src/agent_manager/api/app.py index c5481126..8e63feef 100644 --- a/src/agent_manager/api/app.py +++ b/src/agent_manager/api/app.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any -from fastapi import FastAPI +from fastapi import FastAPI, Response from agent_engine.core.validator import SystemSpecValidator from agent_engine.engine.langgraph.engine import LangGraphEngine @@ -20,9 +20,15 @@ from agent_engine.parsers.yaml.parser import YAMLParser from agent_manager.api.deps import CallerIdentity from agent_manager.api.routes import router +from agent_manager.api.routes.whatsapp import router as whatsapp_router from agent_manager.api.web import mount_web from agent_manager.application import ConversationService -from agent_manager.composition import application_repositories, build_identity_resolver +from agent_manager.composition import ( + application_repositories, + build_identity_resolver, + build_whatsapp_webhook, + channel_worker_lifespan, +) from agent_manager.config import Settings from agent_manager.domain import TitleGenerator from agent_manager.infrastructure.titles import ConversationTitler @@ -65,6 +71,9 @@ async def lifespan(app: FastAPI) -> Any: ) as engine, ): await engine.build(spec) + app.state.whatsapp_webhook = build_whatsapp_webhook( + settings, repositories, spec.meta.name + ) service = ConversationService( engine, repositories.conversations, @@ -79,7 +88,11 @@ async def lifespan(app: FastAPI) -> Any: ) app.state.service = service try: - yield + async with channel_worker_lifespan( + settings, repositories, service, spec.meta.name + ) as worker: + app.state.channel_worker = worker + yield finally: await service.close() @@ -90,9 +103,16 @@ async def lifespan(app: FastAPI) -> Any: ) @app.get("/health") - def health() -> dict[str, str]: - return {"status": "ok", "system": getattr(app.state, "system_name", "")} + def health(response: Response) -> dict[str, str]: + system = getattr(app.state, "system_name", "") + worker = getattr(app.state, "channel_worker", None) + if worker is not None and not worker.running: + response.status_code = 503 + return {"status": "degraded", "system": system} + return {"status": "ok", "system": system} app.include_router(router) + if settings.extra_whatsapp_enabled: + app.include_router(whatsapp_router) mount_web(app, settings) return app diff --git a/src/agent_manager/api/routes/whatsapp.py b/src/agent_manager/api/routes/whatsapp.py new file mode 100644 index 00000000..32d21aa2 --- /dev/null +++ b/src/agent_manager/api/routes/whatsapp.py @@ -0,0 +1,87 @@ +"""Bounded provider HTTP edge. Durable intake is the only acknowledgement.""" + +import logging +from datetime import UTC, datetime +from typing import cast + +from fastapi import APIRouter, HTTPException, Request +from fastapi.responses import PlainTextResponse +from starlette.requests import ClientDisconnect + +from agent_manager.infrastructure.channels.whatsapp.security import ( + WebhookAuthenticationError, + subscription_challenge, +) +from agent_manager.infrastructure.channels.whatsapp.webhook import WhatsAppWebhook + +logger = logging.getLogger(__name__) +router = APIRouter(prefix="/channels/whatsapp", tags=["whatsapp"]) + + +def _webhook(request: Request) -> WhatsAppWebhook: + webhook = getattr(request.app.state, "whatsapp_webhook", None) + if webhook is None: + raise HTTPException(status_code=503, detail="webhook intake unavailable") + return cast(WhatsAppWebhook, webhook) + + +@router.get("/webhook", response_class=PlainTextResponse) +async def verify_subscription(request: Request) -> PlainTextResponse: + webhook = _webhook(request) + names = ("hub.mode", "hub.verify_token", "hub.challenge") + if any(len(request.query_params.getlist(name)) != 1 for name in names): + raise HTTPException(status_code=403, detail="invalid webhook subscription") + try: + challenge = subscription_challenge( + request.query_params.get("hub.mode"), + request.query_params.get("hub.verify_token"), + request.query_params.get("hub.challenge"), + expected=webhook.config.verify_token.get_secret_value(), + ) + except WebhookAuthenticationError: + raise HTTPException(status_code=403, detail="invalid webhook subscription") from None + return PlainTextResponse(challenge) + + +@router.post("/webhook") +async def receive_notification(request: Request) -> dict[str, str]: + webhook = _webhook(request) + signatures = request.headers.getlist("X-Hub-Signature-256") + if len(signatures) != 1: + raise HTTPException(status_code=401, detail="invalid webhook signature") + if request.headers.get("content-encoding", "identity") != "identity": + raise HTTPException(status_code=415, detail="encoded webhook bodies are unsupported") + if request.headers.get("content-type", "").split(";", 1)[0].lower() != "application/json": + raise HTTPException(status_code=415, detail="webhook requires application/json") + body = await _bounded_body(request, webhook.config.max_body_bytes) + try: + await webhook.receive(body, signatures[0], datetime.now(UTC)) + except WebhookAuthenticationError: + raise HTTPException(status_code=401, detail="invalid webhook signature") from None + except Exception as exc: + # The failure boundary must never reflect payloads, identifiers, or DB + # exception text into provider responses or ordinary logs. + logger.warning("webhook intake failed: type=%s", type(exc).__name__) + raise HTTPException( + status_code=503, detail="webhook storage unavailable", headers={"Retry-After": "1"} + ) from None + return {"status": "ok"} + + +async def _bounded_body(request: Request, limit: int) -> bytes: + lengths = request.headers.getlist("content-length") + if len(lengths) > 1 or ( + lengths and (len(lengths[0]) > 20 or not lengths[0].isascii() or not lengths[0].isdecimal()) + ): + raise HTTPException(status_code=400, detail="invalid content length") + if lengths and int(lengths[0]) > limit: + raise HTTPException(status_code=413, detail="webhook body too large") + body = bytearray() + try: + async for chunk in request.stream(): + if len(body) + len(chunk) > limit: + raise HTTPException(status_code=413, detail="webhook body too large") + body.extend(chunk) + except ClientDisconnect: + raise HTTPException(status_code=400, detail="incomplete webhook body") from None + return bytes(body) diff --git a/src/agent_manager/application/channels/__init__.py b/src/agent_manager/application/channels/__init__.py new file mode 100644 index 00000000..f3005507 --- /dev/null +++ b/src/agent_manager/application/channels/__init__.py @@ -0,0 +1 @@ +"""Channel use cases layered over the existing conversation service.""" diff --git a/src/agent_manager/application/channels/controls.py b/src/agent_manager/application/channels/controls.py new file mode 100644 index 00000000..785f3027 --- /dev/null +++ b/src/agent_manager/application/channels/controls.py @@ -0,0 +1,41 @@ +"""Exact normalized controls and static feedback; never an LLM tool.""" + +from agent_manager.domain.channels.controls import ControlRepository +from agent_manager.domain.channels.models import InboxItem, InboxState, bounded_text +from agent_manager.domain.channels.repository import ChannelRepository + + +class ChannelControlService: + def __init__( + self, controls: ControlRepository, channels: ChannelRepository, *, support_contact: str + ) -> None: + bounded_text(support_contact) + if not support_contact.strip() or len(support_contact.encode()) > 512: + raise ValueError("a bounded support contact is required") + self._controls, self._channels, self._support = controls, channels, support_contact + + async def handle(self, item: InboxItem) -> bool: + suppressed = await self._controls.suppressed(item) + if item.message.kind == "text" and not suppressed: + return False + if item.state is not InboxState.PREPARING: + if ( + item.state in {InboxState.PREPARED, InboxState.RUNNING} + and suppressed + and item.claim_token + ): + await self._channels.finish( + item.event_id, item.claim_token, InboxState.NEEDS_REVIEW + ) + return suppressed + replies = { + "reset": "נפתחה שיחה חדשה. אפשר לשלוח הודעת טקסט.", + "help": f"לתמיכה אנושית: {self._support}", + "start": "קבלת ההודעות חודשה. אפשר לשלוח הודעת טקסט.", + "unsupported": "כרגע ניתן לקבל הודעות טקסט בלבד. אנא שלחו את הבקשה בטקסט.", + } + reply = ( + None if suppressed or item.message.kind == "stop" else replies.get(item.message.kind) + ) + await self._controls.complete(item, reply, reset=item.message.kind == "reset") + return True diff --git a/src/agent_manager/application/channels/delivery.py b/src/agent_manager/application/channels/delivery.py new file mode 100644 index 00000000..ffe60d07 --- /dev/null +++ b/src/agent_manager/application/channels/delivery.py @@ -0,0 +1,151 @@ +"""One bounded outbound pump, owned by the connection's supervisor.""" + +import asyncio +import logging +import random +from collections.abc import Callable +from datetime import UTC, datetime, timedelta + +from agent_manager.domain.channels.delivery import ( + ChannelSender, + DeliveryPart, + DeliveryRepository, + SendOutcome, +) +from agent_manager.domain.channels.identity import ChannelIdentityResolver +from agent_manager.domain.identity import Principal + +logger = logging.getLogger(__name__) + + +class ChannelDeliveryService: + def __init__( + self, + repository: DeliveryRepository, + identities: ChannelIdentityResolver, + sender: ChannelSender, + *, + connection_id: str, + system_id: str, + organization_id: str, + render: Callable[[str], tuple[str, ...]], + clock: Callable[[], datetime] = lambda: datetime.now(UTC), + jitter: Callable[[], float] = random.random, + max_attempts: int = 4, + ) -> None: + if not 1 <= max_attempts <= 8: + raise ValueError("send attempt bound must be between 1 and 8") + self._repository, self._identities, self._sender = repository, identities, sender + self._connection, self._system, self._organization = ( + connection_id, + system_id, + organization_id, + ) + self._render, self._clock, self._jitter = render, clock, jitter + self._max_attempts = max_attempts + + async def tick(self, stop: asyncio.Event) -> None: + if stop.is_set(): + return + # Only the exclusive supervisor calls tick, with no prior send in flight. + await self._repository.recover_sending(self._connection, self._system) + await self._repository.project(self._connection, self._system) + for source in await self._repository.sources(self._connection, self._system): + try: + parts: list[str] = [] + for text in source.texts: + parts.extend(self._render(text)) + if len(parts) > 128: + raise ValueError("reply has too many provider parts") + except ValueError: + parts = [] + await self._repository.materialize(source.event_id, tuple(parts)) + if stop.is_set(): + return + part = await self._repository.claim_next(self._connection, self._system, self._clock()) + if part is None: + return + submitted = False + try: + guard = await self._repository.guard(part, self._clock()) + if guard is not None: + await self._repository.complete(part, guard, reason=guard) + return + if part.user_id is not None and not await self._authorized(part): + await self._repository.complete( + part, "blocked_identity", reason="identity_unavailable" + ) + return + # Identity lookup may await an external service. Recheck live state. + guard = await self._repository.guard(part, self._clock()) + if guard is not None: + await self._repository.complete(part, guard, reason=guard) + return + if stop.is_set(): + await self._repository.complete( + part, "retry", next_attempt_at=self._clock(), reason="stopped_before_send" + ) + return + submitted = True + try: + async with asyncio.timeout(20): + outcome = await self._sender.send(part.address.sender_id, part.text) + except asyncio.CancelledError: + raise + except Exception as exc: + logger.warning("channel provider attempt failed: type=%s", type(exc).__name__) + outcome = SendOutcome("unknown") + if outcome.kind == "retry" and part.attempts < self._max_attempts: + delay = min( + 300, max(outcome.retry_after, 2**part.attempts * (0.5 + self._jitter())) + ) + await self._repository.complete( + part, + "retry", + next_attempt_at=self._clock() + timedelta(seconds=delay), + reason="transient_rejection", + ) + else: + state = { + "accepted": "accepted", + "retry": "failed", + "failed": "failed", + "unknown": "delivery_unknown", + }[outcome.kind] + await self._repository.complete( + part, + state, + provider_id=outcome.provider_message_id, + reason=None if state == "accepted" else "send_" + state, + ) + except asyncio.CancelledError: + state = "delivery_unknown" if submitted else "retry" + cleanup = asyncio.create_task( + self._repository.complete( + part, + state, + next_attempt_at=None if submitted else self._clock(), + reason="interrupted_send" if submitted else "stopped_before_send", + ) + ) + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + continue + cleanup.result() + raise + + async def _authorized(self, part: DeliveryPart) -> bool: + try: + async with asyncio.timeout(5): + principal = await self._identities.resolve(part.address) + return ( + isinstance(principal, Principal) + and not principal.is_anonymous + and principal.user_id == part.user_id + and principal.organization_id == part.organization_id == self._organization + ) + except Exception as exc: + logger.warning("channel delivery identity unavailable: type=%s", type(exc).__name__) + return False diff --git a/src/agent_manager/application/channels/preparation.py b/src/agent_manager/application/channels/preparation.py new file mode 100644 index 00000000..208fc2b5 --- /dev/null +++ b/src/agent_manager/application/channels/preparation.py @@ -0,0 +1,146 @@ +"""Prepare an internal turn only after the client proves the sender's identity.""" + +from __future__ import annotations + +import logging +from typing import NoReturn + +from agent_manager.application.conversation_service import ConversationService +from agent_manager.application.prepared_conversation_turn import PreparedConversationTurn +from agent_manager.domain.channels.identity import ( + ChannelIdentityDenied, + ChannelIdentityResolver, + ChannelIdentityUnavailable, +) +from agent_manager.domain.channels.models import ( + ChannelClaimConflict, + ChannelOwnerConflict, + InboxItem, + InboxState, + bounded_identifier, + stable_id, +) +from agent_manager.domain.channels.repository import ChannelRepository +from agent_manager.domain.identity import Principal + +logger = logging.getLogger(__name__) + + +class ChannelTurnPreparer: + def __init__( + self, + service: ConversationService, + repository: ChannelRepository, + identity_resolver: ChannelIdentityResolver, + *, + connection_id: str, + system_id: str, + organization_id: str, + ) -> None: + bounded_identifier(connection_id, limit=128) + bounded_identifier(system_id, limit=256) + bounded_identifier(organization_id) + self._service = service + self._repository = repository + self._identities = identity_resolver + self._connection_id = connection_id + self._system_id = system_id + self._organization_id = organization_id + + async def prepare(self, event_id: str, token: str) -> PreparedConversationTurn: + item = await self._repository.get_inbound(event_id) + if ( + item is None + or not token + or item.claim_token != token + or item.state not in {InboxState.PREPARING, InboxState.PREPARED} + or item.message.address.connection_id != self._connection_id + or item.message.address.system_id != self._system_id + ): + raise ChannelClaimConflict("invalid channel preparation claim or scope") + if item.message.kind != "text": + raise ChannelClaimConflict("only text messages can prepare an agent turn") + principal = await self.authorize(event_id, token) + binding = await self._repository.get_binding(item.message.address) + if binding is None: + raise ChannelClaimConflict("channel message has no sender binding") + conversation_id = binding.conversation_id or stable_id( + "channel-conversation", + self._connection_id, + self._system_id, + item.message.address.sender_id, + str(binding.generation), + ) + # A crash after create but before binding reuses this same id. Ownership + # checks in ConversationService still reject another principal. + await self._service.create(principal, session_id=conversation_id) + try: + await self._repository.bind_conversation( + event_id, token, principal.user_id, self._organization_id, conversation_id + ) + except ChannelOwnerConflict: + await self._deny(item, token) + turn = await self._service.prepare_turn( + conversation_id, + item.message.content, + principal, + idempotency_key=f"channel:{item.event_id}", + ) + await self._repository.record_prepared( + event_id, token, turn.session_id, turn.run_id, turn.message_id + ) + return turn + + async def authorize(self, event_id: str, token: str) -> Principal: + """Refresh the original sender's identity before execution or answer recovery.""" + item = await self._repository.get_inbound(event_id) + if ( + item is None + or not token + or item.claim_token != token + or item.state not in {InboxState.PREPARING, InboxState.PREPARED, InboxState.RUNNING} + or item.message.address.connection_id != self._connection_id + or item.message.address.system_id != self._system_id + ): + raise ChannelClaimConflict("invalid channel authorization claim or scope") + principal = await self._principal(item, token) + binding = await self._repository.get_binding(item.message.address) + if binding is None or binding.active_event_id != event_id: + raise ChannelClaimConflict("channel authorization requires the active sender claim") + if (binding.user_id is not None and binding.user_id != principal.user_id) or ( + binding.organization_id is not None and binding.organization_id != self._organization_id + ): + await self._deny(item, token) + if item.conversation_id is not None and binding.conversation_id != item.conversation_id: + raise ChannelClaimConflict("channel conversation binding changed") + return principal + + async def _principal(self, item: InboxItem, token: str) -> Principal: + try: + principal = await self._identities.resolve(item.message.address) + except Exception as exc: + logger.warning( + "channel identity lookup failed: event=%s type=%s", + item.event_id, + type(exc).__name__, + ) + await self._repository.finish(item.event_id, token, InboxState.NEEDS_REVIEW) + raise ChannelIdentityUnavailable("channel identity could not be verified") from None + if principal is not None and not isinstance(principal, Principal): + await self._repository.finish(item.event_id, token, InboxState.NEEDS_REVIEW) + raise ChannelIdentityUnavailable("identity resolver returned an invalid principal") + if ( + principal is None + or principal.is_anonymous + or principal.organization_id != self._organization_id + ): + await self._deny(item, token) + assert principal is not None + return principal + + async def _deny(self, item: InboxItem, token: str) -> NoReturn: + # A prepared manager message already exists and needs reconciliation; + # it must not be silently abandoned as though it had never been made. + state = InboxState.DENIED if item.state is InboxState.PREPARING else InboxState.NEEDS_REVIEW + await self._repository.finish(item.event_id, token, state) + raise ChannelIdentityDenied("sender has no authorized organization account link") from None diff --git a/src/agent_manager/application/channels/processing.py b/src/agent_manager/application/channels/processing.py new file mode 100644 index 00000000..f1c748b0 --- /dev/null +++ b/src/agent_manager/application/channels/processing.py @@ -0,0 +1,210 @@ +"""Execute one owned turn, or recover only its persisted answer.""" + +from __future__ import annotations + +import asyncio +import logging +from collections.abc import AsyncGenerator +from contextlib import aclosing +from dataclasses import replace +from typing import cast + +from agent_engine.approvals.models import RunStatus +from agent_engine.runs.repository import RunRepository +from agent_engine.runtime.streaming import RunStreamEvent +from agent_manager.application.channels.controls import ChannelControlService +from agent_manager.application.channels.preparation import ChannelTurnPreparer +from agent_manager.application.channels.worker_options import ChannelWorkerOptions +from agent_manager.application.conversation_service import ConversationService +from agent_manager.application.errors import ( + ConversationAccessDenied, + ConversationBranchConflict, + ConversationTokenBudgetExceeded, +) +from agent_manager.application.prepared_conversation_turn import PreparedConversationTurn +from agent_manager.domain.channels.identity import ChannelIdentityDenied, ChannelIdentityUnavailable +from agent_manager.domain.channels.models import ( + ChannelClaimConflict, + InboxItem, + InboxState, + bounded_text, +) +from agent_manager.domain.channels.repository import ChannelRepository + +logger = logging.getLogger(__name__) +_ACTIVE = {InboxState.PREPARING, InboxState.PREPARED, InboxState.RUNNING} + + +class ChannelTurnProcessor: + def __init__( + self, + service: ConversationService, + repository: ChannelRepository, + runs: RunRepository, + preparer: ChannelTurnPreparer, + options: ChannelWorkerOptions, + *, + controls: ChannelControlService | None = None, + ) -> None: + self._service = service + self._repository = repository + self._runs = runs + self._preparer = preparer + self._options = options + self._controls = controls + + async def process(self, item: InboxItem) -> None: + try: + async with asyncio.timeout(self._options.turn_seconds): + current = await self._repository.get_inbound(item.event_id) + if ( + current is None + or current.state not in _ACTIVE + or current.claim_token != item.claim_token + or not current.claim_token + ): + return + await self._process(current) + except (ChannelIdentityDenied, ChannelIdentityUnavailable, ChannelClaimConflict): + return + except ( + ValueError, + ConversationAccessDenied, + ConversationBranchConflict, + ConversationTokenBudgetExceeded, + ): + await self._review(item, "invalid_preparation") + except TimeoutError: + logger.warning("channel turn timed out: event=%s", item.event_id) + except Exception as exc: + # Claims remain durable. A later scan can retry preparation or + # saved-answer staging; it can never repeat a running execution. + logger.warning( + "channel work failed: event=%s type=%s", item.event_id, type(exc).__name__ + ) + + async def _process(self, item: InboxItem) -> None: + assert item.claim_token is not None + if self._controls is not None and await self._controls.handle(item): + return + if item.message.kind != "text": + if item.state is InboxState.PREPARING: + await self._repository.finish(item.event_id, item.claim_token, InboxState.FAILED) + else: + await self._review(item, "unsupported_work_state") + return + if item.state is InboxState.RUNNING: + await self._stage_saved(item) + return + turn = await self._preparer.prepare(item.event_id, item.claim_token) + principal = await self._preparer.authorize(item.event_id, item.claim_token) + turn = replace(turn, principal=principal) + response = await self._service.get_turn_response(turn.session_id, turn.run_id, principal) + run = await self._runs.get(turn.run_id) + if response is None and (run is None or run.status is not RunStatus.RUNNING): + await self._review(item, "terminal_run_without_response") + return + if not await self._repository.claim_execution(item.event_id, item.claim_token): + return + current = await self._repository.get_inbound(item.event_id) + assert current is not None + if response is not None: + await self._stage_saved(current) + return + approval = False + cancellation_confirmed = False + saw_final = False + try: + async with aclosing( + cast(AsyncGenerator[RunStreamEvent, None], self._service.stream_turn(turn)) + ) as stream: + async for event in stream: + if event.type == "pending_approval": + approval = True + cancellation_confirmed = await self._cancel_approval( + current, turn, event.approval_id + ) + break + saw_final = saw_final or event.type == "final" + except asyncio.CancelledError: + await self._service.cancel_turn(turn) + await self._review(current, "interrupted_execution") + raise + except Exception as exc: + await self._service.fail_turn(turn) + logger.warning( + "channel execution failed: event=%s type=%s", item.event_id, type(exc).__name__ + ) + if approval: + await self._review(current, "approval_stream_cleanup_failed") + return + if approval: + # Cancellation alone is insufficient: keep the sender claimed until + # the engine iterator has closed and awaited its owned tasks. + if cancellation_confirmed: + await self._repository.finish(current.event_id, item.claim_token, InboxState.FAILED) + return + if not saw_final: + await self._service.fail_turn(turn) + await self._stage_saved(current) + + async def _stage_saved(self, item: InboxItem) -> None: + assert item.claim_token is not None + principal = await self._preparer.authorize(item.event_id, item.claim_token) + if item.conversation_id is None or item.run_id is None: + await self._review(item, "missing_turn_binding") + return + response = await self._service.get_turn_response( + item.conversation_id, item.run_id, principal + ) + if response is None: + await self._review(item, "execution_without_response") + return + if response.parent_message_id != item.message_id: + await self._review(item, "response_binding_mismatch") + return + bounded_text(response.content) + if not response.content.strip(): + await self._review(item, "empty_response") + return + await self._repository.stage_reply( + item.event_id, item.claim_token, response.message_id, (response.content,) + ) + + async def _cancel_approval( + self, item: InboxItem, turn: PreparedConversationTurn, approval_id: str | None + ) -> bool: + assert item.claim_token is not None + if approval_id is None: + await self._review(item, "missing_approval_binding") + return False + try: + await self._service.cancel_pending_approval( + turn.session_id, turn.run_id, approval_id, turn.principal + ) + record = await self._runs.get(turn.run_id) + if record is not None and record.status is RunStatus.CANCELLED: + return True + except Exception as exc: + logger.warning( + "channel approval cancellation failed: event=%s type=%s", + item.event_id, + type(exc).__name__, + ) + await self._review(item, "unconfirmed_approval_cancellation") + return False + + async def _review(self, item: InboxItem, reason: str) -> None: + current = await self._repository.get_inbound(item.event_id) + if ( + current is not None + and current.state in _ACTIVE + and current.claim_token == item.claim_token + and current.claim_token + ): + await self._repository.finish( + current.event_id, current.claim_token, InboxState.NEEDS_REVIEW + ) + logger.warning( + "channel work requires review: event=%s reason=%s", item.event_id, reason + ) diff --git a/src/agent_manager/application/channels/worker.py b/src/agent_manager/application/channels/worker.py new file mode 100644 index 00000000..002f2d60 --- /dev/null +++ b/src/agent_manager/application/channels/worker.py @@ -0,0 +1,191 @@ +"""One supervised connection owner with bounded independent-sender jobs.""" + +from __future__ import annotations + +import asyncio +import logging +from collections import deque +from contextlib import AsyncExitStack +from types import TracebackType + +from agent_manager.application.channels.delivery import ChannelDeliveryService +from agent_manager.application.channels.processing import ChannelTurnProcessor +from agent_manager.application.channels.worker_options import ChannelWorkerOptions +from agent_manager.domain.channels.models import InboxItem +from agent_manager.domain.channels.ownership import ( + ChannelWorkerOwnership, + ChannelWorkerOwnershipLost, +) +from agent_manager.domain.channels.repository import ChannelRepository + +logger = logging.getLogger(__name__) + + +class ChannelWorker: + def __init__( + self, + repository: ChannelRepository, + processor: ChannelTurnProcessor, + ownership: ChannelWorkerOwnership, + options: ChannelWorkerOptions, + *, + delivery: ChannelDeliveryService | None = None, + ) -> None: + self._repository = repository + self._processor = processor + self._ownership = ownership + self._options = options + self._delivery = delivery + self._stack = AsyncExitStack() + self._stop = asyncio.Event() + self._jobs: dict[str, asyncio.Task[None]] = {} + self._pending: deque[InboxItem] = deque() + self._cursor = 0 + self._driver: asyncio.Task[None] | None = None + + @property + def running(self) -> bool: + return self._driver is not None and not self._driver.done() and not self._stop.is_set() + + async def __aenter__(self) -> ChannelWorker: + if self._driver is not None: + raise RuntimeError("channel worker cannot be started twice") + try: + await self._stack.enter_async_context(self._ownership.hold()) + await self._ownership.check() + self._driver = asyncio.create_task(self._serve()) + self._driver.add_done_callback(self._observe_failure) + except BaseException: + await self._stack.aclose() + raise + return self + + async def __aexit__( + self, + exc_type: type[BaseException] | None, + exc: BaseException | None, + traceback: TracebackType | None, + ) -> None: + self._stop.set() + shutdown = asyncio.create_task(self._shutdown()) + try: + await asyncio.shield(shutdown) + except asyncio.CancelledError: + # The lifespan caller can itself be cancelled during shutdown. + # Keep the complete drain/ownership scope alive, even through + # repeated cancellation, then propagate the caller's cancellation. + while not shutdown.done(): + try: + await asyncio.shield(shutdown) + except asyncio.CancelledError: + continue + shutdown.result() + raise + + async def _shutdown(self) -> None: + try: + if self._driver is not None: + try: + await asyncio.wait_for( + asyncio.shield(self._driver), self._options.drain_seconds + 1 + ) + except TimeoutError: + self._driver.cancel() + await asyncio.gather(self._driver, return_exceptions=True) + finally: + await self._stack.aclose() + + async def _serve(self) -> None: + failed = False + try: + while not self._stop.is_set(): + await self._ownership.check() + self._reap() + try: + await self._schedule() + if self._delivery is not None and not self._stop.is_set(): + await self._ownership.check() + await self._delivery.tick(self._stop) + except ChannelWorkerOwnershipLost: + raise + except Exception as exc: + logger.warning("channel scheduling failed: type=%s", type(exc).__name__) + stop_waiter = asyncio.create_task(self._stop.wait()) + try: + await asyncio.wait( + (stop_waiter, *self._jobs.values()), + timeout=self._options.poll_seconds, + return_when=asyncio.FIRST_COMPLETED, + ) + finally: + stop_waiter.cancel() + await asyncio.gather(stop_waiter, return_exceptions=True) + except BaseException: + self._stop.set() + failed = True + raise + finally: + cleanup = asyncio.create_task(self._drain(immediate=failed)) + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + # A supervisor timeout must not interrupt the engine's own + # cancellation cleanup and release ownership ahead of it. + await cleanup + raise + + async def _schedule(self) -> None: + scanned = False + while not self._stop.is_set() and len(self._jobs) < self._options.concurrency: + if self._pending: + item = self._pending.popleft() + elif not scanned: + page = await self._repository.list_active( + self._options.connection_id, + self._options.system_id, + after_sequence=self._cursor, + ) + self._cursor = page[-1].sequence if page else 0 + self._pending.extend(page) + scanned = True + continue + else: + claimed = await self._repository.claim_next( + self._options.connection_id, self._options.system_id + ) + if claimed is None: + return + item = claimed + if self._stop.is_set(): + # An in-flight SQL claim may commit after shutdown started. + # Keep it preparing for the next owner without starting work. + return + if item.event_id not in self._jobs: + self._jobs[item.event_id] = asyncio.create_task(self._processor.process(item)) + + def _reap(self) -> None: + for event_id, task in tuple(self._jobs.items()): + if task.done(): + if not task.cancelled() and task.exception() is not None: + logger.warning( + "channel task failed: event=%s type=%s", + event_id, + type(task.exception()).__name__, + ) + del self._jobs[event_id] + + async def _drain(self, *, immediate: bool) -> None: + tasks = tuple(self._jobs.values()) + if tasks: + _, pending = await asyncio.wait( + tasks, timeout=0 if immediate else self._options.drain_seconds + ) + for task in pending: + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + self._jobs.clear() + + @staticmethod + def _observe_failure(task: asyncio.Task[None]) -> None: + if not task.cancelled() and task.exception() is not None: + logger.error("channel supervisor stopped: type=%s", type(task.exception()).__name__) diff --git a/src/agent_manager/application/channels/worker_options.py b/src/agent_manager/application/channels/worker_options.py new file mode 100644 index 00000000..fafd3f26 --- /dev/null +++ b/src/agent_manager/application/channels/worker_options.py @@ -0,0 +1,29 @@ +"""Bounded application-owned worker configuration, independent of provider payloads.""" + +from dataclasses import dataclass + +from agent_manager.domain.channels.models import bounded_identifier + + +@dataclass(frozen=True) +class ChannelWorkerOptions: + connection_id: str + system_id: str + organization_id: str + concurrency: int = 4 + poll_seconds: float = 0.5 + turn_seconds: float = 120 + drain_seconds: float = 15 + + def __post_init__(self) -> None: + bounded_identifier(self.connection_id, limit=128) + bounded_identifier(self.system_id, limit=256) + bounded_identifier(self.organization_id) + if not 1 <= self.concurrency <= 16: + raise ValueError("channel worker concurrency must be between 1 and 16") + if not ( + 0 < self.poll_seconds <= 60 + and 0 < self.turn_seconds <= 3600 + and 0 < self.drain_seconds <= 60 + ): + raise ValueError("channel worker time bounds must be finite and positive") diff --git a/src/agent_manager/composition.py b/src/agent_manager/composition.py index a0c9720c..4ffda5d2 100644 --- a/src/agent_manager/composition.py +++ b/src/agent_manager/composition.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import hashlib import hmac import logging @@ -18,8 +19,20 @@ from agent_engine.runs.repository import RunRepository from agent_engine.tool_usage.in_memory import InMemoryToolUsageRepository from agent_engine.tool_usage.repository import ToolUsageRepository +from agent_manager.application.channels.controls import ChannelControlService +from agent_manager.application.channels.delivery import ChannelDeliveryService +from agent_manager.application.channels.preparation import ChannelTurnPreparer +from agent_manager.application.channels.processing import ChannelTurnProcessor +from agent_manager.application.channels.worker import ChannelWorker +from agent_manager.application.channels.worker_options import ChannelWorkerOptions +from agent_manager.application.conversation_service import ConversationService from agent_manager.config import AuthMode, Settings from agent_manager.domain import Repository +from agent_manager.domain.channels.controls import ControlRepository +from agent_manager.domain.channels.delivery import DeliveryRepository +from agent_manager.domain.channels.identity import ChannelIdentityResolver +from agent_manager.domain.channels.repository import ChannelRepository +from agent_manager.domain.channels.webhooks import WebhookRepository from agent_manager.infrastructure.auth.anonymous_identity_source import AnonymousIdentitySource from agent_manager.infrastructure.auth.claim_mapping import ClaimMapping from agent_manager.infrastructure.auth.host_identity_source import HostIdentitySource @@ -28,6 +41,16 @@ from agent_manager.infrastructure.auth.static_secret_key_source import StaticSecretKeySource from agent_manager.infrastructure.auth.token_policy import TokenPolicy from agent_manager.infrastructure.auth.token_verifier import TokenVerifier +from agent_manager.infrastructure.channels.identity_loader import load_channel_identity_resolver +from agent_manager.infrastructure.channels.whatsapp.config import WhatsAppWebhookConfig +from agent_manager.infrastructure.channels.whatsapp.rendering import render_text +from agent_manager.infrastructure.channels.whatsapp.sender import WhatsAppSender +from agent_manager.infrastructure.channels.whatsapp.webhook import WhatsAppWebhook +from agent_manager.infrastructure.persistence.channels.controls import SqlControlRepository +from agent_manager.infrastructure.persistence.channels.delivery import SqlDeliveryRepository +from agent_manager.infrastructure.persistence.channels.ownership import SqlChannelWorkerOwnership +from agent_manager.infrastructure.persistence.channels.repository import SqlChannelRepository +from agent_manager.infrastructure.persistence.channels.webhooks import SqlWebhookRepository from agent_manager.infrastructure.persistence.database import create_db_engine, session_factory from agent_manager.infrastructure.persistence.memory_repository import MemoryRepository from agent_manager.infrastructure.persistence.run_repository import SqlRunRepository @@ -45,6 +68,140 @@ class ApplicationRepositories: session_approvals: SessionApprovalRepository tool_usage: ToolUsageRepository runs: RunRepository + channels: ChannelRepository | None = None + webhooks: WebhookRepository | None = None + deliveries: DeliveryRepository | None = None + controls: ControlRepository | None = None + + +def build_whatsapp_webhook( + settings: Settings, + repositories: ApplicationRepositories, + system_id: str, +) -> WhatsAppWebhook | None: + if not settings.extra_whatsapp_enabled: + return None + if repositories.webhooks is None: + raise ValueError("WhatsApp intake requires a durable webhook repository") + assert settings.extra_whatsapp_account_id is not None + assert settings.extra_whatsapp_phone_number_id is not None + assert settings.extra_whatsapp_app_secret is not None + assert settings.extra_whatsapp_verify_token is not None + return WhatsAppWebhook( + WhatsAppWebhookConfig( + connection_id=settings.extra_whatsapp_connection_id, + system_id=system_id, + account_id=settings.extra_whatsapp_account_id, + phone_number_id=settings.extra_whatsapp_phone_number_id, + app_secret=settings.extra_whatsapp_app_secret, + verify_token=settings.extra_whatsapp_verify_token, + max_body_bytes=settings.extra_whatsapp_max_body_bytes, + ), + repositories.webhooks, + ) + + +@asynccontextmanager +async def channel_worker_lifespan( + settings: Settings, + repositories: ApplicationRepositories, + service: ConversationService, + system_id: str, +) -> AsyncIterator[ChannelWorker | None]: + if not settings.extra_whatsapp_worker_enabled: + yield None + return + if repositories.channels is None: + raise ValueError("channel worker requires durable storage") + assert settings.extra_whatsapp_organization_id is not None + assert settings.extra_whatsapp_identity_resolver is not None + options = ChannelWorkerOptions( + connection_id=settings.extra_whatsapp_connection_id, + system_id=system_id, + organization_id=settings.extra_whatsapp_organization_id, + concurrency=settings.extra_whatsapp_worker_concurrency, + poll_seconds=settings.extra_whatsapp_worker_poll_seconds, + turn_seconds=settings.extra_whatsapp_worker_turn_seconds, + drain_seconds=settings.extra_whatsapp_worker_drain_seconds, + ) + identities = load_channel_identity_resolver(settings.extra_whatsapp_identity_resolver) + sender: WhatsAppSender | None = None + try: + delivery = None + controls = None + if settings.extra_whatsapp_sender_enabled: + assert repositories.deliveries is not None and repositories.controls is not None + assert settings.extra_whatsapp_api_version is not None + assert settings.extra_whatsapp_phone_number_id is not None + assert settings.extra_whatsapp_access_token is not None + assert settings.extra_whatsapp_support_contact is not None + sender = WhatsAppSender( + settings.extra_whatsapp_api_version, + settings.extra_whatsapp_phone_number_id, + settings.extra_whatsapp_access_token, + ) + delivery = ChannelDeliveryService( + repositories.deliveries, + identities, + sender, + connection_id=options.connection_id, + system_id=system_id, + organization_id=options.organization_id, + render=render_text, + ) + controls = ChannelControlService( + repositories.controls, + repositories.channels, + support_contact=settings.extra_whatsapp_support_contact, + ) + preparer = ChannelTurnPreparer( + service, + repositories.channels, + identities, + connection_id=options.connection_id, + system_id=system_id, + organization_id=options.organization_id, + ) + processor = ChannelTurnProcessor( + service, repositories.channels, repositories.runs, preparer, options, controls=controls + ) + ownership = SqlChannelWorkerOwnership( + settings.effective_database_url, options.connection_id + ) + async with ChannelWorker( + repositories.channels, processor, ownership, options, delivery=delivery + ) as worker: + yield worker + finally: + cleanup = asyncio.create_task(_close_channel_clients(sender, identities)) + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + # A cancelled lifespan must still join client cleanup before its + # repositories are disposed, including repeated caller cancellation. + while not cleanup.done(): + try: + await asyncio.shield(cleanup) + except asyncio.CancelledError: + continue + cleanup.result() + raise + + +async def _close_channel_clients( + sender: WhatsAppSender | None, identities: ChannelIdentityResolver +) -> None: + try: + if sender is not None: + try: + await sender.close() + except Exception as exc: + logger.warning("channel sender cleanup failed: type=%s", type(exc).__name__) + finally: + try: + await identities.close() + except Exception as exc: + logger.warning("channel identity cleanup failed: type=%s", type(exc).__name__) def build_session_approval_repository() -> SessionApprovalRepository: @@ -144,6 +301,10 @@ async def application_repositories( session_approvals=session_approvals, tool_usage=tool_usage, runs=SqlRunRepository(sessions), + channels=SqlChannelRepository(sessions), + webhooks=SqlWebhookRepository(sessions), + deliveries=SqlDeliveryRepository(sessions), + controls=SqlControlRepository(sessions), ) finally: await db_engine.dispose() diff --git a/src/agent_manager/config.py b/src/agent_manager/config.py index 4d493251..98ccb451 100644 --- a/src/agent_manager/config.py +++ b/src/agent_manager/config.py @@ -9,7 +9,7 @@ from typing import Annotated, Any, Literal, NamedTuple from urllib.parse import parse_qs, urlsplit -from pydantic import field_validator, model_validator +from pydantic import Field, SecretStr, field_validator, model_validator from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict from sqlalchemy.engine import make_url from sqlalchemy.exc import ArgumentError @@ -127,7 +127,7 @@ def _is_in_memory_sqlite(url: str) -> bool: class Settings(BaseSettings): - model_config = SettingsConfigDict(env_file=".env", extra="ignore") + model_config = SettingsConfigDict(env_file=".env", extra="ignore", hide_input_in_errors=True) extra_db_backend: Literal["sqlite", "postgres"] = "sqlite" extra_db_url: str | None = None @@ -160,6 +160,27 @@ class Settings(BaseSettings): extra_auth_claim_roles: str = "roles" extra_auth_claim_organization_id: str = "org_id" + extra_whatsapp_enabled: bool = False + extra_whatsapp_connection_id: str = "whatsapp" + extra_whatsapp_account_id: str | None = None + extra_whatsapp_phone_number_id: str | None = None + extra_whatsapp_app_secret: SecretStr | None = Field(default=None, repr=False, exclude=True) + extra_whatsapp_verify_token: SecretStr | None = Field(default=None, repr=False, exclude=True) + extra_whatsapp_max_body_bytes: int = Field(default=262_144, ge=1024, le=1_048_576) + extra_whatsapp_worker_enabled: bool = False + extra_whatsapp_organization_id: str | None = None + extra_whatsapp_identity_resolver: str | None = None + extra_whatsapp_worker_concurrency: int = Field(default=4, ge=1, le=16) + extra_whatsapp_worker_poll_seconds: float = Field(default=0.5, gt=0, le=60, allow_inf_nan=False) + extra_whatsapp_worker_turn_seconds: float = Field( + default=120, gt=0, le=3600, allow_inf_nan=False + ) + extra_whatsapp_worker_drain_seconds: float = Field(default=15, gt=0, le=60, allow_inf_nan=False) + extra_whatsapp_sender_enabled: bool = False + extra_whatsapp_api_version: str | None = None + extra_whatsapp_access_token: SecretStr | None = Field(default=None, repr=False, exclude=True) + extra_whatsapp_support_contact: str | None = None + @model_validator(mode="before") @classmethod def _backfill_deprecated_agent_vars(cls, data: object) -> object: @@ -202,6 +223,66 @@ def _require_secret_for_host_identity(self) -> Settings: ) return self + @model_validator(mode="after") + def _validate_whatsapp_intake(self) -> Settings: + if self.extra_whatsapp_sender_enabled: + import re + + from agent_manager.domain.channels.models import bounded_identifier + + if not self.extra_whatsapp_worker_enabled: + raise ValueError("WhatsApp sender requires the supervised worker") + version = self.extra_whatsapp_api_version + if version is None or re.fullmatch(r"v[1-9][0-9]{0,2}\.0", version) is None: + raise ValueError("EXTRA_WHATSAPP_API_VERSION must pin a Graph API version") + access = self.extra_whatsapp_access_token + if access is None or not access.get_secret_value().strip(): + raise ValueError("EXTRA_WHATSAPP_ACCESS_TOKEN is required for sending") + for other in (self.extra_whatsapp_app_secret, self.extra_whatsapp_verify_token): + if other is not None and other.get_secret_value() == access.get_secret_value(): + raise ValueError("WhatsApp access token must differ from webhook secrets") + if self.extra_whatsapp_support_contact is None: + raise ValueError("EXTRA_WHATSAPP_SUPPORT_CONTACT is required for sending") + bounded_identifier(self.extra_whatsapp_support_contact) + if self.extra_whatsapp_worker_enabled: + import re + + from agent_manager.domain.channels.models import bounded_identifier + + if not self.extra_whatsapp_enabled: + raise ValueError("WhatsApp worker requires enabled intake") + if not self.extra_whatsapp_organization_id: + raise ValueError("EXTRA_WHATSAPP_ORGANIZATION_ID is required for the worker") + bounded_identifier(self.extra_whatsapp_organization_id) + plugin = self.extra_whatsapp_identity_resolver + if ( + not plugin + or len(plugin) > 512 + or re.fullmatch(r"[A-Za-z_]\w*(?:\.[A-Za-z_]\w*)*:[A-Za-z_]\w*", plugin) is None + ): + raise ValueError("EXTRA_WHATSAPP_IDENTITY_RESOLVER must name module:Class") + if not self.extra_whatsapp_enabled: + return self + from agent_manager.domain.channels.models import bounded_identifier + + bounded_identifier(self.extra_whatsapp_connection_id, limit=128) + for name in ("account_id", "phone_number_id"): + value = getattr(self, f"extra_whatsapp_{name}") + if not value: + raise ValueError(f"EXTRA_WHATSAPP_{name.upper()} is required for intake") + bounded_identifier(value) + secret = self.extra_whatsapp_app_secret + token = self.extra_whatsapp_verify_token + if secret is None or not secret.get_secret_value().strip(): + raise ValueError("EXTRA_WHATSAPP_APP_SECRET is required for intake") + if token is None or not token.get_secret_value().strip(): + raise ValueError("EXTRA_WHATSAPP_VERIFY_TOKEN is required for intake") + if secret.get_secret_value() == token.get_secret_value(): + raise ValueError("WhatsApp verification token must differ from the app secret") + if self.uses_process_memory: + raise ValueError("WhatsApp intake requires persistent SQL storage") + return self + @classmethod def from_values(cls, **values: Any) -> Settings: """Explicit values only, ignoring any local `.env`, so callers that must diff --git a/src/agent_manager/infrastructure/channels/__init__.py b/src/agent_manager/infrastructure/channels/__init__.py new file mode 100644 index 00000000..ea0755f9 --- /dev/null +++ b/src/agent_manager/infrastructure/channels/__init__.py @@ -0,0 +1 @@ +"""Provider adapters for external conversation channels.""" diff --git a/src/agent_manager/infrastructure/channels/identity_loader.py b/src/agent_manager/infrastructure/channels/identity_loader.py new file mode 100644 index 00000000..de531344 --- /dev/null +++ b/src/agent_manager/infrastructure/channels/identity_loader.py @@ -0,0 +1,21 @@ +"""Load a trusted deployment plugin once; no provider or account logic here.""" + +import importlib +import inspect + +from agent_manager.domain.channels.identity import ChannelIdentityResolver + + +def load_channel_identity_resolver(path: str) -> ChannelIdentityResolver: + try: + module_name, name = path.split(":", 1) + plugin_type = getattr(importlib.import_module(module_name), name) + if not inspect.isclass(plugin_type) or not issubclass(plugin_type, ChannelIdentityResolver): + raise TypeError("resolver must implement the channel identity port") + if not inspect.iscoroutinefunction(plugin_type.resolve) or not inspect.iscoroutinefunction( + plugin_type.close + ): + raise TypeError("resolver lookup and cleanup must be async") + return plugin_type() + except Exception: + raise ValueError("channel identity resolver could not be loaded") from None diff --git a/src/agent_manager/infrastructure/channels/whatsapp/__init__.py b/src/agent_manager/infrastructure/channels/whatsapp/__init__.py new file mode 100644 index 00000000..f45f4851 --- /dev/null +++ b/src/agent_manager/infrastructure/channels/whatsapp/__init__.py @@ -0,0 +1 @@ +"""WhatsApp Cloud API intake. No agent execution or outgoing transport.""" diff --git a/src/agent_manager/infrastructure/channels/whatsapp/config.py b/src/agent_manager/infrastructure/channels/whatsapp/config.py new file mode 100644 index 00000000..861ff2b1 --- /dev/null +++ b/src/agent_manager/infrastructure/channels/whatsapp/config.py @@ -0,0 +1,24 @@ +"""Validated, immutable startup target; secret values are never represented.""" + +from dataclasses import dataclass, field + +from pydantic import SecretStr + +from agent_manager.domain.channels.models import bounded_identifier + + +@dataclass(frozen=True) +class WhatsAppWebhookConfig: + connection_id: str + system_id: str + account_id: str + phone_number_id: str + app_secret: SecretStr = field(repr=False) + verify_token: SecretStr = field(repr=False) + max_body_bytes: int = 262_144 + + def __post_init__(self) -> None: + bounded_identifier(self.connection_id, limit=128) + bounded_identifier(self.system_id, limit=256) + bounded_identifier(self.account_id) + bounded_identifier(self.phone_number_id) diff --git a/src/agent_manager/infrastructure/channels/whatsapp/normalization.py b/src/agent_manager/infrastructure/channels/whatsapp/normalization.py new file mode 100644 index 00000000..b4125993 --- /dev/null +++ b/src/agent_manager/infrastructure/channels/whatsapp/normalization.py @@ -0,0 +1,156 @@ +"""Pure normalization of a verified batch; invalid siblings are quarantined.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +from pydantic import ValidationError + +from agent_manager.domain.channels.models import ( + ChannelAddress, + InboundMessage, + MessageKind, + bounded_identifier, +) +from agent_manager.domain.channels.webhooks import DeliveryEvent, WebhookBatch, WebhookIssue +from agent_manager.infrastructure.channels.whatsapp.config import WhatsAppWebhookConfig +from agent_manager.infrastructure.channels.whatsapp.payloads import ( + Change, + Entry, + Envelope, + Message, + Messages, + RoutingValue, + Status, + Statuses, +) + + +def normalize(body: bytes, config: WhatsAppWebhookConfig, received_at: datetime) -> WebhookBatch: + messages: list[InboundMessage] = [] + delivery: list[DeliveryEvent] = [] + issues: list[WebhookIssue] = [] + try: + envelope = Envelope.model_validate_json(body) + except (ValidationError, ValueError): + return WebhookBatch(issues=(WebhookIssue("body", "invalid_envelope"),)) + if envelope.object != "whatsapp_business_account": + return WebhookBatch() + if "entry" not in envelope.model_fields_set: + return WebhookBatch(issues=(WebhookIssue("body", "invalid_envelope"),)) + for entry_index, raw_entry in enumerate(envelope.entry): + path = f"entry.{entry_index}" + # Other accounts must not affect this connection, even when malformed. + if ( + isinstance(raw_entry, dict) + and "id" in raw_entry + and raw_entry["id"] != config.account_id + ): + continue + try: + entry = Entry.model_validate(raw_entry) + except ValidationError: + issues.append(WebhookIssue(path, "invalid_entry")) + continue + for change_index, raw_change in enumerate(entry.changes): + location = f"{path}.changes.{change_index}" + try: + change = Change.model_validate(raw_change) + except ValidationError: + issues.append(WebhookIssue(location, "invalid_change")) + continue + if change.field != "messages": + continue + try: + routing = RoutingValue.model_validate(change.value) + if routing.metadata.phone_number_id != config.phone_number_id: + continue + except ValidationError: + issues.append(WebhookIssue(location, "invalid_value")) + continue + try: + inbound = Messages.model_validate(change.value).messages + except ValidationError: + issues.append(WebhookIssue(f"{location}.messages", "invalid_value")) + inbound = [] + try: + statuses = Statuses.model_validate(change.value).statuses + except ValidationError: + issues.append(WebhookIssue(f"{location}.statuses", "invalid_value")) + statuses = [] + for index, raw in enumerate(inbound): + item_path = f"{location}.messages.{index}" + try: + message = Message.model_validate(raw) + if message.type not in { + "text", + "image", + "audio", + "video", + "document", + "sticker", + "location", + "contacts", + "interactive", + "button", + "order", + }: + raise ValueError("not a supported customer message kind") + sent_at = _timestamp(message.timestamp, received_at) + if message.type == "text" and message.text is None: + raise ValueError("missing text") + messages.append( + InboundMessage( + ChannelAddress(config.connection_id, config.system_id, message.sender), + message.id, + message.text.body if message.type == "text" and message.text else "", + received_at, + sent_at, + _kind(message.text.body) + if message.type == "text" and message.text + else "unsupported", + ) + ) + except (ValidationError, ValueError, OverflowError, OSError): + issues.append(WebhookIssue(item_path, "invalid_message")) + for index, raw in enumerate(statuses): + item_path = f"{location}.statuses.{index}" + try: + status = Status.model_validate(raw) + for identifier in (status.id, status.recipient_id): + bounded_identifier(identifier) + bounded_identifier(status.status, limit=64) + delivery.append( + DeliveryEvent( + status.id, + status.recipient_id, + status.status, + _timestamp(status.timestamp, received_at), + ) + ) + except (ValidationError, ValueError, OverflowError, OSError): + issues.append(WebhookIssue(item_path, "invalid_status")) + if len(issues) > 256: + issues = [*issues[:255], WebhookIssue("body", "issue_limit")] + return WebhookBatch(tuple(messages), tuple(delivery), tuple(issues)) + + +def _kind(body: str) -> MessageKind: + commands: dict[str, MessageKind] = { + "/new": "reset", + "שיחה חדשה": "reset", + "/help": "help", + "נציג": "help", + "/stop": "stop", + "הפסק": "stop", + "/start": "start", + "המשך": "start", + } + return commands.get(body.strip().casefold(), "text") + + +def _timestamp(value: str, received_at: datetime) -> datetime: + timestamp = datetime.fromtimestamp(int(value), tz=UTC) + if timestamp > received_at + timedelta(minutes=5): + raise ValueError("future provider timestamp") + return timestamp diff --git a/src/agent_manager/infrastructure/channels/whatsapp/payloads.py b/src/agent_manager/infrastructure/channels/whatsapp/payloads.py new file mode 100644 index 00000000..c8d78368 --- /dev/null +++ b/src/agent_manager/infrastructure/channels/whatsapp/payloads.py @@ -0,0 +1,64 @@ +"""Strict boundary models. Unknown provider fields never become domain context.""" + +import builtins +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field, StringConstraints + +Identifier = Annotated[str, StringConstraints(min_length=1, max_length=512)] +Timestamp = Annotated[str, StringConstraints(pattern=r"^[0-9]{1,20}$")] + + +class Payload(BaseModel): + model_config = ConfigDict(strict=True, extra="ignore", hide_input_in_errors=True) + + +class Envelope(Payload): + object: Identifier + entry: list[builtins.object] = Field(default_factory=list, max_length=100) + + +class Entry(Payload): + id: Identifier + changes: list[object] = Field(max_length=100) + + +class Change(Payload): + field: Identifier + value: object + + +class Metadata(Payload): + phone_number_id: Identifier + + +class RoutingValue(Payload): + messaging_product: Literal["whatsapp"] + metadata: Metadata + + +class Messages(Payload): + messages: list[object] = Field(default_factory=list, max_length=1000) + + +class Statuses(Payload): + statuses: list[object] = Field(default_factory=list, max_length=1000) + + +class Text(Payload): + body: Annotated[str, StringConstraints(min_length=1, max_length=65_536)] + + +class Message(Payload): + id: Identifier + sender: Identifier = Field(alias="from") + timestamp: Timestamp + type: Identifier + text: Text | None = None + + +class Status(Payload): + id: Identifier + recipient_id: Identifier + timestamp: Timestamp + status: Annotated[str, StringConstraints(min_length=1, max_length=64)] diff --git a/src/agent_manager/infrastructure/channels/whatsapp/rendering.py b/src/agent_manager/infrastructure/channels/whatsapp/rendering.py new file mode 100644 index 00000000..541798dc --- /dev/null +++ b/src/agent_manager/infrastructure/channels/whatsapp/rendering.py @@ -0,0 +1,33 @@ +"""Plain text, exact grapheme boundaries and conservative UTF-16 bounds.""" + +import regex + +from agent_manager.domain.channels.models import bounded_text + + +def render_text(text: str, *, limit: int = 4096) -> tuple[str, ...]: + bounded_text(text) + if not text.strip() or not 1 <= limit <= 4096: + raise ValueError("invalid WhatsApp text or part bound") + clusters = regex.findall(r"\X", text) + parts: list[str] = [] + start = 0 + while start < len(clusters): + end, units, paragraph = start, 0, None + while end < len(clusters): + size = len(clusters[end].encode("utf-16-le")) // 2 + if units + size > limit: + break + units += size + end += 1 + if end >= 2 and clusters[end - 2 : end] == ["\n", "\n"]: + paragraph = end + if end == start: + raise ValueError("one grapheme exceeds the WhatsApp part bound") + if end < len(clusters) and paragraph is not None: + end = paragraph + parts.append("".join(clusters[start:end])) + start = end + if len(parts) > 128: + raise ValueError("WhatsApp reply exceeds 128 parts") + return tuple(parts) diff --git a/src/agent_manager/infrastructure/channels/whatsapp/security.py b/src/agent_manager/infrastructure/channels/whatsapp/security.py new file mode 100644 index 00000000..97351947 --- /dev/null +++ b/src/agent_manager/infrastructure/channels/whatsapp/security.py @@ -0,0 +1,33 @@ +"""Byte-level Meta authentication, independent of browser/user credentials.""" + +import hashlib +import hmac +import re + + +class WebhookAuthenticationError(PermissionError): + """The provider request could not be authenticated.""" + + +def verify_signature(body: bytes, signature: str | None, app_secret: str) -> None: + if signature is None or re.fullmatch(r"sha256=[0-9a-fA-F]{64}", signature) is None: + raise WebhookAuthenticationError("invalid webhook signature") + supplied = bytes.fromhex(signature.removeprefix("sha256=")) + expected = hmac.new(app_secret.encode(), body, hashlib.sha256).digest() + if not hmac.compare_digest(supplied, expected): + raise WebhookAuthenticationError("invalid webhook signature") + + +def subscription_challenge( + mode: str | None, token: str | None, challenge: str | None, expected: str +) -> str: + if ( + mode != "subscribe" + or token is None + or challenge is None + or not challenge + or len(challenge.encode()) > 1024 + or not hmac.compare_digest(token.encode(), expected.encode()) + ): + raise WebhookAuthenticationError("invalid webhook subscription") + return challenge diff --git a/src/agent_manager/infrastructure/channels/whatsapp/sender.py b/src/agent_manager/infrastructure/channels/whatsapp/sender.py new file mode 100644 index 00000000..1072f32a --- /dev/null +++ b/src/agent_manager/infrastructure/channels/whatsapp/sender.py @@ -0,0 +1,106 @@ +"""A bounded single-attempt HTTPS adapter. Never log provider response bodies.""" + +import json +import re +from urllib.parse import quote + +import httpx +from pydantic import SecretStr + +from agent_manager.domain.channels.delivery import ChannelSender, SendOutcome +from agent_manager.domain.channels.models import bounded_identifier, bounded_text + + +class WhatsAppSender(ChannelSender): + def __init__( + self, + version: str, + phone_number_id: str, + token: SecretStr, + *, + client: httpx.AsyncClient | None = None, + ) -> None: + if re.fullmatch(r"v[1-9][0-9]{0,2}\.0", version) is None: + raise ValueError("WhatsApp sender requires an explicit Graph API version") + bounded_identifier(phone_number_id) + if re.fullmatch(r"[A-Za-z0-9_-]+", phone_number_id) is None: + raise ValueError("invalid WhatsApp sending number identifier") + if not token.get_secret_value().strip(): + raise ValueError("WhatsApp sender requires an access token") + self._url = ( + f"https://graph.facebook.com/{version}/{quote(phone_number_id, safe='')}/messages" + ) + self._token = token + self._owns_client = client is None + self._client = client or httpx.AsyncClient( + timeout=httpx.Timeout(15, connect=5, pool=5), + follow_redirects=False, + trust_env=False, + transport=httpx.AsyncHTTPTransport(retries=0), + ) + + async def close(self) -> None: + if self._owns_client: + await self._client.aclose() + + async def send(self, recipient: str, text: str) -> SendOutcome: + bounded_identifier(recipient) + bounded_text(text) + if not text or len(text.encode("utf-16-le")) // 2 > 4096: + raise ValueError("invalid WhatsApp provider text part") + try: + async with self._client.stream( + "POST", + self._url, + headers={"Authorization": f"Bearer {self._token.get_secret_value()}"}, + json={ + "messaging_product": "whatsapp", + "recipient_type": "individual", + "to": recipient, + "type": "text", + "text": {"body": text, "preview_url": False}, + }, + follow_redirects=False, + ) as response: + body = bytearray() + async for chunk in response.aiter_bytes(): + if len(body) + len(chunk) > 65_536: + return SendOutcome("unknown") + body.extend(chunk) + return _outcome( + response.status_code, bytes(body), response.headers.get("Retry-After", "") + ) + except (httpx.ConnectTimeout, httpx.ConnectError, httpx.PoolTimeout): + return SendOutcome("retry") + except httpx.RequestError: + return SendOutcome("unknown") + + +def _outcome(status: int, body: bytes, retry_after: str) -> SendOutcome: + try: + data = json.loads(body) + if 200 <= status < 300: + messages = data["messages"] + if not isinstance(messages, list) or len(messages) != 1: + return SendOutcome("unknown") + provider_id = messages[0]["id"] + if not isinstance(provider_id, str): + return SendOutcome("unknown") + bounded_identifier(provider_id) + return SendOutcome("accepted", provider_id) + error = data.get("error") if isinstance(data, dict) else None + known_rejection = isinstance(error, dict) and type(error.get("code")) is int + if known_rejection and ( + status == 429 + or 500 <= status < 600 + or (status == 400 and isinstance(error, dict) and error.get("is_transient") is True) + ): + delay = ( + int(retry_after) + if len(retry_after) <= 3 and retry_after.isascii() and retry_after.isdecimal() + else 0 + ) + return SendOutcome("retry", retry_after=min(delay, 300)) + except (ValueError, TypeError, KeyError, IndexError): + pass + return SendOutcome("unknown" if status >= 500 or 200 <= status < 300 else "failed") diff --git a/src/agent_manager/infrastructure/channels/whatsapp/webhook.py b/src/agent_manager/infrastructure/channels/whatsapp/webhook.py new file mode 100644 index 00000000..30b3ec8b --- /dev/null +++ b/src/agent_manager/infrastructure/channels/whatsapp/webhook.py @@ -0,0 +1,28 @@ +"""Authenticate, normalize, and durably accept a callback, without executing.""" + +import hashlib +from datetime import datetime + +from agent_manager.domain.channels.webhooks import WebhookReceipt, WebhookRepository +from agent_manager.infrastructure.channels.whatsapp.config import WhatsAppWebhookConfig +from agent_manager.infrastructure.channels.whatsapp.normalization import normalize +from agent_manager.infrastructure.channels.whatsapp.security import verify_signature + + +class WhatsAppWebhook: + def __init__(self, config: WhatsAppWebhookConfig, repository: WebhookRepository) -> None: + self.config = config + self._repository = repository + + async def receive( + self, body: bytes, signature: str | None, received_at: datetime + ) -> WebhookReceipt: + verify_signature(body, signature, self.config.app_secret.get_secret_value()) + batch = normalize(body, self.config, received_at) + return await self._repository.accept( + self.config.connection_id, + self.config.system_id, + hashlib.sha256(body).hexdigest(), + received_at, + batch, + ) diff --git a/tests/agent_manager/test_channel_controls.py b/tests/agent_manager/test_channel_controls.py new file mode 100644 index 00000000..05c65980 --- /dev/null +++ b/tests/agent_manager/test_channel_controls.py @@ -0,0 +1,136 @@ +"""Deterministic controls never create a model run or reveal internal history.""" + +import asyncio + +import pytest +from sqlmodel.ext.asyncio.session import AsyncSession + +from agent_manager.application.channels.controls import ChannelControlService +from agent_manager.domain.channels.models import InboundMessage, InboxState +from agent_manager.infrastructure.persistence.channels.controls import SqlControlRepository +from agent_manager.infrastructure.persistence.channels.tables import ChannelBindingRow +from agent_manager.infrastructure.persistence.database import session_factory +from tests.agent_manager.test_channel_worker import ADDRESS, ALICE, NOW +from tests.agent_manager.test_channel_worker import work as work + + +def controls(work): + return ChannelControlService( + SqlControlRepository(session_factory(work.database)), + work.repository, + support_contact="support@example.test", + ) + + +@pytest.mark.parametrize( + "kind,body,fragment", + [ + ("help", "/help", "support@example.test"), + ("reset", "/new", "שיחה חדשה"), + ("start", "/start", "חודשה"), + ("unsupported", "", "טקסט"), + ], +) +async def test_controls_stage_static_replies_and_never_prepare_agent_turns( + work, kind, body, fragment +): + item = await work.claim(content=body, kind=kind) + assert await controls(work).handle(item) + assert fragment in (await work.repository.list_replies(item.event_id))[0].text + assert work.engine.prompts == [] + assert (await work.repository.get_inbound(item.event_id)).run_id is None + + +async def test_reset_is_atomic_idempotent_preserves_history_and_changes_generation(work): + old = await work.claim() + await work.processor().process(old) + before = await work.repository.get_binding(ADDRESS) + item = await work.claim("reset", content="/new", kind="reset") + assert await controls(work).handle(item) + assert await controls(work).handle(item) + after = await work.repository.get_binding(ADDRESS) + assert after.generation == before.generation + 1 and after.conversation_id is None + assert len(await work.service.history(before.conversation_id, ALICE)) == 2 + assert len(await work.repository.list_replies(item.event_id)) == 1 + + +async def test_stop_applies_before_worker_and_start_does_not_execute_old_queued_text(work): + old = await work.repository.ingest(InboundMessage(ADDRESS, "old", "old question", NOW, NOW)) + await work.repository.ingest(InboundMessage(ADDRESS, "stop", "/stop", NOW, NOW, "stop")) + await work.repository.ingest( + InboundMessage(ADDRESS, "during", "question while stopped", NOW, NOW) + ) + await work.repository.ingest(InboundMessage(ADDRESS, "start", "/start", NOW, NOW, "start")) + service = controls(work) + for _ in range(4): + item = await work.repository.claim_next("connection", "system") + assert await service.handle(item) + assert work.engine.prompts == [] + assert (await work.repository.get_inbound(old.event_id)).state is InboxState.DONE + assert await work.repository.list_replies(old.event_id) == () + + +async def test_unrecognized_command_stays_normal_user_text(work): + item = await work.claim(content="/help me find a file") + assert not await controls(work).handle(item) + + +async def test_delayed_duplicate_control_reset_cannot_clear_a_new_conversation(work, monkeypatch): + await work.processor().process(await work.claim()) + item = await work.claim("reset", content="/new", kind="reset") + repository = SqlControlRepository(session_factory(work.database)) + ready, release = asyncio.Event(), asyncio.Event() + original = AsyncSession.get + slow = None + + async def delay_binding(self, model, *args, **kwargs): + result = await original(self, model, *args, **kwargs) + if asyncio.current_task() is slow and model is ChannelBindingRow: + ready.set() + await release.wait() + return result + + monkeypatch.setattr(AsyncSession, "get", delay_binding) + slow = asyncio.create_task(repository.complete(item, None, reset=True)) + try: + await asyncio.wait_for(ready.wait(), 2) + await repository.complete(item, None, reset=True) + next_item = await work.claim("next") + await work.repository.bind_conversation( + next_item.event_id, next_item.claim_token, ALICE.user_id, "org", "new-conversation" + ) + finally: + release.set() + await slow + binding = await work.repository.get_binding(ADDRESS) + assert binding.generation == 1 and binding.conversation_id == "new-conversation" + + +@pytest.mark.parametrize("path", ["legacy", "control"]) +async def test_reset_reconciles_a_commit_between_inbox_and_binding_reads(work, monkeypatch, path): + item = await work.claim(content="/new", kind="reset") + ready, release = asyncio.Event(), asyncio.Event() + original = AsyncSession.get + slow = None + + async def operation(): + if path == "legacy": + return await work.repository.reset_conversation(item.event_id, item.claim_token) + await SqlControlRepository(session_factory(work.database)).complete(item, None, reset=True) + return 1 + + async def delay_binding(self, model, *args, **kwargs): + if asyncio.current_task() is slow and model is ChannelBindingRow: + ready.set() + await release.wait() + return await original(self, model, *args, **kwargs) + + monkeypatch.setattr(AsyncSession, "get", delay_binding) + slow = asyncio.create_task(operation()) + try: + await asyncio.wait_for(ready.wait(), 2) + assert await operation() == 1 + finally: + release.set() + assert await slow == 1 + assert (await work.repository.get_binding(ADDRESS)).generation == 1 diff --git a/tests/agent_manager/test_channel_delivery.py b/tests/agent_manager/test_channel_delivery.py new file mode 100644 index 00000000..d013ea50 --- /dev/null +++ b/tests/agent_manager/test_channel_delivery.py @@ -0,0 +1,185 @@ +"""No-network delivery processing under current authorization and durable state.""" + +import asyncio +from datetime import timedelta + +import pytest + +from agent_manager.application.channels.delivery import ChannelDeliveryService +from agent_manager.domain import Principal +from agent_manager.domain.channels.delivery import ChannelSender, SendOutcome +from agent_manager.infrastructure.channels.whatsapp.rendering import render_text +from tests.agent_manager.test_channel_delivery_storage import answered +from tests.agent_manager.test_channel_delivery_storage import delivery as delivery +from tests.agent_manager.test_channel_worker import ALICE, NOW, Identities +from tests.agent_manager.test_channel_worker import work as work + + +class Sender(ChannelSender): + def __init__(self, *outcomes): + self.outcomes = list(outcomes or (SendOutcome("accepted", "wamid.one"),)) + self.calls = [] + + async def send(self, recipient, text): + self.calls.append((recipient, text)) + return self.outcomes.pop(0) + + +def pump(work, delivery, sender, *, now=NOW): + return ChannelDeliveryService( + delivery, + work.identities, + sender, + connection_id="connection", + system_id="system", + organization_id="org", + render=render_text, + clock=lambda: now, + jitter=lambda: 0.5, + ) + + +async def test_delivery_sends_saved_answer_without_another_model_call(work, delivery): + item = await answered(work) + sender = Sender() + service = pump(work, delivery, sender) + await service.tick(asyncio.Event()) + await service.tick(asyncio.Event()) + assert sender.calls == [("sender", "answer:hello")] + assert work.engine.prompts == ["hello"] + assert (await delivery.parts(item.event_id))[0].state == "accepted" + + +@pytest.mark.parametrize( + "principal", + [ + None, + Principal.anonymous("v"), + Principal.external("other", organization_id="org"), + Principal.external("alice", organization_id="other"), + ], +) +async def test_delivery_reauthorizes_original_owner_before_exposing_internal_text( + work, delivery, principal +): + item = await answered(work) + work.identities = Identities(principal) + sender = Sender() + await pump(work, delivery, sender).tick(asyncio.Event()) + assert sender.calls == [] + assert (await delivery.parts(item.event_id))[0].state == "blocked_identity" + + +async def test_delivery_checks_window_again_on_retry_and_preserves_saved_text(work, delivery): + item = await answered(work) + sender = Sender(SendOutcome("retry", retry_after=1)) + await pump(work, delivery, sender).tick(asyncio.Event()) + part = (await delivery.parts(item.event_id))[0] + assert part.state == "retry" and part.next_attempt_at > NOW + await pump(work, delivery, sender, now=NOW + timedelta(days=1)).tick(asyncio.Event()) + assert len(sender.calls) == 1 + assert (await delivery.parts(item.event_id))[0].state == "blocked_window" + assert work.engine.prompts == ["hello"] + + +async def test_unknown_submission_blocks_same_recipient_without_retry(work, delivery): + item = await answered(work) + await answered(work, "two") + sender = Sender(SendOutcome("unknown")) + service = pump(work, delivery, sender) + await service.tick(asyncio.Event()) + await service.tick(asyncio.Event()) + assert len(sender.calls) == 1 + assert (await delivery.parts(item.event_id))[0].state == "delivery_unknown" + + +async def test_cancelled_send_is_joined_and_persisted_as_unknown(work, delivery): + started = asyncio.Event() + + class BlockedSender(Sender): + async def send(self, recipient, text): + started.set() + await asyncio.Event().wait() + + item = await answered(work) + task = asyncio.create_task(pump(work, delivery, BlockedSender()).tick(asyncio.Event())) + await asyncio.wait_for(started.wait(), 2) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert (await delivery.parts(item.event_id))[0].state == "delivery_unknown" + + +async def test_shutdown_after_identity_lookup_prevents_network_submission(work, delivery): + stop = asyncio.Event() + + class StopIdentity(Identities): + async def resolve(self, address): + stop.set() + return ALICE + + await answered(work) + work.identities = StopIdentity() + sender = Sender() + await pump(work, delivery, sender).tick(stop) + assert sender.calls == [] + + +async def test_each_part_refreshes_identity_and_revocation_stops_remaining_parts(work, delivery): + item = await work.claim(content="x" * 9000) + await work.processor().process(item) + sender = Sender() + await pump(work, delivery, sender).tick(asyncio.Event()) + work.identities = Identities(None) + await pump(work, delivery, sender).tick(asyncio.Event()) + assert len(sender.calls) == 1 + assert [part.state for part in await delivery.parts(item.event_id)] == [ + "accepted", + "blocked_identity", + "ready", + ] + + +async def test_lost_acceptance_commit_becomes_unknown_without_regenerating_or_resending( + work, delivery, monkeypatch +): + item = await answered(work) + original = delivery.complete + + async def unavailable(*args, **kwargs): + raise OSError("private-storage-credential") + + sender = Sender() + monkeypatch.setattr(delivery, "complete", unavailable) + with pytest.raises(OSError): + await pump(work, delivery, sender).tick(asyncio.Event()) + monkeypatch.setattr(delivery, "complete", original) + await pump(work, delivery, sender).tick(asyncio.Event()) + assert len(sender.calls) == 1 and work.engine.prompts == ["hello"] + assert (await delivery.parts(item.event_id))[0].state == "delivery_unknown" + + +async def test_known_retries_stop_at_the_attempt_budget(work, delivery): + item = await answered(work) + sender = Sender(*(SendOutcome("retry") for _ in range(4))) + current = NOW + for _ in range(4): + await pump(work, delivery, sender, now=current).tick(asyncio.Event()) + part = (await delivery.parts(item.event_id))[0] + if part.next_attempt_at: + current = part.next_attempt_at + await pump(work, delivery, sender, now=current + timedelta(seconds=100)).tick(asyncio.Event()) + assert len(sender.calls) == 4 and part.state == "failed" + + +async def test_identity_failure_is_sanitized_and_never_exposes_the_saved_answer( + work, delivery, caplog +): + item = await answered(work) + work.identities = Identities(RuntimeError("private-identity-credential")) + sender = Sender() + await pump(work, delivery, sender).tick(asyncio.Event()) + assert sender.calls == [] + assert (await delivery.parts(item.event_id))[0].state == "blocked_identity" + assert "private-identity-credential" not in caplog.text + assert "answer:hello" not in caplog.text diff --git a/tests/agent_manager/test_channel_delivery_storage.py b/tests/agent_manager/test_channel_delivery_storage.py new file mode 100644 index 00000000..0adcf23e --- /dev/null +++ b/tests/agent_manager/test_channel_delivery_storage.py @@ -0,0 +1,225 @@ +"""Real local transactions for provider parts and recipient ordering.""" + +from datetime import timedelta + +import pytest + +from agent_manager.domain.channels.models import ChannelMessageConflict, InboundMessage +from agent_manager.domain.channels.webhooks import DeliveryEvent, WebhookBatch +from agent_manager.infrastructure.persistence.channels.delivery import SqlDeliveryRepository +from agent_manager.infrastructure.persistence.channels.webhooks import SqlWebhookRepository +from agent_manager.infrastructure.persistence.database import session_factory +from tests.agent_manager.test_channel_worker import ADDRESS, NOW +from tests.agent_manager.test_channel_worker import work as work + + +@pytest.fixture +def delivery(work): + return SqlDeliveryRepository(session_factory(work.database)) + + +async def answered(work, provider="one", sender="sender"): + item = await work.claim(provider, sender=sender) + await work.processor().process(item) + return item + + +async def test_delivery_planning_is_immutable_atomic_and_reuses_source_ids(work, delivery): + item = await answered(work) + sources = await delivery.sources("connection", "system") + assert [(source.event_id, source.texts) for source in sources] == [ + (item.event_id, ("answer:hello",)) + ] + await delivery.materialize(item.event_id, ("part one", "part two")) + first = await delivery.parts(item.event_id) + await delivery.materialize(item.event_id, ("part one", "part two")) + assert await delivery.parts(item.event_id) == first + assert [part.text for part in first] == ["part one", "part two"] + assert await delivery.sources("connection", "system") == () + with pytest.raises(ChannelMessageConflict): + await delivery.materialize(item.event_id, ("replacement",)) + + +async def test_parts_and_turns_wait_for_prior_acceptance_but_other_senders_progress(work, delivery): + one = await answered(work) + two = await answered(work, "two") + other = await answered(work, "other", "other") + for item, texts in ((one, ("a", "b")), (two, ("c",)), (other, ("d",))): + await delivery.materialize(item.event_id, texts) + first = await delivery.claim_next("connection", "system", NOW) + assert first.event_id == one.event_id and first.attempts == 1 + independent = await delivery.claim_next("connection", "system", NOW) + assert independent.event_id == other.event_id + assert await delivery.claim_next("connection", "system", NOW) is None + await delivery.complete(first, "accepted", provider_id="wamid.a") + second = await delivery.claim_next("connection", "system", NOW) + assert second.event_id == one.event_id and second.part_index == 1 + await delivery.complete(second, "accepted", provider_id="wamid.b") + assert (await delivery.claim_next("connection", "system", NOW)).event_id == two.event_id + + +async def test_retry_due_time_and_abandoned_submission_never_resend(work, delivery): + item = await answered(work) + await delivery.materialize(item.event_id, ("a",)) + part = await delivery.claim_next("connection", "system", NOW) + await delivery.complete(part, "retry", next_attempt_at=NOW + timedelta(seconds=10)) + assert await delivery.claim_next("connection", "system", NOW) is None + retried = await delivery.claim_next("connection", "system", NOW + timedelta(seconds=10)) + assert retried.delivery_id == part.delivery_id and retried.attempts == 2 + await delivery.recover_sending("connection", "system") + assert (await delivery.parts(item.event_id))[0].state == "delivery_unknown" + assert await delivery.claim_next("connection", "system", NOW + timedelta(days=1)) is None + + +async def test_guard_uses_newest_verified_inbound_time_and_does_not_reopen_from_status( + work, delivery +): + item = await answered(work) + await delivery.materialize(item.event_id, ("a",)) + part = await delivery.claim_next("connection", "system", NOW) + assert await delivery.guard(part, NOW + timedelta(hours=24)) == "blocked_window" + await work.repository.ingest( + InboundMessage(ADDRESS, "late", "late", NOW, NOW - timedelta(days=2)) + ) + assert await delivery.guard(part, NOW + timedelta(hours=24)) == "blocked_window" + await work.repository.ingest( + InboundMessage(ADDRESS, "new", "new", NOW + timedelta(hours=23), NOW + timedelta(hours=23)) + ) + assert await delivery.guard(part, NOW + timedelta(hours=24)) is None + + +async def test_stop_and_start_never_resurrect_previously_pending_replies(work, delivery): + item = await answered(work) + await delivery.materialize(item.event_id, ("a",)) + part = await delivery.claim_next("connection", "system", NOW) + await work.repository.ingest(InboundMessage(ADDRESS, "stop", "/stop", NOW, NOW, "stop")) + assert await delivery.guard(part, NOW) == "suppressed" + await work.repository.ingest(InboundMessage(ADDRESS, "start", "/start", NOW, NOW, "start")) + assert await delivery.guard(part, NOW) == "suppressed" + + +async def test_projection_handles_early_callbacks_exact_recipient_and_late_failures(work, delivery): + item = await answered(work) + await delivery.materialize(item.event_id, ("a",)) + part = await delivery.claim_next("connection", "system", NOW) + webhook = SqlWebhookRepository(session_factory(work.database)) + await webhook.accept( + "connection", + "system", + "a" * 64, + NOW, + WebhookBatch( + delivery_events=( + DeliveryEvent("wamid.a", "other", "read", NOW), + DeliveryEvent("wamid.a", "sender", "delivered", NOW), + ) + ), + ) + await delivery.project("connection", "system") + await delivery.complete(part, "accepted", provider_id="wamid.a") + await delivery.project("connection", "system") + assert (await delivery.parts(item.event_id))[0].state == "delivered" + await webhook.accept( + "connection", + "system", + "b" * 64, + NOW, + WebhookBatch( + delivery_events=( + DeliveryEvent("wamid.a", "sender", "sent", NOW - timedelta(seconds=1)), + DeliveryEvent("wamid.a", "sender", "failed", NOW + timedelta(seconds=1)), + DeliveryEvent("wamid.a", "sender", "read", NOW + timedelta(seconds=2)), + ) + ), + ) + await delivery.project("connection", "system") + assert (await delivery.parts(item.event_id))[0].state == "read" + + +async def test_many_blocked_parts_cannot_starve_an_independent_recipient(work, delivery): + item = await answered(work) + other = await answered(work, "independent", "other") + await delivery.materialize(item.event_id, tuple("a" for _ in range(40))) + await delivery.materialize(other.event_id, ("other",)) + first = await delivery.claim_next("connection", "system", NOW) + await delivery.complete(first, "retry", next_attempt_at=NOW + timedelta(hours=1)) + next_part = await delivery.claim_next("connection", "system", NOW) + assert next_part is not None and next_part.event_id == other.event_id + + +async def test_an_unplanned_earlier_answer_prevents_later_recipient_delivery(work, delivery): + await answered(work) + later = await answered(work, "later") + await delivery.materialize(later.event_id, ("later",)) + assert await delivery.claim_next("connection", "system", NOW) is None + + +async def test_internal_source_without_conversation_owner_cannot_become_public(work, delivery): + from sqlalchemy import update + from sqlmodel import col + + from agent_manager.infrastructure.persistence.channels.tables import ChannelInboxRow + + item = await answered(work) + async with session_factory(work.database)() as session, session.begin(): + await session.exec( + update(ChannelInboxRow) + .where(col(ChannelInboxRow.event_id) == item.event_id) + .values(conversation_id=None) + ) + await delivery.materialize(item.event_id, ("internal answer",)) + assert await delivery.claim_next("connection", "system", NOW) is None + + +async def test_late_delivery_proof_recovers_a_failed_callback_without_resending_the_part( + work, delivery +): + item = await answered(work) + await delivery.materialize(item.event_id, ("a", "b")) + first = await delivery.claim_next("connection", "system", NOW) + await delivery.complete(first, "accepted", provider_id="wamid.a") + webhook = SqlWebhookRepository(session_factory(work.database)) + await webhook.accept( + "connection", + "system", + "f" * 64, + NOW, + WebhookBatch(delivery_events=(DeliveryEvent("wamid.a", "sender", "failed", NOW),)), + ) + await delivery.project("connection", "system") + assert await delivery.claim_next("connection", "system", NOW) is None + await webhook.accept( + "connection", + "system", + "d" * 64, + NOW, + WebhookBatch(delivery_events=(DeliveryEvent("wamid.a", "sender", "delivered", NOW),)), + ) + await delivery.project("connection", "system") + second = await delivery.claim_next("connection", "system", NOW) + assert second is not None and second.part_index == 1 + assert (await delivery.parts(item.event_id))[0].reason is None + + +async def test_reopened_storage_preserves_acceptance_and_holds_an_interrupted_next_part( + work, delivery +): + from agent_manager.infrastructure.persistence.database import create_db_engine + + item = await answered(work) + await delivery.materialize(item.event_id, ("a", "b")) + first = await delivery.claim_next("connection", "system", NOW) + await delivery.complete(first, "accepted", provider_id="wamid.a") + await delivery.claim_next("connection", "system", NOW) + url = str(work.database.url) + await work.database.dispose() + reopened = create_db_engine(url) + try: + repository = SqlDeliveryRepository(session_factory(reopened)) + await repository.recover_sending("connection", "system") + parts = await repository.parts(item.event_id) + assert [part.state for part in parts] == ["accepted", "delivery_unknown"] + assert parts[0].provider_message_id == "wamid.a" + assert await repository.claim_next("connection", "system", NOW) is None + finally: + await reopened.dispose() diff --git a/tests/agent_manager/test_channel_identity.py b/tests/agent_manager/test_channel_identity.py new file mode 100644 index 00000000..fe5bb068 --- /dev/null +++ b/tests/agent_manager/test_channel_identity.py @@ -0,0 +1,268 @@ +"""Internal channel turns require freshly resolved organization identity.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from dataclasses import replace +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from sqlalchemy.ext.asyncio import AsyncEngine +from sqlmodel import SQLModel, select + +from agent_manager.application.channels.preparation import ChannelTurnPreparer +from agent_manager.application.conversation_service import ConversationService +from agent_manager.domain import Principal +from agent_manager.domain.channels.identity import ( + ChannelIdentityDenied, + ChannelIdentityResolver, + ChannelIdentityUnavailable, +) +from agent_manager.domain.channels.models import ChannelAddress, InboundMessage, InboxState +from agent_manager.infrastructure.persistence.channels.repository import SqlChannelRepository +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelBindingRow, + ChannelInboxRow, + ChannelOutboxRow, +) +from agent_manager.infrastructure.persistence.database import create_db_engine, session_factory +from agent_manager.infrastructure.persistence.run_repository import SqlRunRepository +from agent_manager.infrastructure.persistence.sql_repository import SqlRepository +from tests.agent_manager.conftest import RecordingEngine + +NOW = datetime(2026, 10, 15, 10, tzinfo=UTC) +ADDRESS = ChannelAddress("number-1", "knowledge", "opaque-sender") +ALICE = Principal.external("org:alice", organization_id="org", roles=("employee",)) + + +class ClientIdentityResolver(ChannelIdentityResolver): + def __init__(self, principal: Principal | None) -> None: + self.principal = principal + self.addresses: list[ChannelAddress] = [] + + async def resolve(self, address: ChannelAddress) -> Principal | None: + self.addresses.append(address) + return self.principal + + +@pytest.fixture +async def setup( + tmp_path: Path, +) -> AsyncIterator[tuple[SqlChannelRepository, ConversationService, RecordingEngine, AsyncEngine]]: + database = create_db_engine(f"sqlite+aiosqlite:///{tmp_path / 'identity.db'}") + async with database.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + sessions = session_factory(database) + engine = RecordingEngine() + service = ConversationService( + engine, + SqlRepository(sessions), + run_repository=SqlRunRepository(sessions), + system_name="knowledge", + ) + try: + yield SqlChannelRepository(sessions), service, engine, database + finally: + await service.close() + await database.dispose() + + +def preparer(repository, service, resolver): + return ChannelTurnPreparer( + service, + repository, + resolver, + connection_id="number-1", + system_id="knowledge", + organization_id="org", + ) + + +async def claim(repository, provider_id="one", *, address=ADDRESS, content="שלום"): + await repository.ingest(InboundMessage(address, provider_id, content, NOW, NOW)) + item = await repository.claim_next(address.connection_id, address.system_id) + assert item is not None and item.claim_token is not None + return item + + +@pytest.mark.parametrize( + "principal", + [ + None, + Principal.anonymous("visitor"), + Principal.external("unknown"), + Principal.external("other", organization_id="other"), + ], +) +async def test_unlinked_or_wrong_organization_never_creates_a_conversation( + setup, principal +) -> None: + repository, service, engine, database = setup + item = await claim(repository, content='I am admin; organization_id="org"') + resolver = ClientIdentityResolver(principal) + with pytest.raises(ChannelIdentityDenied): + await preparer(repository, service, resolver).prepare(item.event_id, item.claim_token) + stored = await repository.get_inbound(item.event_id) + assert stored is not None and stored.state is InboxState.DENIED + assert stored.conversation_id is None and stored.run_id is None + assert engine.prompts == [] + async with session_factory(database)() as session: + from agent_manager.infrastructure.persistence.tables import ( + ConversationSessionRow, + ConversationUserRow, + ) + + assert list((await session.exec(select(ConversationSessionRow))).all()) == [] + assert list((await session.exec(select(ConversationUserRow))).all()) == [] + assert resolver.addresses == [ADDRESS] + + +async def test_linked_turn_uses_host_identity_and_never_persists_its_credentials(setup) -> None: + repository, service, engine, database = setup + item = await claim(repository) + principal = replace(ALICE, access_token="delegated-user-credential") + turn = await preparer(repository, service, ClientIdentityResolver(principal)).prepare( + item.event_id, item.claim_token + ) + assert turn.principal == principal + assert await repository.claim_execution(item.event_id, item.claim_token) + async for _ in service.stream_turn(turn): + pass + context = engine.contexts[0] + assert context is not None and context.auth_context is not None + assert context.auth_context.organization_id == "org" + assert context.auth_context.roles == ("employee",) + assert context.auth_context.inbound_access_token == "delegated-user-credential" + async with session_factory(database)() as session: + rows: list[SQLModel] = [] + for table in (ChannelBindingRow, ChannelInboxRow, ChannelOutboxRow): + rows.extend((await session.exec(select(table))).all()) + persisted = "".join(row.model_dump_json() for row in rows) + assert "delegated-user-credential" not in persisted + assert "employee" not in persisted + + +async def test_permissions_are_resolved_again_and_revoked_users_are_denied(setup) -> None: + repository, service, engine, _ = setup + resolver = ClientIdentityResolver(ALICE) + first = await claim(repository) + prepared = await preparer(repository, service, resolver).prepare( + first.event_id, first.claim_token + ) + await repository.finish(first.event_id, first.claim_token, InboxState.FAILED) + resolver.principal = None + second = await claim(repository, "two") + with pytest.raises(ChannelIdentityDenied): + await preparer(repository, service, resolver).prepare(second.event_id, second.claim_token) + assert len(resolver.addresses) == 2 + assert [m.content for m in await service.history(prepared.session_id, ALICE)] == ["שלום"] + assert engine.prompts == [] + + +async def test_sender_reassigned_to_another_user_cannot_read_the_old_conversation(setup) -> None: + repository, service, _, _ = setup + first = await claim(repository) + prepared = await preparer(repository, service, ClientIdentityResolver(ALICE)).prepare( + first.event_id, first.claim_token + ) + await repository.finish(first.event_id, first.claim_token, InboxState.FAILED) + bob = Principal.external("org:bob", organization_id="org") + second = await claim(repository, "two") + with pytest.raises(ChannelIdentityDenied): + await preparer(repository, service, ClientIdentityResolver(bob)).prepare( + second.event_id, second.claim_token + ) + assert (await service.list_conversations(bob)).items == [] + assert len(await service.history(prepared.session_id, ALICE)) == 1 + + +async def test_identity_backend_error_is_sanitized_and_keeps_work_for_review(setup, caplog) -> None: + repository, service, engine, _ = setup + + class BrokenResolver(ChannelIdentityResolver): + async def resolve(self, address: ChannelAddress) -> Principal | None: + raise OSError("sensitive-backend-token") + + item = await claim(repository) + with pytest.raises(ChannelIdentityUnavailable) as failure: + await preparer(repository, service, BrokenResolver()).prepare( + item.event_id, item.claim_token + ) + assert "sensitive-backend-token" not in str(failure.value) + assert "sensitive-backend-token" not in caplog.text + assert (await repository.get_inbound(item.event_id)).state is InboxState.NEEDS_REVIEW + assert engine.prompts == [] + + +async def test_queue_bookkeeping_failure_reuses_the_committed_manager_turn( + setup, monkeypatch +) -> None: + repository, service, _, _ = setup + item = await claim(repository) + prepare = preparer(repository, service, ClientIdentityResolver(ALICE)) + original = repository.record_prepared + + async def fail_record(*args, **kwargs): + raise OSError("queue unavailable") + + monkeypatch.setattr(repository, "record_prepared", fail_record) + with pytest.raises(OSError, match="queue unavailable"): + await prepare.prepare(item.event_id, item.claim_token) + binding = await repository.get_binding(ADDRESS) + assert binding is not None and binding.conversation_id is not None + user_message = (await service.history(binding.conversation_id, ALICE))[0] + monkeypatch.setattr(repository, "record_prepared", original) + recovered = await prepare.prepare(item.event_id, item.claim_token) + assert (recovered.message_id, recovered.run_id) == ( + user_message.message_id, + user_message.run_id, + ) + assert len(await service.history(recovered.session_id, ALICE)) == 1 + + +async def test_two_senders_bound_to_the_same_user_keep_separate_histories(setup) -> None: + repository, service, _, _ = setup + prepare = preparer(repository, service, ClientIdentityResolver(ALICE)) + first = await claim(repository) + one = await prepare.prepare(first.event_id, first.claim_token) + second = await claim( + repository, "two", address=replace(ADDRESS, sender_id="another"), content="second sender" + ) + two = await prepare.prepare(second.event_id, second.claim_token) + assert one.session_id != two.session_id + assert [m.content for m in await service.history(one.session_id, ALICE)] == ["שלום"] + assert [m.content for m in await service.history(two.session_id, ALICE)] == ["second sender"] + + +async def test_malformed_identity_result_fails_closed_with_a_controlled_error( + setup, caplog +) -> None: + repository, service, engine, _ = setup + + class InvalidResolver(ChannelIdentityResolver): + async def resolve(self, address: ChannelAddress): + return "invalid-principal" + + item = await claim(repository) + with pytest.raises(ChannelIdentityUnavailable): + await preparer(repository, service, InvalidResolver()).prepare( + item.event_id, item.claim_token + ) + assert (await repository.get_inbound(item.event_id)).state is InboxState.NEEDS_REVIEW + assert "invalid-principal" not in caplog.text + assert engine.prompts == [] + + +async def test_revocation_after_preparation_keeps_the_sender_blocked_for_review(setup) -> None: + repository, service, _, _ = setup + item = await claim(repository) + resolver = ClientIdentityResolver(ALICE) + prepare = preparer(repository, service, resolver) + turn = await prepare.prepare(item.event_id, item.claim_token) + resolver.principal = None + with pytest.raises(ChannelIdentityDenied): + await prepare.prepare(item.event_id, item.claim_token) + assert (await repository.get_inbound(item.event_id)).state is InboxState.NEEDS_REVIEW + assert len(await service.history(turn.session_id, ALICE)) == 1 + assert not await repository.claim_execution(item.event_id, item.claim_token) diff --git a/tests/agent_manager/test_channel_storage.py b/tests/agent_manager/test_channel_storage.py new file mode 100644 index 00000000..56de17ba --- /dev/null +++ b/tests/agent_manager/test_channel_storage.py @@ -0,0 +1,481 @@ +"""Durable channel work is isolated, ordered, and claimed once.""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from dataclasses import replace +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from sqlalchemy import event +from sqlalchemy.ext.asyncio import AsyncEngine +from sqlmodel import SQLModel +from sqlmodel.ext.asyncio.session import AsyncSession + +from agent_manager.composition import application_repositories +from agent_manager.config import Settings +from agent_manager.domain.channels.models import ( + ChannelAddress, + ChannelClaimConflict, + ChannelMessageConflict, + ChannelOwnerConflict, + InboundMessage, + InboxState, +) +from agent_manager.infrastructure.persistence.channels.repository import SqlChannelRepository +from agent_manager.infrastructure.persistence.database import create_db_engine, session_factory + +NOW = datetime(2026, 10, 15, 10, tzinfo=UTC) +ADDRESS = ChannelAddress("number-1", "knowledge", "opaque-sender") + + +@pytest.fixture +async def storage(tmp_path: Path) -> AsyncIterator[tuple[SqlChannelRepository, AsyncEngine]]: + database = create_db_engine(f"sqlite+aiosqlite:///{tmp_path / 'work.db'}") + async with database.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + try: + yield SqlChannelRepository(session_factory(database)), database + finally: + await database.dispose() + + +def incoming(provider_id: str, *, address: ChannelAddress = ADDRESS) -> InboundMessage: + return InboundMessage(address, provider_id, f"message:{provider_id}", NOW, NOW) + + +async def test_composed_storage_persists_and_recovers_an_inbound_message(tmp_path: Path) -> None: + url = f"sqlite+aiosqlite:///{tmp_path / 'channels.db'}" + settings = Settings.from_values(database_url=url) + database = create_db_engine(url) + async with database.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + await database.dispose() + + async with application_repositories(settings) as repositories: + channels = repositories.channels + assert channels is not None, "SQL composition must provide durable channel storage" + + original = await channels.ingest( + InboundMessage( + ChannelAddress("number-1", "knowledge", "opaque-sender"), + "provider-event-1", + "שלום", + NOW, + NOW, + ) + ) + + async with application_repositories(settings) as repositories: + assert repositories.channels is not None + recovered = await repositories.channels.get_inbound(original.event_id) + assert recovered is not None + assert recovered.message.content == "שלום" + assert recovered.message.address.sender_id == "opaque-sender" + + +async def test_process_memory_has_no_durable_channel_backend() -> None: + async with application_repositories( + Settings.from_values(database_url="sqlite+aiosqlite:///:memory:") + ) as repositories: + assert repositories.channels is None + + +async def test_duplicate_intake_keeps_one_event_and_original_receive_order(storage) -> None: + repository, _ = storage + first = await repository.ingest(incoming("event-1")) + duplicate = await repository.ingest( + replace(incoming("event-1"), received_at=NOW.replace(hour=11)) + ) + second = await repository.ingest(incoming("event-2")) + + assert duplicate == first + assert first.sequence < second.sequence + claimed = await repository.claim_next("number-1", "knowledge") + assert claimed is not None and claimed.event_id == first.event_id + assert await repository.claim_next("number-1", "knowledge") is None + + +@pytest.mark.parametrize("changed", ["sender", "system", "content", "kind", "sent_at"]) +async def test_provider_receipt_cannot_be_retargeted(storage, changed: str) -> None: + repository, _ = storage + original = incoming("one") + stored = await repository.ingest(original) + candidate = { + "sender": replace(original, address=replace(ADDRESS, sender_id="other")), + "system": replace(original, address=replace(ADDRESS, system_id="other")), + "content": replace(original, content="changed"), + "kind": replace(original, kind="reset"), + "sent_at": replace(original, sent_at=NOW.replace(hour=9)), + }[changed] + with pytest.raises(ChannelMessageConflict): + await repository.ingest(candidate) + assert await repository.get_inbound(stored.event_id) == stored + if changed in {"sender", "system"}: + assert await repository.get_binding(candidate.address) is None + + +async def test_concurrent_intake_and_claim_have_one_owner(storage) -> None: + repository, _ = storage + items = await asyncio.gather(*[repository.ingest(incoming("same")) for _ in range(8)]) + assert len({item.event_id for item in items}) == 1 + claims = await asyncio.gather( + *[repository.claim_next("number-1", "knowledge") for _ in range(8)] + ) + winners = [item for item in claims if item is not None] + assert len(winners) == 1 + assert winners[0].state is InboxState.PREPARING + assert winners[0].claim_token + + +async def test_active_sender_does_not_block_another_sender_or_connection(storage) -> None: + repository, _ = storage + one = await repository.ingest(incoming("same")) + await repository.ingest(incoming("later")) + other_address = replace(ADDRESS, sender_id="other") + other = await repository.ingest(incoming("different", address=other_address)) + other_connection = replace(ADDRESS, connection_id="number-2") + separate = await repository.ingest(incoming("same", address=other_connection)) + + first = await repository.claim_next("number-1", "knowledge") + second = await repository.claim_next("number-1", "knowledge") + third = await repository.claim_next("number-2", "knowledge") + assert first is not None and first.event_id == one.event_id + assert second is not None and second.event_id == other.event_id + assert third is not None and third.event_id == separate.event_id + + +async def test_stale_claim_cannot_bind_finish_or_prepare(storage) -> None: + repository, _ = storage + item = await repository.ingest(incoming("one")) + claimed = await repository.claim_next("number-1", "knowledge") + assert claimed is not None + with pytest.raises(ChannelClaimConflict): + await repository.bind_conversation(item.event_id, "wrong", "alice", "org", "session") + with pytest.raises(ChannelClaimConflict): + await repository.record_prepared(item.event_id, "wrong", "session", "run", "message") + with pytest.raises(ChannelClaimConflict): + await repository.finish(item.event_id, "wrong", InboxState.DENIED) + assert await repository.claim_execution(item.event_id, "wrong") is False + assert (await repository.get_inbound(item.event_id)).state is InboxState.PREPARING + + +async def prepare_claim(repository: SqlChannelRepository, provider_id: str = "one"): + await repository.ingest(incoming(provider_id)) + item = await repository.claim_next("number-1", "knowledge") + assert item is not None and item.claim_token is not None + await repository.bind_conversation(item.event_id, item.claim_token, "alice", "org", "session") + prepared = await repository.record_prepared( + item.event_id, item.claim_token, "session", "run", "message" + ) + return prepared + + +async def test_preparation_and_execution_are_distinct_one_way_claims(storage) -> None: + repository, _ = storage + prepared = await prepare_claim(repository) + token = prepared.claim_token + assert token is not None + replay = await repository.record_prepared(prepared.event_id, token, "session", "run", "message") + assert replay == prepared + with pytest.raises(ChannelClaimConflict): + await repository.record_prepared(prepared.event_id, token, "session", "another", "message") + results = await asyncio.gather( + *[repository.claim_execution(prepared.event_id, token) for _ in range(6)] + ) + assert results.count(True) == 1 + assert (await repository.get_inbound(prepared.event_id)).state is InboxState.RUNNING + + +async def test_ambiguous_work_blocks_later_turns_without_expiring(storage) -> None: + repository, _ = storage + prepared = await prepare_claim(repository) + assert prepared.claim_token is not None + await repository.ingest(incoming("later")) + await repository.finish(prepared.event_id, prepared.claim_token, InboxState.NEEDS_REVIEW) + assert await repository.claim_next("number-1", "knowledge") is None + assert not await repository.claim_execution(prepared.event_id, prepared.claim_token) + with pytest.raises(ChannelClaimConflict): + await repository.finish(prepared.event_id, prepared.claim_token, InboxState.DONE) + + +@pytest.mark.parametrize("changed", ["user", "organization", "conversation"]) +async def test_existing_binding_cannot_change_owner_or_conversation(storage, changed: str) -> None: + repository, _ = storage + prepared = await prepare_claim(repository) + assert prepared.claim_token is not None + user, organization, conversation = { + "user": ("bob", "org", "session"), + "organization": ("alice", "another", "session"), + "conversation": ("alice", "org", "different"), + }[changed] + with pytest.raises(ChannelOwnerConflict): + await repository.bind_conversation( + prepared.event_id, prepared.claim_token, user, organization, conversation + ) + binding = await repository.get_binding(ADDRESS) + assert binding is not None + assert (binding.user_id, binding.organization_id, binding.conversation_id) == ( + "alice", + "org", + "session", + ) + + +async def test_reset_is_ordered_and_advances_generation_once(storage) -> None: + repository, _ = storage + prepared = await prepare_claim(repository) + assert prepared.claim_token is not None + await repository.ingest(replace(incoming("reset"), kind="reset", content="/new")) + await repository.ingest(incoming("after-reset")) + assert await repository.claim_next("number-1", "knowledge") is None + await repository.finish(prepared.event_id, prepared.claim_token, InboxState.FAILED) + reset = await repository.claim_next("number-1", "knowledge") + assert reset is not None and reset.claim_token is not None + + assert await repository.reset_conversation(reset.event_id, reset.claim_token) == 1 + assert await repository.reset_conversation(reset.event_id, reset.claim_token) == 1 + binding = await repository.get_binding(ADDRESS) + assert binding is not None and binding.generation == 1 + assert binding.conversation_id is None and binding.user_id == "alice" + following = await repository.claim_next("number-1", "knowledge") + assert following is not None and following.message.provider_message_id == "after-reset" + assert (await repository.get_inbound(prepared.event_id)).conversation_id == "session" + + +async def test_reply_staging_is_ordered_immutable_and_releases_next_turn(storage) -> None: + repository, _ = storage + prepared = await prepare_claim(repository) + token = prepared.claim_token + assert token is not None + assert await repository.claim_execution(prepared.event_id, token) + await repository.ingest(incoming("later")) + replies = await repository.stage_reply(prepared.event_id, token, "assistant-1", ("שלום", "👋")) + replay = await repository.stage_reply(prepared.event_id, token, "assistant-1", ("שלום", "👋")) + assert replay == replies + assert [(part.part_index, part.text) for part in replies] == [(0, "שלום"), (1, "👋")] + assert len({part.reply_id for part in replies}) == 2 + with pytest.raises(ChannelMessageConflict): + await repository.stage_reply(prepared.event_id, token, "assistant-2", ("different",)) + with pytest.raises(ChannelMessageConflict): + await repository.stage_reply(prepared.event_id, token, "assistant-1", ("changed",)) + assert await repository.list_replies(prepared.event_id) == replies + following = await repository.claim_next("number-1", "knowledge") + assert following is not None and following.message.provider_message_id == "later" + + +async def test_outbox_failure_rolls_back_parts_and_keeps_sender_claim(storage) -> None: + repository, database = storage + prepared = await prepare_claim(repository) + assert prepared.claim_token is not None + assert await repository.claim_execution(prepared.event_id, prepared.claim_token) + + def fail_after_outbox_write(conn, cursor, statement, parameters, context, executemany): + if statement.startswith("INSERT INTO channel_outbox"): + raise OSError("outbox unavailable") + + event.listen(database.sync_engine, "after_cursor_execute", fail_after_outbox_write) + try: + with pytest.raises(OSError, match="outbox unavailable"): + await repository.stage_reply( + prepared.event_id, prepared.claim_token, "answer", ("a", "b") + ) + finally: + event.remove(database.sync_engine, "after_cursor_execute", fail_after_outbox_write) + assert await repository.list_replies(prepared.event_id) == () + assert (await repository.get_inbound(prepared.event_id)).state is InboxState.RUNNING + binding = await repository.get_binding(ADDRESS) + assert binding is not None and binding.active_event_id == prepared.event_id + + +async def test_reopening_storage_keeps_claims_and_reply_keys(tmp_path: Path) -> None: + url = f"sqlite+aiosqlite:///{tmp_path / 'restart.db'}" + database = create_db_engine(url) + async with database.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + repository = SqlChannelRepository(session_factory(database)) + prepared = await prepare_claim(repository) + assert prepared.claim_token is not None + assert await repository.claim_execution(prepared.event_id, prepared.claim_token) + replies = await repository.stage_reply( + prepared.event_id, prepared.claim_token, "answer", ("reply",) + ) + await database.dispose() + reopened = create_db_engine(url) + try: + repository = SqlChannelRepository(session_factory(reopened)) + assert await repository.list_replies(prepared.event_id) == replies + assert not await repository.claim_execution(prepared.event_id, prepared.claim_token) + finally: + await reopened.dispose() + + +async def test_concurrent_preparation_bookkeeping_reuses_one_receipt(storage) -> None: + repository, _ = storage + await repository.ingest(incoming("one")) + item = await repository.claim_next("number-1", "knowledge") + assert item is not None and item.claim_token is not None + await repository.bind_conversation(item.event_id, item.claim_token, "alice", "org", "session") + results = await asyncio.gather( + *[ + repository.record_prepared(item.event_id, item.claim_token, "session", "run", "message") + for _ in range(8) + ], + return_exceptions=True, + ) + assert all(not isinstance(result, Exception) for result in results), results + assert all(result == results[0] for result in results) + + +async def test_concurrent_resets_advance_only_once(storage) -> None: + repository, _ = storage + await repository.ingest(replace(incoming("reset"), kind="reset", content="/new")) + item = await repository.claim_next("number-1", "knowledge") + assert item is not None and item.claim_token is not None + generations = await asyncio.gather( + *[repository.reset_conversation(item.event_id, item.claim_token) for _ in range(8)] + ) + assert generations == [1] * 8 + assert (await repository.get_binding(ADDRESS)).generation == 1 + + +async def test_concurrent_reply_staging_cannot_duplicate_parts(storage) -> None: + repository, _ = storage + item = await prepare_claim(repository) + assert item.claim_token is not None + assert await repository.claim_execution(item.event_id, item.claim_token) + replies = await asyncio.gather( + *[ + repository.stage_reply(item.event_id, item.claim_token, "answer", ("a", "b")) + for _ in range(8) + ] + ) + assert all(result == replies[0] for result in replies) + assert len(await repository.list_replies(item.event_id)) == 2 + + +async def test_reply_cannot_be_staged_before_execution_claim(storage) -> None: + repository, _ = storage + item = await prepare_claim(repository) + assert item.claim_token is not None + with pytest.raises(ChannelClaimConflict): + await repository.stage_reply(item.event_id, item.claim_token, "answer", ("a",)) + assert await repository.list_replies(item.event_id) == () + assert (await repository.get_inbound(item.event_id)).state is InboxState.PREPARED + + +async def test_inbox_failure_rolls_back_the_receipt_and_binding(storage) -> None: + from sqlmodel import select + + from agent_manager.infrastructure.persistence.channels.tables import ChannelInboxRow + + repository, database = storage + + def fail_after_binding(conn, cursor, statement, parameters, context, executemany): + if statement.startswith("INSERT INTO channel_bindings"): + raise OSError("binding unavailable") + + event.listen(database.sync_engine, "after_cursor_execute", fail_after_binding) + try: + with pytest.raises(OSError, match="binding unavailable"): + await repository.ingest(incoming("one")) + finally: + event.remove(database.sync_engine, "after_cursor_execute", fail_after_binding) + assert await repository.get_binding(ADDRESS) is None + async with session_factory(database)() as session: + assert list((await session.exec(select(ChannelInboxRow))).all()) == [] + retried = await repository.ingest(incoming("one")) + assert retried.state is InboxState.QUEUED + + +@pytest.mark.parametrize( + "change", + [ + "empty_id", + "long_id", + "nul_id", + "large_text", + "empty_text", + "nul_text", + "naive_time", + "unknown_kind", + ], +) +def test_invalid_normalized_input_is_rejected_before_storage(change: str) -> None: + with pytest.raises(ValueError): + if change == "empty_id": + replace(ADDRESS, sender_id=" ") + elif change == "long_id": + replace(ADDRESS, sender_id="x" * 513) + elif change == "nul_id": + replace(ADDRESS, sender_id="private\x00sender") + elif change == "large_text": + replace(incoming("one"), content="ש" * 32769) + elif change == "empty_text": + replace(incoming("one"), content=" ") + elif change == "nul_text": + replace(incoming("one"), content="private\x00text") + elif change == "naive_time": + replace(incoming("one"), received_at=NOW.replace(tzinfo=None)) + else: + replace(incoming("one"), kind="unknown") # type: ignore[arg-type] # Unvalidated input. + + +async def test_unpersistable_reply_keeps_claim_and_writes_no_outbox_parts(storage) -> None: + repository, _ = storage + item = await prepare_claim(repository) + assert await repository.claim_execution(item.event_id, item.claim_token) + with pytest.raises(ValueError): + await repository.stage_reply( + item.event_id, item.claim_token, "answer", ("valid", "private\x00answer") + ) + assert await repository.list_replies(item.event_id) == () + assert (await repository.get_inbound(item.event_id)).state is InboxState.RUNNING + assert (await repository.get_binding(ADDRESS)).active_event_id == item.event_id + + +async def test_a_delayed_duplicate_reset_cannot_clear_a_newer_conversation( + storage, monkeypatch +) -> None: + repository, _ = storage + previous = await prepare_claim(repository) + assert previous.claim_token is not None + await repository.finish(previous.event_id, previous.claim_token, InboxState.FAILED) + await repository.ingest(replace(incoming("reset"), kind="reset", content="/new")) + reset = await repository.claim_next("number-1", "knowledge") + assert reset is not None and reset.claim_token is not None + ready = asyncio.Event() + release = asyncio.Event() + slow: asyncio.Task[int] | None = None + original_exec = AsyncSession.exec + + async def pause_before_reset_write(self, statement, *args, **kwargs): + if asyncio.current_task() is slow and str(statement).startswith("UPDATE channel_inbox"): + ready.set() + await release.wait() + return await original_exec(self, statement, *args, **kwargs) + + monkeypatch.setattr(AsyncSession, "exec", pause_before_reset_write) + slow = asyncio.create_task(repository.reset_conversation(reset.event_id, reset.claim_token)) + try: + await asyncio.wait_for(ready.wait(), timeout=2) + assert await repository.reset_conversation(reset.event_id, reset.claim_token) == 1 + await repository.ingest(incoming("after-reset")) + newer = await repository.claim_next("number-1", "knowledge") + assert newer is not None and newer.claim_token is not None + await repository.bind_conversation( + newer.event_id, newer.claim_token, "alice", "org", "new-session" + ) + release.set() + assert await asyncio.wait_for(slow, timeout=2) == 1 + finally: + release.set() + if not slow.done(): + slow.cancel() + await asyncio.gather(slow, return_exceptions=True) + binding = await repository.get_binding(ADDRESS) + assert binding is not None + assert binding.conversation_id == "new-session" + assert binding.active_event_id == newer.event_id diff --git a/tests/agent_manager/test_channel_worker.py b/tests/agent_manager/test_channel_worker.py new file mode 100644 index 00000000..952ea41a --- /dev/null +++ b/tests/agent_manager/test_channel_worker.py @@ -0,0 +1,681 @@ +"""Turn execution/recovery through manager ports and real local SQL.""" + +import asyncio +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass, replace +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from sqlalchemy.ext.asyncio import AsyncEngine +from sqlmodel import SQLModel + +from agent_engine.approvals.models import RunStatus +from agent_engine.engine.types import RunResult +from agent_engine.runtime.streaming import RunStreamEvent +from agent_manager.application.channels.preparation import ChannelTurnPreparer +from agent_manager.application.channels.worker_options import ChannelWorkerOptions +from agent_manager.application.conversation_service import ConversationService +from agent_manager.application.errors import ConversationAccessDenied +from agent_manager.domain import Principal +from agent_manager.domain.channels.identity import ChannelIdentityResolver +from agent_manager.domain.channels.models import ChannelAddress, InboundMessage, InboxState +from agent_manager.domain.channels.ownership import ( + ChannelWorkerOwnership, + ChannelWorkerOwnershipLost, +) +from agent_manager.infrastructure.persistence.channels.repository import SqlChannelRepository +from agent_manager.infrastructure.persistence.database import create_db_engine, session_factory +from agent_manager.infrastructure.persistence.run_repository import SqlRunRepository +from agent_manager.infrastructure.persistence.sql_repository import SqlRepository +from tests.agent_manager.conftest import RecordingEngine + +ALICE = Principal.external("alice", organization_id="org", roles=("employee",)) +ADDRESS = ChannelAddress("connection", "system", "sender") +OPTIONS = ChannelWorkerOptions("connection", "system", "org", poll_seconds=0.01, drain_seconds=0.1) +NOW = datetime(2026, 10, 16, tzinfo=UTC) + + +class Identities(ChannelIdentityResolver): + def __init__(self, *principals): + self.principals = list(principals or (ALICE,)) + self.calls = 0 + + async def resolve(self, address): + self.calls += 1 + value = self.principals[min(self.calls - 1, len(self.principals) - 1)] + if isinstance(value, Exception): + raise value + return value + + +@dataclass +class Work: + repository: SqlChannelRepository + service: ConversationService + engine: RecordingEngine + runs: SqlRunRepository + identities: Identities + database: AsyncEngine + + def preparer(self): + return ChannelTurnPreparer( + self.service, + self.repository, + self.identities, + connection_id="connection", + system_id="system", + organization_id="org", + ) + + def processor(self, options=OPTIONS): + from agent_manager.application.channels.processing import ChannelTurnProcessor + + return ChannelTurnProcessor( + self.service, self.repository, self.runs, self.preparer(), options + ) + + async def claim(self, provider_id="one", *, sender="sender", content="hello", kind="text"): + await self.repository.ingest( + InboundMessage(replace(ADDRESS, sender_id=sender), provider_id, content, NOW, NOW, kind) + ) + item = await self.repository.claim_next("connection", "system") + assert item is not None and item.claim_token is not None + return item + + +@pytest.fixture +async def work(tmp_path: Path) -> AsyncIterator[Work]: + database = create_db_engine(f"sqlite+aiosqlite:///{tmp_path / 'turns.db'}") + async with database.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + sessions = session_factory(database) + engine = RecordingEngine() + runs = SqlRunRepository(sessions) + service = ConversationService( + engine, SqlRepository(sessions), run_repository=runs, system_name="system" + ) + try: + yield Work(SqlChannelRepository(sessions), service, engine, runs, Identities(), database) + finally: + await service.close() + await database.dispose() + + +async def test_completed_response_lookup_checks_owner_and_preserves_the_original_answer(work): + conversation = await work.service.create(ALICE) + turn = await work.service.prepare_turn(conversation, "question", ALICE) + assert await work.service.get_turn_response(conversation, turn.run_id, ALICE) is None + await work.service.complete_turn( + turn, RunResult(system_name="system", visited=[], answer="stored answer") + ) + response = await work.service.get_turn_response(conversation, turn.run_id, ALICE) + assert response is not None and response.content == "stored answer" + assert response.parent_message_id == turn.message_id + with pytest.raises(ConversationAccessDenied): + await work.service.get_turn_response( + conversation, turn.run_id, Principal.external("other", organization_id="org") + ) + + +async def test_active_scan_is_bounded_ordered_and_skips_review(work): + first = await work.claim() + second = await work.claim("two", sender="another") + await work.repository.finish(first.event_id, first.claim_token, InboxState.NEEDS_REVIEW) + page = await work.repository.list_active("connection", "system", limit=1) + assert [item.event_id for item in page] == [second.event_id] + assert ( + await work.repository.list_active("connection", "system", after_sequence=second.sequence) + == () + ) + assert await work.repository.list_active("other", "system") == () + + +async def test_processing_executes_once_then_stages_the_persisted_response(work): + item = await work.claim() + await work.processor().process(item) + await work.processor().process(item) + assert work.engine.prompts == ["hello"] + assert (await work.repository.get_inbound(item.event_id)).state is InboxState.ANSWERED + reply = (await work.repository.list_replies(item.event_id))[0] + assert reply.text == "answer:hello" + assert work.identities.calls >= 2 + + +async def test_identity_is_refreshed_after_preparation_and_credentials_stay_request_scoped(work): + current = replace( + ALICE, roles=("current-permission",), access_token="test-only-current-credential" + ) + work.identities = Identities(ALICE, current) + await work.processor().process(await work.claim()) + context = work.engine.contexts[0] + assert context.auth_context.roles == ("current-permission",) + assert context.auth_context.inbound_access_token == "test-only-current-credential" + + +@pytest.mark.parametrize( + "principal", + [None, Principal.anonymous("visitor"), Principal.external("alice", organization_id="other")], +) +async def test_revocation_before_execution_blocks_prepared_work(work, principal): + work.identities = Identities(ALICE, principal) + item = await work.claim() + await work.processor().process(item) + assert work.engine.prompts == [] + assert (await work.repository.get_inbound(item.event_id)).state is InboxState.NEEDS_REVIEW + assert await work.repository.list_replies(item.event_id) == () + + +async def test_prepared_receipt_recovers_without_adding_another_user_message(work): + item = await work.claim() + prepared = await work.preparer().prepare(item.event_id, item.claim_token) + await work.processor().process(await work.repository.get_inbound(item.event_id)) + history = await work.service.history(prepared.session_id, ALICE) + assert [(message.role.value, message.content) for message in history] == [ + ("user", "hello"), + ("assistant", "answer:hello"), + ] + assert work.engine.prompts == ["hello"] + + +@pytest.mark.parametrize( + "status", [RunStatus.RUNNING, RunStatus.COMPLETED, RunStatus.FAILED, RunStatus.CANCELLED] +) +async def test_running_without_stored_answer_is_never_reexecuted(work, status): + item = await work.claim() + await work.preparer().prepare(item.event_id, item.claim_token) + assert await work.repository.claim_execution(item.event_id, item.claim_token) + if status is not RunStatus.RUNNING: + await work.runs.transition_if_allowed( + (await work.repository.get_inbound(item.event_id)).run_id, status + ) + await work.processor().process(await work.repository.get_inbound(item.event_id)) + assert work.engine.prompts == [] + assert (await work.repository.get_inbound(item.event_id)).state is InboxState.NEEDS_REVIEW + + +async def test_saved_answer_is_recovered_after_staging_failure_without_regeneration( + work, monkeypatch +): + item = await work.claim() + original = work.repository.stage_reply + + async def unavailable(*args, **kwargs): + raise OSError("private-storage-credential") + + monkeypatch.setattr(work.repository, "stage_reply", unavailable) + await work.processor().process(item) + assert (await work.repository.get_inbound(item.event_id)).state is InboxState.RUNNING + monkeypatch.setattr(work.repository, "stage_reply", original) + await work.processor().process(await work.repository.get_inbound(item.event_id)) + assert work.engine.prompts == ["hello"] + assert (await work.repository.list_replies(item.event_id))[0].text == "answer:hello" + + +async def test_recovery_of_internal_answer_still_requires_the_original_account_link(work): + item = await work.claim() + turn = await work.preparer().prepare(item.event_id, item.claim_token) + assert await work.repository.claim_execution(item.event_id, item.claim_token) + await work.service.complete_turn( + turn, RunResult(system_name="system", visited=[], answer="internal answer") + ) + work.identities = Identities(None) + await work.processor().process(await work.repository.get_inbound(item.event_id)) + assert await work.repository.list_replies(item.event_id) == () + assert (await work.repository.get_inbound(item.event_id)).state is InboxState.NEEDS_REVIEW + + +class Ownership(ChannelWorkerOwnership): + lost = False + + @asynccontextmanager + async def hold(self): + yield + + async def check(self): + if self.lost: + raise ChannelWorkerOwnershipLost("test ownership lost") + + +async def wait_state(work, event_id, state): + async with asyncio.timeout(2): + while (await work.repository.get_inbound(event_id)).state is not state: + await asyncio.sleep(0.005) + + +async def test_scheduler_keeps_sender_order_while_an_independent_sender_progresses( + work, monkeypatch +): + from agent_manager.application.channels.worker import ChannelWorker + + first = await work.claim(content="blocked") + second = await work.repository.ingest(InboundMessage(ADDRESS, "two", "next", NOW, NOW)) + another = await work.repository.ingest( + InboundMessage(replace(ADDRESS, sender_id="other"), "three", "independent", NOW, NOW) + ) + started, release = asyncio.Event(), asyncio.Event() + original = work.engine.stream + + async def stream(message, **kwargs): + if message == "blocked": + started.set() + await release.wait() + async for event in original(message, **kwargs): + yield event + + monkeypatch.setattr(work.engine, "stream", stream) + async with ChannelWorker(work.repository, work.processor(), Ownership(), OPTIONS): + await asyncio.wait_for(started.wait(), 2) + await wait_state(work, another.event_id, InboxState.ANSWERED) + assert (await work.repository.get_inbound(second.event_id)).state is InboxState.QUEUED + release.set() + await wait_state(work, first.event_id, InboxState.ANSWERED) + await wait_state(work, second.event_id, InboxState.ANSWERED) + assert work.engine.prompts.count("blocked") == work.engine.prompts.count("next") == 1 + + +async def test_shutdown_cancels_and_awaits_active_stream_and_blocks_reexecution(work, monkeypatch): + from agent_manager.application.channels.worker import ChannelWorker + + item = await work.claim() + started, closed = asyncio.Event(), asyncio.Event() + + async def stream(message, **kwargs): + try: + started.set() + await asyncio.Event().wait() + yield + finally: + closed.set() + + monkeypatch.setattr(work.engine, "stream", stream) + options = replace(OPTIONS, drain_seconds=0.02) + async with ChannelWorker(work.repository, work.processor(), Ownership(), options): + await asyncio.wait_for(started.wait(), 2) + assert closed.is_set() + assert (await work.repository.get_inbound(item.event_id)).state is InboxState.NEEDS_REVIEW + assert await work.repository.claim_next("connection", "system") is None + + +async def test_ownership_loss_aborts_active_execution_instead_of_continuing(work, monkeypatch): + from agent_manager.application.channels.worker import ChannelWorker + + item = await work.claim() + started, closed = asyncio.Event(), asyncio.Event() + + async def stream(message, **kwargs): + try: + started.set() + await asyncio.Event().wait() + yield + finally: + closed.set() + + monkeypatch.setattr(work.engine, "stream", stream) + ownership = Ownership() + with pytest.raises(ChannelWorkerOwnershipLost): + async with ChannelWorker(work.repository, work.processor(), ownership, OPTIONS): + await asyncio.wait_for(started.wait(), 2) + ownership.lost = True + await asyncio.wait_for(closed.wait(), 2) + assert (await work.repository.get_inbound(item.event_id)).state is InboxState.NEEDS_REVIEW + + +async def test_ownership_loss_reports_unhealthy_while_engine_cleanup_is_blocked(work, monkeypatch): + from agent_manager.application.channels.worker import ChannelWorker + + started, closing, release = (asyncio.Event() for _ in range(3)) + + async def stream(message, **kwargs): + try: + started.set() + await asyncio.Event().wait() + yield + finally: + closing.set() + await release.wait() + + monkeypatch.setattr(work.engine, "stream", stream) + await work.claim() + ownership = Ownership() + worker = ChannelWorker(work.repository, work.processor(), ownership, OPTIONS) + await worker.__aenter__() + try: + await asyncio.wait_for(started.wait(), 2) + ownership.lost = True + await asyncio.wait_for(closing.wait(), 2) + assert not worker.running + finally: + release.set() + with pytest.raises(ChannelWorkerOwnershipLost): + await worker.__aexit__(None, None, None) + + +class ApprovalEngine(RecordingEngine): + def __init__(self, runs, *, confirmation="confirmed", closing=None, release=None): + super().__init__() + self.runs = runs + self.confirmation = confirmation + self.closing = closing + self.release = release + self.cancelled = [] + self.executed = False + + async def stream(self, message, *, context=None, **kwargs): + self.prompts.append(message) + await self.runs.transition_if_allowed(context.run_id, RunStatus.PENDING_APPROVAL) + try: + yield RunStreamEvent( + type="pending_approval", + run_id=context.run_id, + approval_id="approval", + arguments={"token": "private-tool-credential"}, + ) + self.executed = True + yield RunStreamEvent(type="final", content="unexpected execution") + finally: + if self.closing is not None: + self.closing.set() + if self.release is not None: + await self.release.wait() + + async def cancel_pending_approval( + self, run_id, approval_id, *, caller_user_id=None, caller_session_id=None + ): + self.cancelled.append((run_id, approval_id, caller_user_id, caller_session_id)) + if self.confirmation == "error": + raise RuntimeError("private-cancellation-credential") + if self.confirmation == "confirmed": + await self.runs.transition_if_allowed(run_id, RunStatus.CANCELLED) + + +async def use_engine(work, engine): + await work.service.close() + work.engine = engine + work.service = ConversationService( + engine, + SqlRepository(session_factory(work.database)), + run_repository=work.runs, + system_name="system", + ) + + +@pytest.mark.parametrize("confirmation", ["confirmed", "unconfirmed", "error"]) +async def test_pending_approval_is_cancelled_or_held_for_review_never_approved( + work, confirmation, caplog +): + engine = ApprovalEngine(work.runs, confirmation=confirmation) + await use_engine(work, engine) + item = await work.claim() + await work.processor().process(item) + stored = await work.repository.get_inbound(item.event_id) + assert stored.state is ( + InboxState.FAILED if confirmation == "confirmed" else InboxState.NEEDS_REVIEW + ) + assert engine.executed is False + assert engine.cancelled == [(stored.run_id, "approval", ALICE.user_id, stored.conversation_id)] + assert await work.repository.list_replies(item.event_id) == () + assert "private-cancellation-credential" not in caplog.text + assert "private-tool-credential" not in caplog.text + + +async def test_approval_cancellation_keeps_sender_owned_until_stream_cleanup_finishes(work): + closing, release = asyncio.Event(), asyncio.Event() + engine = ApprovalEngine(work.runs, closing=closing, release=release) + await use_engine(work, engine) + first = await work.claim() + await work.repository.ingest(InboundMessage(ADDRESS, "two", "next", NOW, NOW)) + task = asyncio.create_task(work.processor().process(first)) + try: + await asyncio.wait_for(closing.wait(), 2) + assert await work.repository.claim_next("connection", "system") is None + finally: + release.set() + await task + assert await work.repository.claim_next("connection", "system") is not None + + +async def test_stream_without_final_answer_records_a_failed_run_and_review(work, monkeypatch): + async def stream(message, **kwargs): + yield RunStreamEvent(type="answer_delta", content="unfinished") + + monkeypatch.setattr(work.engine, "stream", stream) + item = await work.claim() + await work.processor().process(item) + stored = await work.repository.get_inbound(item.event_id) + assert stored.state is InboxState.NEEDS_REVIEW + assert (await work.runs.get(stored.run_id)).status is RunStatus.FAILED + + +async def test_safe_preparation_retries_its_receipt_after_queue_bookkeeping_failure( + work, monkeypatch +): + item = await work.claim() + original = work.repository.record_prepared + + async def unavailable(*args, **kwargs): + raise OSError("injected bookkeeping failure") + + monkeypatch.setattr(work.repository, "record_prepared", unavailable) + await work.processor().process(item) + assert (await work.repository.get_inbound(item.event_id)).state is InboxState.PREPARING + monkeypatch.setattr(work.repository, "record_prepared", original) + await work.processor().process(item) + binding = await work.repository.get_binding(ADDRESS) + assert len(await work.service.history(binding.conversation_id, ALICE)) == 2 + assert work.engine.prompts == ["hello"] + + +async def test_reopened_manager_recovers_a_saved_answer_without_starting_the_engine(work): + item = await work.claim() + turn = await work.preparer().prepare(item.event_id, item.claim_token) + assert await work.repository.claim_execution(item.event_id, item.claim_token) + await work.service.complete_turn( + turn, RunResult(system_name="system", visited=[], answer="committed before crash") + ) + await work.service.close() + url = work.database.url.render_as_string(hide_password=False) + await work.database.dispose() + database = create_db_engine(url) + sessions = session_factory(database) + engine = RecordingEngine() + runs = SqlRunRepository(sessions) + service = ConversationService( + engine, SqlRepository(sessions), run_repository=runs, system_name="system" + ) + reopened = Work(SqlChannelRepository(sessions), service, engine, runs, Identities(), database) + try: + recovered = await reopened.repository.get_inbound(item.event_id) + await reopened.processor().process(recovered) + original_reply = await reopened.repository.list_replies(item.event_id) + await reopened.processor().process(recovered) + assert await reopened.repository.list_replies(item.event_id) == original_reply + assert original_reply[0].text == "committed before crash" + assert engine.prompts == [] + finally: + await service.close() + await database.dispose() + + +@pytest.mark.parametrize("content", ["", "x" * 65_537], ids=["empty", "oversized"]) +async def test_unusable_answer_requires_review_and_is_never_truncated(work, monkeypatch, content): + async def stream(message, **kwargs): + yield RunStreamEvent(type="final", content=content) + + monkeypatch.setattr(work.engine, "stream", stream) + item = await work.claim() + await work.processor().process(item) + stored = await work.repository.get_inbound(item.event_id) + assert stored.state is InboxState.NEEDS_REVIEW + assert await work.repository.list_replies(item.event_id) == () + response = await work.service.get_turn_response(stored.conversation_id, stored.run_id, ALICE) + assert response.content == content + + +async def test_late_engine_error_recovers_the_persisted_answer_without_leaking_error_text( + work, monkeypatch, caplog +): + original = work.engine.stream + + async def stream(message, **kwargs): + async for event in original(message, **kwargs): + yield event + raise OSError("private-engine-credential") + + monkeypatch.setattr(work.engine, "stream", stream) + item = await work.claim() + await work.processor().process(item) + assert work.engine.prompts == ["hello"] + assert (await work.repository.list_replies(item.event_id))[0].text == "answer:hello" + assert "private-engine-credential" not in caplog.text + + +async def test_timeout_closes_the_engine_and_prevents_execution_retry(work, monkeypatch): + closed = asyncio.Event() + + async def stream(message, **kwargs): + try: + yield RunStreamEvent(type="answer_delta", content="partial") + await asyncio.Event().wait() + finally: + closed.set() + + monkeypatch.setattr(work.engine, "stream", stream) + item = await work.claim() + await work.preparer().prepare(item.event_id, item.claim_token) + await work.processor(replace(OPTIONS, turn_seconds=1)).process(item) + stored = await work.repository.get_inbound(item.event_id) + assert stored.state is InboxState.NEEDS_REVIEW and closed.is_set() + assert (await work.runs.get(stored.run_id)).status is RunStatus.CANCELLED + + +async def test_unsupported_media_does_no_agent_work_and_does_not_block_next_text(work): + item = await work.claim(content="", kind="unsupported") + await work.processor().process(item) + assert work.engine.prompts == [] + assert (await work.repository.get_inbound(item.event_id)).state is InboxState.FAILED + next_item = await work.claim("next", content="next text") + await work.processor().process(next_item) + assert work.engine.prompts == ["next text"] + + +async def test_shutdown_waits_for_slow_engine_cleanup_before_releasing_ownership(work, monkeypatch): + from agent_manager.application.channels.worker import ChannelWorker + + started, closing, release, closed = (asyncio.Event() for _ in range(4)) + + async def stream(message, **kwargs): + try: + started.set() + await asyncio.Event().wait() + yield + finally: + closing.set() + await release.wait() + closed.set() + + monkeypatch.setattr(work.engine, "stream", stream) + await work.claim() + worker = ChannelWorker( + work.repository, work.processor(), Ownership(), replace(OPTIONS, drain_seconds=0.01) + ) + await worker.__aenter__() + await asyncio.wait_for(started.wait(), 2) + stopped = asyncio.create_task(worker.__aexit__(None, None, None)) + try: + await asyncio.wait_for(closing.wait(), 2) + with pytest.raises(TimeoutError): + await asyncio.wait_for(asyncio.shield(stopped), 1.2) + assert not closed.is_set() + finally: + release.set() + await stopped + assert closed.is_set() + + +async def test_shutdown_does_not_start_a_claim_that_returns_after_stopping(work, monkeypatch): + from agent_manager.application.channels.worker import ChannelWorker + + item = await work.repository.ingest(InboundMessage(ADDRESS, "delayed", "queued", NOW, NOW)) + claimed, release = asyncio.Event(), asyncio.Event() + original = work.repository.claim_next + + async def delayed_claim(*args, **kwargs): + result = await original(*args, **kwargs) + if result is not None: + claimed.set() + await release.wait() + return result + + monkeypatch.setattr(work.repository, "claim_next", delayed_claim) + worker = ChannelWorker(work.repository, work.processor(), Ownership(), OPTIONS) + await worker.__aenter__() + await asyncio.wait_for(claimed.wait(), 2) + stopped = asyncio.create_task(worker.__aexit__(None, None, None)) + try: + async with asyncio.timeout(2): + while worker.running: + await asyncio.sleep(0) + finally: + release.set() + await stopped + assert work.engine.prompts == [] + assert (await work.repository.get_inbound(item.event_id)).state is InboxState.PREPARING + monkeypatch.setattr(work.repository, "claim_next", original) + async with ChannelWorker(work.repository, work.processor(), Ownership(), OPTIONS): + await wait_state(work, item.event_id, InboxState.ANSWERED) + assert work.engine.prompts == ["queued"] + + +@pytest.mark.parametrize("cancellations", [1, 2]) +async def test_cancelled_shutdown_retains_real_ownership_until_stream_cleanup( + work, monkeypatch, cancellations +): + from agent_manager.application.channels.worker import ChannelWorker + from agent_manager.domain.channels.ownership import ChannelWorkerAlreadyRunning + from agent_manager.infrastructure.persistence.channels.ownership import ( + SqlChannelWorkerOwnership, + ) + + started, closing, release = (asyncio.Event() for _ in range(3)) + + async def stream(message, **kwargs): + try: + started.set() + await asyncio.Event().wait() + yield + finally: + closing.set() + await release.wait() + + monkeypatch.setattr(work.engine, "stream", stream) + item = await work.claim() + url = str(work.database.url) + worker = ChannelWorker( + work.repository, + work.processor(), + SqlChannelWorkerOwnership(url, "connection"), + replace(OPTIONS, drain_seconds=0.01), + ) + await worker.__aenter__() + await asyncio.wait_for(started.wait(), 2) + stopped = asyncio.create_task(worker.__aexit__(None, None, None)) + try: + await asyncio.wait_for(closing.wait(), 2) + for _ in range(cancellations): + stopped.cancel() + await asyncio.wait((stopped,), timeout=0.02) + with pytest.raises(ChannelWorkerAlreadyRunning): + async with SqlChannelWorkerOwnership(url, "connection").hold(): + pytest.fail("ownership released while old engine cleanup was still active") + assert not stopped.done() + finally: + release.set() + await asyncio.gather(stopped, return_exceptions=True) + # Also join the public shutdown path if the regression detached it. + await worker.__aexit__(None, None, None) + assert stopped.cancelled() + assert (await work.repository.get_inbound(item.event_id)).state is InboxState.NEEDS_REVIEW + async with SqlChannelWorkerOwnership(url, "connection").hold(): + pass diff --git a/tests/agent_manager/test_channel_worker_lifecycle.py b/tests/agent_manager/test_channel_worker_lifecycle.py new file mode 100644 index 00000000..7406c561 --- /dev/null +++ b/tests/agent_manager/test_channel_worker_lifecycle.py @@ -0,0 +1,236 @@ +"""Opt-in worker lifecycle through the real manager HTTP and file-SQL wiring.""" + +import asyncio +import time +from dataclasses import replace +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from pydantic import ValidationError +from sqlmodel import SQLModel + +from agent_manager.api.app import create_app +from agent_manager.config import Settings +from agent_manager.domain import Principal +from agent_manager.domain.channels.identity import ChannelIdentityResolver +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelInboxRow, + ChannelOutboxRow, +) +from agent_manager.infrastructure.persistence.database import create_db_engine +from tests.agent_manager.test_whatsapp_webhook import ( + ENDPOINT, + StartupEngine, + change, + encoded, + message, + payload, + saved_rows, + settings_values, + signed, +) + +PLUGIN = "tests.agent_manager.test_channel_worker_lifecycle:OrganizationIdentityResolver" +ALICE = Principal.external("alice", organization_id="org", roles=("employee",)) + + +class OrganizationIdentityResolver(ChannelIdentityResolver): + closed = False + instances = 0 + + def __init__(self): + type(self).instances += 1 + + async def resolve(self, address): + return replace(ALICE, access_token="test-only-delegated-token") + + async def close(self): + type(self).closed = True + + +class BrokenIdentityResolver(ChannelIdentityResolver): + def __init__(self): + raise RuntimeError("private-plugin-credential") + + async def resolve(self, address): + return None + + +class CleanupErrorResolver(OrganizationIdentityResolver): + async def close(self): + raise RuntimeError("private-cleanup-credential") + + +class SyncCleanupResolver(OrganizationIdentityResolver): + def close(self): + pass + + +def worker_values(**overrides): + return settings_values( + extra_whatsapp_worker_enabled=True, + extra_whatsapp_organization_id="org", + extra_whatsapp_identity_resolver=PLUGIN, + extra_whatsapp_worker_poll_seconds=0.02, + **overrides, + ) + + +@pytest.mark.parametrize( + "overrides", + [ + {"extra_whatsapp_organization_id": None}, + {"extra_whatsapp_identity_resolver": None}, + {"extra_whatsapp_enabled": False}, + {"extra_whatsapp_identity_resolver": "bad import path"}, + ], +) +def test_enabling_worker_requires_explicit_identity_and_intake(overrides): + with pytest.raises(ValidationError): + Settings.from_values(**{**worker_values(), **overrides}) + + +def test_worker_consumes_once_and_keeps_the_next_turns_history(tmp_path: Path, monkeypatch): + url = f"sqlite+aiosqlite:///{tmp_path / 'worker.db'}" + + async def schema(): + database = create_db_engine(url) + async with database.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + await database.dispose() + + asyncio.run(schema()) + engine = StartupEngine() + monkeypatch.setattr("agent_manager.api.app.LangGraphEngine", lambda *a, **kw: engine) + config = ( + Path(__file__).resolve().parents[2] / "examples/enterprise-knowledge-assistant/agents.yaml" + ) + settings = Settings.from_values(database_url=url, **worker_values()) + app = create_app(str(config), settings) + OrganizationIdentityResolver.closed = False + OrganizationIdentityResolver.instances = 0 + with TestClient(app) as client: + first = encoded(payload(change(message("one", text={"body": "first"})))) + assert client.post(ENDPOINT, content=first, headers=signed(first)).status_code == 200 + assert client.post(ENDPOINT, content=first, headers=signed(first)).status_code == 200 + wait_for_answers(url, 1) + second = encoded(payload(change(message("two", text={"body": "second"})))) + assert client.post(ENDPOINT, content=second, headers=signed(second)).status_code == 200 + wait_for_answers(url, 2) + assert engine.prompts == ["first", "second"] + assert [(m.role, m.content) for m in engine.histories[1]] == [ + ("user", "first"), + ("assistant", "answer:first"), + ] + assert engine.builds == 1 + for context in engine.contexts: + assert context is not None and context.auth_context is not None + assert context.auth_context.inbound_access_token == "test-only-delegated-token" + persisted = "".join(row.model_dump_json() for row in saved_rows(url, ChannelInboxRow)) + assert "test-only-delegated-token" not in persisted + assert OrganizationIdentityResolver.closed + assert OrganizationIdentityResolver.instances == 1 + assert app.state.channel_worker.running is False + + +def wait_for_answers(url, count): + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + rows = saved_rows(url, ChannelOutboxRow) + if len(rows) == count: + return + time.sleep(0.01) + pytest.fail(f"worker did not stage {count} answers") + + +@pytest.mark.parametrize( + "plugin", + [ + "not_an_installed_module:Resolver", + "tests.agent_manager.test_channel_worker_lifecycle:BrokenIdentityResolver", + "tests.agent_manager.test_channel_worker_lifecycle:StartupEngine", + "tests.agent_manager.test_channel_worker_lifecycle:SyncCleanupResolver", + ], +) +def test_invalid_identity_plugin_fails_startup_without_secret_text( + tmp_path: Path, monkeypatch, plugin +): + monkeypatch.setattr("agent_manager.api.app.LangGraphEngine", lambda *a, **kw: StartupEngine()) + config = ( + Path(__file__).resolve().parents[2] / "examples/enterprise-knowledge-assistant/agents.yaml" + ) + settings = Settings.from_values( + database_url=f"sqlite+aiosqlite:///{tmp_path / 'invalid.db'}", + **{**worker_values(), "extra_whatsapp_identity_resolver": plugin}, + ) + with pytest.raises(ValueError) as error, TestClient(create_app(str(config), settings)): + pass + assert "private-plugin-credential" not in str(error.value) + + +def test_identity_cleanup_error_is_sanitized_without_preventing_manager_shutdown( + tmp_path, monkeypatch, caplog +): + url = f"sqlite+aiosqlite:///{tmp_path / 'cleanup.db'}" + + async def schema(): + database = create_db_engine(url) + async with database.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + await database.dispose() + + asyncio.run(schema()) + monkeypatch.setattr("agent_manager.api.app.LangGraphEngine", lambda *a, **kw: StartupEngine()) + config = ( + Path(__file__).resolve().parents[2] / "examples/enterprise-knowledge-assistant/agents.yaml" + ) + settings = Settings.from_values( + database_url=url, + **{ + **worker_values(), + "extra_whatsapp_identity_resolver": ( + "tests.agent_manager.test_channel_worker_lifecycle:CleanupErrorResolver" + ), + }, + ) + with TestClient(create_app(str(config), settings)) as client: + assert client.get("/health").status_code == 200 + assert "private-cleanup-credential" not in caplog.text + + +def test_failed_supervisor_is_reported_as_unhealthy(tmp_path, monkeypatch): + from agent_manager.domain.channels.ownership import ChannelWorkerOwnershipLost + from agent_manager.infrastructure.persistence.channels.ownership import ( + SqlChannelWorkerOwnership, + ) + + url = f"sqlite+aiosqlite:///{tmp_path / 'health.db'}" + + async def schema(): + database = create_db_engine(url) + async with database.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + await database.dispose() + + asyncio.run(schema()) + monkeypatch.setattr("agent_manager.api.app.LangGraphEngine", lambda *a, **kw: StartupEngine()) + config = ( + Path(__file__).resolve().parents[2] / "examples/enterprise-knowledge-assistant/agents.yaml" + ) + app = create_app(str(config), Settings.from_values(database_url=url, **worker_values())) + + async def lost(self): + raise ChannelWorkerOwnershipLost("injected ownership loss") + + response = None + with pytest.raises(ChannelWorkerOwnershipLost), TestClient(app) as client: + monkeypatch.setattr(SqlChannelWorkerOwnership, "check", lost) + deadline = time.monotonic() + 2 + while time.monotonic() < deadline: + response = client.get("/health") + if response.status_code == 503: + break + time.sleep(0.01) + assert response is not None and response.status_code == 503 + assert response.json()["status"] == "degraded" diff --git a/tests/agent_manager/test_whatsapp_delivery_lifecycle.py b/tests/agent_manager/test_whatsapp_delivery_lifecycle.py new file mode 100644 index 00000000..3efd5f35 --- /dev/null +++ b/tests/agent_manager/test_whatsapp_delivery_lifecycle.py @@ -0,0 +1,256 @@ +"""Opt-in composition: signed intake to a fake provider, with one shared engine.""" + +import asyncio +import time +from dataclasses import replace +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from pydantic import ValidationError +from sqlmodel import SQLModel + +from agent_manager.api.app import create_app +from agent_manager.application.channels.worker import ChannelWorker +from agent_manager.config import Settings +from agent_manager.domain.channels.delivery import SendOutcome +from agent_manager.domain.channels.ownership import ChannelWorkerAlreadyRunning +from agent_manager.infrastructure.persistence.channels.ownership import SqlChannelWorkerOwnership +from agent_manager.infrastructure.persistence.database import create_db_engine +from tests.agent_manager.test_channel_delivery import Sender, pump +from tests.agent_manager.test_channel_delivery_storage import answered +from tests.agent_manager.test_channel_delivery_storage import delivery as delivery +from tests.agent_manager.test_channel_worker import OPTIONS +from tests.agent_manager.test_channel_worker import work as work +from tests.agent_manager.test_channel_worker_lifecycle import worker_values +from tests.agent_manager.test_whatsapp_webhook import ( + ENDPOINT, + StartupEngine, + change, + encoded, + message, + payload, + signed, +) + + +def outbound_values(**overrides): + return { + **worker_values(), + "extra_whatsapp_sender_enabled": True, + "extra_whatsapp_api_version": "v99.0", + "extra_whatsapp_access_token": "test-outbound-secret", + "extra_whatsapp_support_contact": "support@example.test", + **overrides, + } + + +@pytest.mark.parametrize( + "override", + [ + {"extra_whatsapp_worker_enabled": False}, + {"extra_whatsapp_access_token": None}, + {"extra_whatsapp_api_version": None}, + {"extra_whatsapp_api_version": "latest"}, + {"extra_whatsapp_support_contact": None}, + {"extra_whatsapp_access_token": "test-only-whatsapp-app-secret"}, + ], +) +def test_sender_enablement_requires_explicit_safe_configuration(override): + with pytest.raises(ValidationError): + Settings.from_values(**outbound_values(**override)) + + +def test_outbound_token_is_excluded_from_settings_representations(): + settings = Settings.from_values(**outbound_values()) + secret = settings.extra_whatsapp_access_token + assert secret is not None and secret.get_secret_value() == "test-outbound-secret" + assert "test-outbound-secret" not in repr(settings) + assert "extra_whatsapp_access_token" not in settings.model_dump() + + +def test_signed_intake_runs_once_sends_answer_and_handles_help_without_model( + tmp_path: Path, monkeypatch +): + url = f"sqlite+aiosqlite:///{tmp_path / 'roundtrip.db'}" + + async def schema(): + database = create_db_engine(url) + try: + async with database.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + finally: + await database.dispose() + + asyncio.run(schema()) + engine = StartupEngine() + + class Provider: + def __init__(self, *args, **kwargs): + self.calls = [] + self.closed = False + + async def send(self, recipient, text): + self.calls.append((recipient, text)) + return SendOutcome("accepted", f"wamid.{len(self.calls)}") + + async def close(self): + self.closed = True + + provider = Provider() + monkeypatch.setattr("agent_manager.api.app.LangGraphEngine", lambda *a, **kw: engine) + monkeypatch.setattr("agent_manager.composition.WhatsAppSender", lambda *a, **kw: provider) + settings = Settings.from_values(**outbound_values(database_url=url)) + # Keep the plugin namespace shared with the other in-process HTTP fixtures. + config = Path(__file__).parents[2] / "examples/enterprise-knowledge-assistant/agents.yaml" + app = create_app(str(config), settings) + with TestClient(app) as client: + for provider_id, body, expected in ( + ("question", "hello", "answer:hello"), + ("help", "/help", "support@example.test"), + ): + raw = encoded(payload(change(message(provider_id, text={"body": body})))) + assert client.post(ENDPOINT, content=raw, headers=signed(raw)).status_code == 200 + deadline = time.monotonic() + 3 + while not any(expected in text for _, text in provider.calls): + assert time.monotonic() < deadline + time.sleep(0.01) + assert client.get("/health").status_code == 200 + assert engine.builds == 1 and engine.prompts == ["hello"] + assert provider.closed + + +async def test_shutdown_keeps_ownership_until_send_cleanup_and_unknown_receipt_finish( + work, delivery +): + started, closing, release = (asyncio.Event() for _ in range(3)) + + class Provider(Sender): + async def send(self, recipient, text): + try: + started.set() + await asyncio.Event().wait() + finally: + closing.set() + await release.wait() + + item = await answered(work) + url = str(work.database.url) + worker = ChannelWorker( + work.repository, + work.processor(), + SqlChannelWorkerOwnership(url, "connection"), + replace(OPTIONS, drain_seconds=0.01), + delivery=pump(work, delivery, Provider()), + ) + await worker.__aenter__() + await asyncio.wait_for(started.wait(), 2) + stopped = asyncio.create_task(worker.__aexit__(None, None, None)) + try: + await asyncio.wait_for(closing.wait(), 2) + with pytest.raises(ChannelWorkerAlreadyRunning): + async with SqlChannelWorkerOwnership(url, "connection").hold(): + pytest.fail("send cleanup lost ownership") + finally: + release.set() + await stopped + assert (await delivery.parts(item.event_id))[0].state == "delivery_unknown" + async with SqlChannelWorkerOwnership(url, "connection").hold(): + pass + + +async def test_ownership_failure_before_delivery_is_fatal_without_waiting_for_next_poll( + work, delivery +): + from contextlib import suppress + + from agent_manager.domain.channels.ownership import ChannelWorkerOwnershipLost + from tests.agent_manager.test_channel_worker import Ownership + + class LostBeforeSend(Ownership): + calls = 0 + + async def check(self): + self.calls += 1 + if self.calls >= 3: + raise ChannelWorkerOwnershipLost("injected ownership loss") + + await answered(work) + sender = Sender() + worker = ChannelWorker( + work.repository, + work.processor(), + LostBeforeSend(), + replace(OPTIONS, poll_seconds=60), + delivery=pump(work, delivery, sender), + ) + await worker.__aenter__() + try: + async with asyncio.timeout(1): + while worker.running: + await asyncio.sleep(0.005) + assert sender.calls == [] + finally: + with suppress(ChannelWorkerOwnershipLost): + await worker.__aexit__(None, None, None) + + +@pytest.mark.parametrize( + "sender_enabled,blocked_client", [(True, "sender"), (True, "identity"), (False, "identity")] +) +async def test_lifespan_joins_client_cleanup_through_repeated_cancellation( + work, monkeypatch, sender_enabled, blocked_client +): + from agent_manager.composition import application_repositories, channel_worker_lifespan + from tests.agent_manager.test_channel_worker import Identities + + ready, shutdown, closing, release = (asyncio.Event() for _ in range(4)) + closed = [] + + async def close_client(name): + if name == blocked_client: + closing.set() + await release.wait() + closed.append(name) + + class Identity(Identities): + async def close(self): + await close_client("identity") + + class Provider(Sender): + async def close(self): + await close_client("sender") + + monkeypatch.setattr( + "agent_manager.composition.load_channel_identity_resolver", lambda _: Identity() + ) + monkeypatch.setattr("agent_manager.composition.WhatsAppSender", lambda *a: Provider()) + settings = Settings.from_values( + **outbound_values( + database_url=str(work.database.url), extra_whatsapp_sender_enabled=sender_enabled + ) + ) + + async def lifetime(): + async with ( + application_repositories(settings) as repositories, + channel_worker_lifespan(settings, repositories, work.service, "system"), + ): + ready.set() + await shutdown.wait() + + task = asyncio.create_task(lifetime()) + try: + await asyncio.wait_for(ready.wait(), 2) + shutdown.set() + await asyncio.wait_for(closing.wait(), 2) + for _ in range(2): + task.cancel() + done, _ = await asyncio.wait({task}, timeout=0.02) + assert not done, "lifespan returned while a channel client was still closing" + assert blocked_client not in closed + finally: + release.set() + result = (await asyncio.gather(task, return_exceptions=True))[0] + assert isinstance(result, asyncio.CancelledError) + assert closed == (["sender", "identity"] if sender_enabled else ["identity"]) diff --git a/tests/agent_manager/test_whatsapp_normalization.py b/tests/agent_manager/test_whatsapp_normalization.py new file mode 100644 index 00000000..9a355b7d --- /dev/null +++ b/tests/agent_manager/test_whatsapp_normalization.py @@ -0,0 +1,163 @@ +"""Malformed input isolation and conservative provider-contract normalization.""" + +from dataclasses import replace +from datetime import UTC, datetime + +import pytest +from pydantic import SecretStr + +from agent_manager.infrastructure.channels.whatsapp.config import WhatsAppWebhookConfig +from agent_manager.infrastructure.channels.whatsapp.normalization import normalize +from tests.agent_manager.test_whatsapp_webhook import change, encoded, message, payload + +NOW = datetime(2026, 10, 15, 12, tzinfo=UTC) +CONFIG = WhatsAppWebhookConfig( + "connection", "system", "account-1", "number-1", SecretStr("test-app"), SecretStr("test-verify") +) + + +def inbound(**overrides): + return message(timestamp=str(int(NOW.timestamp())), **overrides) + + +def test_missing_entry_or_missing_entry_identity_is_quarantined(): + for data in ( + {"object": "whatsapp_business_account"}, + {"object": "whatsapp_business_account", "entry": [{"changes": []}]}, + ): + batch = normalize(encoded(data), CONFIG, NOW) + assert batch.issues and not batch.messages + + +def test_invalid_message_collection_keeps_valid_delivery_events(): + data = payload( + change( + statuses=[ + { + "id": "outbound", + "recipient_id": "sender", + "status": "read", + "timestamp": str(int(NOW.timestamp())), + } + ] + ) + ) + data["entry"][0]["changes"][0]["value"]["messages"] = "invalid collection" + batch = normalize(encoded(data), CONFIG, NOW) + assert len(batch.delivery_events) == 1 + assert batch.issues and not batch.messages + + +def test_invalid_status_collection_keeps_valid_messages(): + data = payload(change(inbound())) + data["entry"][0]["changes"][0]["value"]["statuses"] = {} + batch = normalize(encoded(data), CONFIG, NOW) + assert len(batch.messages) == 1 + assert batch.issues and not batch.delivery_events + + +@pytest.mark.parametrize( + "changed", + [ + {"from": ""}, + {"from": " "}, + {"from": "private\x00sender"}, + {"from": "🙂" * 129}, + {"id": 123}, + {"id": ""}, + {"id": "private\x00message"}, + {"text": None}, + {"text": {"body": " "}}, + {"text": {"body": "private\x00text"}}, + {"text": {"body": 123}}, + {"text": {"body": "🙂" * 20_000}}, + {"timestamp": "invalid"}, + {"timestamp": "99999999999999999999"}, + {"timestamp": str(int(NOW.timestamp()) + 3600)}, + ], +) +def test_invalid_message_is_quarantined_without_private_input(changed): + data = payload(change({**inbound(), **changed}, inbound(id="second"))) + batch = normalize(encoded(data), CONFIG, NOW) + assert [item.provider_message_id for item in batch.messages] == ["second"] + assert [(issue.location, issue.reason) for issue in batch.issues] == [ + ("entry.0.changes.0.messages.0", "invalid_message") + ] + + +def test_unsupported_media_is_not_downloaded_or_promoted_to_agent_text(): + data = payload( + change( + inbound( + type="image", + image={"link": "https://private.invalid/image", "caption": "private caption"}, + text=None, + ) + ) + ) + batch = normalize(encoded(data), CONFIG, NOW) + assert len(batch.messages) == 1 + assert batch.messages[0].kind == "unsupported" and batch.messages[0].content == "" + assert not batch.issues + + +def test_unverified_new_sender_contract_is_quarantined_instead_of_guessed(): + data = inbound(from_user_id="new-business-scoped-id") + del data["from"] + batch = normalize(encoded(payload(change(data))), CONFIG, NOW) + assert not batch.messages and batch.issues[0].reason == "invalid_message" + + +def test_quarantine_metadata_is_bounded_for_many_bad_messages(): + batch = normalize(encoded(payload(change(*([None] * 300)))), CONFIG, NOW) + assert len(batch.issues) == 256 + assert batch.issues[-1].reason == "issue_limit" + + +def test_unpersistable_startup_system_id_is_rejected_before_accepting_callbacks(): + with pytest.raises(ValueError): + replace(CONFIG, system_id="x" * 257) + + +@pytest.mark.parametrize( + "body,kind", + [ + (" /NEW ", "reset"), + ("שיחה חדשה", "reset"), + ("נציג", "help"), + ("/stop", "stop"), + ("הפסק", "stop"), + ("/start", "start"), + ("המשך", "start"), + ("/help find a file", "text"), + ], +) +def test_only_exact_control_commands_bypass_agent_text(body, kind): + batch = normalize(encoded(payload(change(inbound(text={"body": body})))), CONFIG, NOW) + assert batch.messages[0].kind == kind + assert batch.messages[0].content == body + + +@pytest.mark.parametrize("kind", ["system", "unknown", "unverified-provider-kind"]) +def test_provider_system_notifications_do_not_open_a_customer_service_window(kind): + batch = normalize(encoded(payload(change(inbound(type=kind)))), CONFIG, NOW) + assert batch.messages == () + assert batch.issues[0].reason == "invalid_message" + + +@pytest.mark.parametrize("field", ["id", "recipient_id", "status"]) +def test_unpersistable_status_keeps_valid_messages_and_status_siblings(field): + status = { + "id": "outbound", + "recipient_id": "sender", + "status": "delivered", + "timestamp": str(int(NOW.timestamp())), + } + data = payload(change(inbound(), statuses=[{**status, field: "private\x00value"}, status])) + batch = normalize(encoded(data), CONFIG, NOW) + assert [item.provider_message_id for item in batch.messages] == ["wamid.one"] + assert len(batch.delivery_events) == 1 + assert batch.delivery_events[0].provider_message_id == "outbound" + assert [(issue.location, issue.reason) for issue in batch.issues] == [ + ("entry.0.changes.0.statuses.0", "invalid_status") + ] diff --git a/tests/agent_manager/test_whatsapp_rendering.py b/tests/agent_manager/test_whatsapp_rendering.py new file mode 100644 index 00000000..cdccc931 --- /dev/null +++ b/tests/agent_manager/test_whatsapp_rendering.py @@ -0,0 +1,43 @@ +"""Exact text roundtrips and grapheme boundaries, with no new test dependency.""" + +import random + +import pytest + +from agent_manager.infrastructure.channels.whatsapp.rendering import render_text + + +def test_rendering_prefers_paragraphs_and_preserves_every_character(): + assert render_text("aaaa\n\nbbbb", limit=6) == ("aaaa\n\n", "bbbb") + + +@pytest.mark.parametrize("cluster", ["שָׁ", "👩🏽‍💻", "🇮🇱", "é", "\r\n", "가"]) +def test_rendering_never_splits_a_grapheme(cluster): + size = len(cluster.encode("utf-16-le")) // 2 + text = "x" + cluster * 5 + "y" + parts = render_text(text, limit=max(size, 2)) + assert "".join(parts) == text + assert all(len(part.encode("utf-16-le")) // 2 <= max(size, 2) for part in parts) + assert all(part.strip("xy").replace(cluster, "") == "" for part in parts) + + +def test_rendering_roundtrip_bounds_and_determinism_over_unicode_corpus(): + rng = random.Random(771) + alphabet = ["א", "שָׁ", "a", "👨‍👩‍👧‍👦", "🇮🇱", "\n", " ", "é", "中文", "\u200f"] + for _ in range(200): + text = "x" + "".join(rng.choices(alphabet, k=rng.randrange(1, 100))) + "y" + parts = render_text(text, limit=32) + assert "".join(parts) == text + assert all(0 < len(part.encode("utf-16-le")) // 2 <= 32 for part in parts) + assert render_text(text, limit=32) == parts + + +@pytest.mark.parametrize("text", ["", " ", "private\x00text", "x" + "́" * 4096]) +def test_unrenderable_input_is_rejected_without_truncation(text): + with pytest.raises(ValueError): + render_text(text) + + +def test_rendering_caps_part_count(): + with pytest.raises(ValueError): + render_text("x" * 129, limit=1) diff --git a/tests/agent_manager/test_whatsapp_sender.py b/tests/agent_manager/test_whatsapp_sender.py new file mode 100644 index 00000000..96e11ee6 --- /dev/null +++ b/tests/agent_manager/test_whatsapp_sender.py @@ -0,0 +1,108 @@ +"""Wire format and safe send outcomes through httpx's fake transport.""" + +import json + +import httpx +import pytest +from pydantic import SecretStr + +from agent_manager.infrastructure.channels.whatsapp.sender import WhatsAppSender + + +async def test_sender_uses_pinned_https_version_and_preserves_text(): + requests = [] + + def provider(request): + requests.append(request) + return httpx.Response(200, json={"messages": [{"id": "wamid.accepted"}]}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(provider)) as client: + sender = WhatsAppSender("v99.0", "123", SecretStr("test-only-token"), client=client) + outcome = await sender.send("opaque-recipient", "שלום 👋\nlink https://example.test") + assert outcome.kind == "accepted" and outcome.provider_message_id == "wamid.accepted" + assert str(requests[0].url) == "https://graph.facebook.com/v99.0/123/messages" + assert requests[0].headers["authorization"] == "Bearer test-only-token" + assert json.loads(requests[0].content) == { + "messaging_product": "whatsapp", + "recipient_type": "individual", + "to": "opaque-recipient", + "type": "text", + "text": {"body": "שלום 👋\nlink https://example.test", "preview_url": False}, + } + assert "test-only-token" not in repr(sender) + + +@pytest.mark.parametrize( + "status,body,kind", + [ + (400, {"error": {"code": 1, "is_transient": True}}, "retry"), + (429, {"error": {"code": 130429}}, "retry"), + (503, {"error": {"code": 2}}, "retry"), + (401, {"error": {"code": 190}}, "failed"), + (400, {"error": {"code": 100}}, "failed"), + (503, "private provider error", "unknown"), + (200, {"messages": []}, "unknown"), + (200, {"messages": [{"id": ""}]}, "unknown"), + (302, {"error": {"code": 1}}, "failed"), + ], +) +async def test_sender_classifies_explicit_rejection_and_ambiguous_responses(status, body, kind): + def provider(request): + return httpx.Response(status, json=body, headers={"Retry-After": "7"}) + + async with httpx.AsyncClient(transport=httpx.MockTransport(provider)) as client: + sender = WhatsAppSender("v99.0", "123", SecretStr("test-only-token"), client=client) + outcome = await sender.send("recipient", "hello") + assert outcome.kind == kind + assert "private" not in repr(outcome) + if kind == "retry": + assert outcome.retry_after == 7 + + +@pytest.mark.parametrize( + "error,kind", + [ + (httpx.ConnectTimeout, "retry"), + (httpx.ConnectError, "retry"), + (httpx.PoolTimeout, "retry"), + (httpx.ReadTimeout, "unknown"), + (httpx.WriteTimeout, "unknown"), + (httpx.RemoteProtocolError, "unknown"), + ], +) +async def test_sender_never_blindly_retries_a_possibly_submitted_request(error, kind): + def provider(request): + raise error("private-provider-credential", request=request) + + async with httpx.AsyncClient(transport=httpx.MockTransport(provider)) as client: + sender = WhatsAppSender("v99.0", "123", SecretStr("test-only-token"), client=client) + result = await sender.send("recipient", "hello") + assert result.kind == kind + assert "private" not in repr(result) + + +async def test_sender_bounds_response_bytes(): + async with httpx.AsyncClient( + transport=httpx.MockTransport(lambda request: httpx.Response(200, content=b"x" * 65_537)) + ) as client: + result = await WhatsAppSender( + "v99.0", "123", SecretStr("test-only-token"), client=client + ).send("recipient", "hello") + assert result.kind == "unknown" + + +@pytest.mark.parametrize("version", ["latest", "v1.0/path", "v0.0", "https://evil.test"]) +def test_sender_rejects_unpinned_or_injectable_versions(version): + with pytest.raises(ValueError): + WhatsAppSender(version, "123", SecretStr("test-only-token")) + + +@pytest.mark.parametrize("body", ["", "\x00", "x" * 4097]) +async def test_sender_rejects_invalid_bodies_before_any_network_io(body): + def provider(request): + pytest.fail("invalid body reached provider") + + async with httpx.AsyncClient(transport=httpx.MockTransport(provider)) as client: + sender = WhatsAppSender("v99.0", "123", SecretStr("test-only-token"), client=client) + with pytest.raises(ValueError): + await sender.send("recipient", body) diff --git a/tests/agent_manager/test_whatsapp_webhook.py b/tests/agent_manager/test_whatsapp_webhook.py new file mode 100644 index 00000000..79816430 --- /dev/null +++ b/tests/agent_manager/test_whatsapp_webhook.py @@ -0,0 +1,383 @@ +"""Real manager HTTP composition and file SQL; no provider, model, or MCP calls.""" + +from __future__ import annotations + +import asyncio +import hashlib +import hmac +import json +from collections.abc import Iterator +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from pydantic import ValidationError +from sqlmodel import SQLModel, select + +from agent_manager.api.app import create_app +from agent_manager.config import Settings +from agent_manager.infrastructure.persistence.channels.tables import ChannelInboxRow +from agent_manager.infrastructure.persistence.database import create_db_engine, session_factory +from agent_manager.infrastructure.persistence.tables import ConversationSessionRow +from tests.agent_manager.conftest import RecordingEngine + +ENDPOINT = "/channels/whatsapp/webhook" +APP_SECRET = "test-only-whatsapp-app-secret" +VERIFY_TOKEN = "test-only-subscription-token" + + +def settings_values(**overrides): + return { + "extra_whatsapp_enabled": True, + "extra_whatsapp_connection_id": "business-number", + "extra_whatsapp_account_id": "account-1", + "extra_whatsapp_phone_number_id": "number-1", + "extra_whatsapp_app_secret": APP_SECRET, + "extra_whatsapp_verify_token": VERIFY_TOKEN, + **overrides, + } + + +def message(provider_id="wamid.one", **overrides): + return { + "id": provider_id, + "from": "opaque-sender", + "timestamp": str(int(datetime.now(UTC).timestamp()) - 60), + "type": "text", + "text": {"body": "שלום 👋\nHow can I help?"}, + **overrides, + } + + +def change(*messages, phone="number-1", statuses=None): + return { + "field": "messages", + "value": { + "messaging_product": "whatsapp", + "metadata": {"phone_number_id": phone, "display_phone_number": "not-an-identity"}, + "messages": list(messages), + "statuses": statuses or [], + }, + } + + +def payload(*changes, account="account-1"): + return { + "object": "whatsapp_business_account", + "entry": [{"id": account, "changes": list(changes)}], + } + + +def encoded(value) -> bytes: + return json.dumps(value, ensure_ascii=False).encode() + + +def signed(body: bytes) -> dict[str, str]: + signature = hmac.new(APP_SECRET.encode(), body, hashlib.sha256).hexdigest() + return {"X-Hub-Signature-256": f"sha256={signature}", "Content-Type": "application/json"} + + +class StartupEngine(RecordingEngine): + def __init__(self): + super().__init__() + self.builds = 0 + + async def build(self, _spec: object) -> None: + self.builds += 1 + + +@pytest.fixture +def channel( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> Iterator[tuple[TestClient, str, StartupEngine]]: + url = f"sqlite+aiosqlite:///{tmp_path / 'webhook.db'}" + + async def schema(): + database = create_db_engine(url) + async with database.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + await database.dispose() + + asyncio.run(schema()) + engine = StartupEngine() + monkeypatch.setattr("agent_manager.api.app.LangGraphEngine", lambda *a, **kw: engine) + config = ( + Path(__file__).resolve().parents[2] / "examples/enterprise-knowledge-assistant/agents.yaml" + ) + app = create_app(str(config), Settings.from_values(database_url=url, **settings_values())) + with TestClient(app) as client: + yield client, url, engine + + +def saved_rows(url, model): + async def read(): + database = create_db_engine(url) + try: + async with session_factory(database)() as session: + return list((await session.exec(select(model))).all()) + finally: + await database.dispose() + + return asyncio.run(read()) + + +def test_disabled_webhook_is_absent_without_building_an_engine() -> None: + app = create_app("unused.yml", Settings.from_values()) + client = TestClient(app) + assert client.get(ENDPOINT).status_code == 404 + assert client.post(ENDPOINT).status_code == 404 + + +@pytest.mark.parametrize("missing", ["account_id", "phone_number_id", "app_secret", "verify_token"]) +def test_enabled_intake_rejects_missing_configuration(missing): + with pytest.raises(ValidationError): + Settings.from_values(**settings_values(**{f"extra_whatsapp_{missing}": None})) + + +@pytest.mark.parametrize( + "url", + ["sqlite+aiosqlite:///:memory:", "sqlite://", "sqlite:///file:queue?mode=memory&uri=true"], +) +def test_enabled_intake_requires_durable_storage(url): + with pytest.raises(ValidationError): + Settings.from_values(database_url=url, **settings_values()) + + +def test_configuration_secrets_are_redacted_even_on_failure(): + config = Settings.from_values(**settings_values()) + for output in (repr(config), config.model_dump_json()): + assert APP_SECRET not in output and VERIFY_TOKEN not in output + with pytest.raises(ValidationError) as failure: + Settings.from_values(**settings_values(extra_whatsapp_verify_token=APP_SECRET)) + assert APP_SECRET not in str(failure.value) + + +def test_subscription_returns_exact_plain_text_challenge(channel): + client, _, engine = channel + result = client.get( + ENDPOINT, + params={ + "hub.mode": "subscribe", + "hub.verify_token": VERIFY_TOKEN, + "hub.challenge": "000123+challenge", + }, + ) + assert result.status_code == 200 + assert result.text == "000123+challenge" + assert result.headers["content-type"].startswith("text/plain") + assert engine.builds == 1 and engine.prompts == [] + + +@pytest.mark.parametrize( + "params", + [ + {}, + {"hub.mode": "unsubscribe", "hub.verify_token": VERIFY_TOKEN, "hub.challenge": "1"}, + {"hub.mode": "subscribe", "hub.verify_token": "wrong", "hub.challenge": "1"}, + {"hub.mode": "subscribe", "hub.verify_token": VERIFY_TOKEN}, + ], +) +def test_invalid_subscription_fails_closed(channel, params): + client, _, _ = channel + assert client.get(ENDPOINT, params=params).status_code == 403 + + +@pytest.mark.parametrize( + "signature", [None, "", "sha1=123", "sha256=123", "sha256=" + "z" * 64, "sha256=" + "0" * 64] +) +def test_unsigned_or_bad_signature_never_reaches_storage(channel, signature): + client, url, _ = channel + headers = {"Content-Type": "application/json"} + if signature is not None: + headers["X-Hub-Signature-256"] = signature + assert ( + client.post( + ENDPOINT, content=encoded(payload(change(message()))), headers=headers + ).status_code + == 401 + ) + assert saved_rows(url, ChannelInboxRow) == [] + + +def test_signature_covers_original_bytes_and_rejects_ambiguous_headers(channel): + client, url, _ = channel + body = encoded(payload(change(message()))) + assert client.post(ENDPOINT, content=body + b" ", headers=signed(body)).status_code == 401 + headers = [*signed(body).items(), ("X-Hub-Signature-256", signed(body)["X-Hub-Signature-256"])] + assert client.post(ENDPOINT, content=body, headers=headers).status_code == 401 + assert saved_rows(url, ChannelInboxRow) == [] + + +def test_all_entries_changes_and_messages_are_persisted_without_running_an_agent(channel): + client, url, engine = channel + data = payload(change(message("one"), message("two")), change(message("three"))) + data["entry"].insert(0, {"id": "unrelated", "changes": [change(message("foreign"))]}) + data["entry"].append({"id": "account-1", "changes": [change(message("four"))]}) + body = encoded(data) + assert client.post(ENDPOINT, content=body, headers=signed(body)).status_code == 200 + assert client.post(ENDPOINT, content=body, headers=signed(body)).status_code == 200 + rows = saved_rows(url, ChannelInboxRow) + assert [row.provider_message_id for row in rows] == ["one", "two", "three", "four"] + assert rows[0].content == "שלום 👋\nHow can I help?" + assert rows[0].sender_id == "opaque-sender" + assert all( + row.system_id == "Enterprise Knowledge Assistant" and row.state == "queued" for row in rows + ) + assert saved_rows(url, ConversationSessionRow) == [] + assert engine.builds == 1 and engine.prompts == [] + + +def test_other_receiving_number_is_ignored(channel): + client, url, _ = channel + body = encoded(payload(change(message(), phone="other-number"))) + assert client.post(ENDPOINT, content=body, headers=signed(body)).status_code == 200 + assert saved_rows(url, ChannelInboxRow) == [] + + +def test_invalid_message_does_not_discard_its_valid_sibling(channel): + client, url, _ = channel + body = encoded(payload(change(message("bad", timestamp="not-a-timestamp"), message("good")))) + assert client.post(ENDPOINT, content=body, headers=signed(body)).status_code == 200 + assert [row.provider_message_id for row in saved_rows(url, ChannelInboxRow)] == ["good"] + + +def test_body_limit_applies_even_without_a_content_length(channel): + client, url, _ = channel + body = b" " * 262_145 + assert client.post(ENDPOINT, content=body, headers=signed(body)).status_code == 413 + assert ( + client.post( + ENDPOINT, content=iter([body[:200_000], body[200_000:]]), headers=signed(body) + ).status_code + == 413 + ) + assert saved_rows(url, ChannelInboxRow) == [] + + +def test_delivery_status_is_durable_and_does_not_create_agent_input(channel): + client, url, engine = channel + status = { + "id": "outbound-one", + "recipient_id": "opaque-sender", + "status": "delivered", + "timestamp": message()["timestamp"], + "errors": [{"message": "private provider error"}], + } + body = encoded(payload(change(statuses=[status]))) + assert client.post(ENDPOINT, content=body, headers=signed(body)).status_code == 200 + from agent_manager.infrastructure.persistence.channels.tables import ChannelDeliveryEventRow + + events = saved_rows(url, ChannelDeliveryEventRow) + assert [(event.provider_message_id, event.status) for event in events] == [ + ("outbound-one", "delivered") + ] + assert "private provider error" not in events[0].model_dump_json() + assert saved_rows(url, ChannelInboxRow) == [] and engine.prompts == [] + + +@pytest.mark.parametrize( + "body", + [b"not JSON", b"[]", b'{"object":"whatsapp_business_account","entry":"invalid"}', b'"invalid"'], +) +def test_signed_poison_callback_is_recorded_before_acknowledgement(channel, body): + client, url, _ = channel + assert client.post(ENDPOINT, content=body, headers=signed(body)).status_code == 200 + from agent_manager.infrastructure.persistence.channels.tables import ChannelWebhookReceiptRow + + receipts = saved_rows(url, ChannelWebhookReceiptRow) + assert len(receipts) == 1 + assert receipts[0].issues == [{"location": "body", "reason": "invalid_envelope"}] + assert receipts[0].body_digest == hashlib.sha256(body).hexdigest() + assert saved_rows(url, ChannelInboxRow) == [] + + +def test_conflicting_provider_input_is_quarantined_without_rebinding(channel): + client, url, _ = channel + first = message() + original = encoded(payload(change(first))) + changed = encoded( + payload(change({**first, "from": "attacker", "text": {"body": "private conflicting text"}})) + ) + assert client.post(ENDPOINT, content=original, headers=signed(original)).status_code == 200 + assert client.post(ENDPOINT, content=changed, headers=signed(changed)).status_code == 200 + from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelBindingRow, + ChannelWebhookReceiptRow, + ) + + assert len(saved_rows(url, ChannelInboxRow)) == 1 + assert [row.sender_id for row in saved_rows(url, ChannelBindingRow)] == ["opaque-sender"] + receipts = saved_rows(url, ChannelWebhookReceiptRow) + assert receipts[-1].issues[0]["reason"] == "conflicting_message" + assert "private conflicting text" not in receipts[-1].model_dump_json() + + +def test_storage_failure_is_retryable_and_never_acknowledged(channel, monkeypatch, caplog): + client, url, _ = channel + + async def unavailable(*args, **kwargs): + raise OSError("private-storage-credential") + + from agent_manager.infrastructure.persistence.channels.webhooks import SqlWebhookRepository + + monkeypatch.setattr(SqlWebhookRepository, "accept", unavailable) + body = encoded(payload(change(message()))) + response = client.post(ENDPOINT, content=body, headers=signed(body)) + assert response.status_code == 503 + assert "private-storage-credential" not in response.text + caplog.text + assert saved_rows(url, ChannelInboxRow) == [] + + +def test_excessive_content_length_is_rejected_without_integer_conversion_failure(channel): + client, _, _ = channel + body = b"{}" + headers = {**signed(body), "Content-Length": "9" * 5000} + assert client.post(ENDPOINT, content=body, headers=headers).status_code == 400 + + +def test_failed_commit_returns_retryable_failure_and_leaves_no_receipt(channel): + from sqlalchemy import event + from sqlalchemy.orm import Session + + from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelBindingRow, + ChannelWebhookReceiptRow, + ) + + client, url, _ = channel + + def reject_commit(session): + raise OSError("injected commit failure") + + body = encoded(payload(change(message()))) + event.listen(Session, "before_commit", reject_commit) + try: + response = client.post(ENDPOINT, content=body, headers=signed(body)) + assert response.status_code == 503 + finally: + event.remove(Session, "before_commit", reject_commit) + for model in (ChannelInboxRow, ChannelBindingRow, ChannelWebhookReceiptRow): + assert saved_rows(url, model) == [] + assert client.post(ENDPOINT, content=body, headers=signed(body)).status_code == 200 + assert len(saved_rows(url, ChannelInboxRow)) == 1 + + +@pytest.mark.parametrize("headers", [{"Content-Type": "text/plain"}, {"Content-Encoding": "gzip"}]) +def test_unsupported_wire_encoding_is_rejected_before_storage(channel, headers): + client, url, _ = channel + body = encoded(payload(change(message()))) + response = client.post(ENDPOINT, content=body, headers={**signed(body), **headers}) + assert response.status_code == 415 + assert saved_rows(url, ChannelInboxRow) == [] + + +def test_duplicate_subscription_parameters_are_rejected(channel): + client, _, _ = channel + params = [ + ("hub.mode", "subscribe"), + ("hub.verify_token", VERIFY_TOKEN), + ("hub.challenge", "one"), + ("hub.challenge", "two"), + ] + assert client.get(ENDPOINT, params=params).status_code == 403 From dd2957818616eeb24131ad652c3ccf4496602915 Mon Sep 17 00:00:00 2001 From: Asaf Varon Date: Fri, 16 Oct 2026 15:00:54 +0300 Subject: [PATCH 5/7] docs: document the WhatsApp integration Integration plan, ADRs 0007-0010, implementation tasks 0014-0017 and 0019-0020, review records, and index entries. Co-Authored-By: Claude Opus 5.5 --- docs/README.md | 35 ++ docs/WHATSAPP_DELIVERY_REVIEW.md | 100 ++++ docs/WHATSAPP_INTEGRATION_PLAN.md | 511 ++++++++++++++++++ docs/WHATSAPP_REVIEW.md | 227 ++++++++ .../0007-durable-channel-identity-boundary.md | 73 +++ .../adr/0008-authenticated-whatsapp-intake.md | 73 +++ ...9-channel-worker-ownership-and-recovery.md | 104 ++++ docs/adr/0010-whatsapp-outbound-delivery.md | 100 ++++ tasks/0014-channel-storage-and-identity.md | 90 +++ tasks/0015-whatsapp-webhook.md | 105 ++++ tasks/0016-channel-turn-worker.md | 115 ++++ tasks/0017-whatsapp-review-and-hardening.md | 82 +++ tasks/0019-whatsapp-delivery-and-controls.md | 180 ++++++ tasks/0020-whatsapp-delivery-review.md | 82 +++ tasks/README.md | 8 + 15 files changed, 1885 insertions(+) create mode 100644 docs/WHATSAPP_DELIVERY_REVIEW.md create mode 100644 docs/WHATSAPP_INTEGRATION_PLAN.md create mode 100644 docs/WHATSAPP_REVIEW.md create mode 100644 docs/adr/0007-durable-channel-identity-boundary.md create mode 100644 docs/adr/0008-authenticated-whatsapp-intake.md create mode 100644 docs/adr/0009-channel-worker-ownership-and-recovery.md create mode 100644 docs/adr/0010-whatsapp-outbound-delivery.md create mode 100644 tasks/0014-channel-storage-and-identity.md create mode 100644 tasks/0015-whatsapp-webhook.md create mode 100644 tasks/0016-channel-turn-worker.md create mode 100644 tasks/0017-whatsapp-review-and-hardening.md create mode 100644 tasks/0019-whatsapp-delivery-and-controls.md create mode 100644 tasks/0020-whatsapp-delivery-review.md diff --git a/docs/README.md b/docs/README.md index c88c35ee..b2ba5f59 100644 --- a/docs/README.md +++ b/docs/README.md @@ -31,6 +31,41 @@ implemented, so protected-node access control is not actually enforced today. 14. [adr/](adr/) — architecture decision records for contract-level changes, including [immutable conversation branches](adr/0002-conversation-lifecycle-and-immutable-branches.md). +## Integrations in development + +- [WhatsApp integration plan](WHATSAPP_INTEGRATION_PLAN.md) — Cloud API + channel for one organization-owned number, with durable delivery, conversation + reuse, identity boundaries, and scoped implementation tasks. Generic turn + preparation, SQL channel storage, authenticated webhook intake, an + opt-in turn worker with conservative recovery, and opt-in provider delivery + with text controls are implemented. Customer account linking and live pilot + packaging remain planned. + +[ADR 0006](adr/0006-idempotent-conversation-turn-preparation.md) records the +implemented provider-independent preparation prerequisite for channels. +[ADR 0007](adr/0007-durable-channel-identity-boundary.md) records the implemented +durable channel storage and generic identity port. Subsequent ADRs record the +implemented transport; the customer's Google/JWT account-linking integration +remains pending. +[ADR 0008](adr/0008-authenticated-whatsapp-intake.md) records implemented +authenticated batch intake and commit-before-acknowledgement semantics. +[ADR 0009](adr/0009-channel-worker-ownership-and-recovery.md) records exclusive +worker ownership, fresh identity checks, saved-answer recovery, and shutdown. + +[WhatsApp explanation and review](WHATSAPP_REVIEW.md) describes the implemented +path, the shutdown, health, and normalized-input corrections, regression +evidence, and the remaining sending, identity, and pilot work. + +[ADR 0010](adr/0010-whatsapp-outbound-delivery.md) records the implemented +WA-05 provider delivery, immutable text plans, authorization/window/suppression +checks and controls in [task 0019](../tasks/0019-whatsapp-delivery-and-controls.md). +The earlier review is a dated snapshot of WA-01–WA-04; its sending follow-up is +now implemented locally. No real account/number has been activated. + +[WA-05 delivery review](WHATSAPP_DELIVERY_REVIEW.md) records the subsequent +delivery/control review and cancellation-safe channel client cleanup correction +in [task 0020](../tasks/0020-whatsapp-delivery-review.md). + ## Relationship to other directories - `AGENTS.md` (root) — the operating manual for agents; references these docs. diff --git a/docs/WHATSAPP_DELIVERY_REVIEW.md b/docs/WHATSAPP_DELIVERY_REVIEW.md new file mode 100644 index 00000000..8b71cda6 --- /dev/null +++ b/docs/WHATSAPP_DELIVERY_REVIEW.md @@ -0,0 +1,100 @@ +# WhatsApp delivery review — WA-05 + +Date: 2026-10-16. Scope: [task 0019](../tasks/0019-whatsapp-delivery-and-controls.md), +reviewed and corrected under [task 0020](../tasks/0020-whatsapp-delivery-review.md). +This supplements the earlier [WA-01–WA-04 review](WHATSAPP_REVIEW.md). + +## 1. Summary + +Reviewed the opt-in sender, immutable delivery planning, recipient ordering, +fresh identity checks, suppression/window guards, callback projection, exact +controls, migration and supervisor/client lifecycles. The implementation keeps +provider delivery separate from the engine and from immutable logical answers. +One lifecycle defect was reproduced and corrected; the full validation gate +passes with 1,423 tests. + +## 2. Blocking issues + +**[P2, fixed] Cancellation could abandon channel client cleanup.** After the +worker had drained, the composition context awaited HTTP client close and then +identity client close directly. Cancelling application shutdown during either +await could return from the lifespan with that client still open; cancellation +during HTTP close could also skip identity close entirely. This affects resource +cleanup during deployment/restart rather than authorizing messages or repeating +an engine run. + +The composition layer now owns a separate cleanup task, shields and joins it +through repeated caller cancellation, and propagates cancellation afterwards. +Identity close also runs in a finally block after sender close. The existing +worker drain/ownership ordering and sanitized ordinary close errors are retained. +No open blocking finding remains from the inspected scope. + +## 3. Non-blocking issues + +No additional actionable defect was confirmed in the reviewed delivery/control +paths. Existing operating limits remain explicit: failed/blocked/unknown work +needs operator reconciliation, and a live downgrade must preserve delivery +evidence before sending resumes. These are documented constraints, not newly +implemented recovery capabilities. + +## 4. Architecture concerns + +No boundary violation found. Client resource ownership stays in composition; +provider HTTP/rendering and SQL projection stay in infrastructure; processing +uses domain ports. The correction adds no runtime request state and changes no +engine, YAML, conversation HTTP, identity resolver or persistence contract. +[ADR 0010](adr/0010-whatsapp-outbound-delivery.md) remains applicable; no new +architectural decision or migration is needed. + +## 5. Security concerns + +The reviewed send path checks the original account and configured organization +again before each internal part, then rechecks persisted suppression/window +state after the identity await. Static controls contain no internal answer. +Callbacks correlate exact connection/system/provider ID/recipient, and positive +delivery/read evidence does not regress on late sent/failed observations. +Ambiguous sends are held rather than retried automatically. The fixed Graph +host, secret settings, body limits and sanitized error logging remain intact. + +The customer's trusted identity plugin must still implement revocable account +linking and current permissions; transport signatures and sender IDs do not +grant organization access. Google/JWT account linking is not implemented by this +review. Data/tool authorization remains the responsibility of client plugins. + +## 6. Testing gaps + +Three new deterministic cases use events to suspend HTTP or identity close, +cancel shutdown twice, assert the lifespan remains pending, release cleanup, +and verify close order plus propagated cancellation. They cover sender enabled +and disabled. All three failed before the implementation correction and passed +afterwards. Existing ownership/send-cleanup and identity-error tests remain in +the focused verification set. Only local SQLite and fake provider, identity and +engine dependencies are used. + +Validation: all 129 focused delivery, controls, Unicode, sender, worker and +lifecycle cases pass. Final `make check` passed: 356 files formatted, lint clean, +mypy clean for 352 files, **1,423 tests passed / 27 warnings**, and generated-stub +checks passed. The 27 warning instances are unchanged from the pre-review +baseline of 1,420 tests. No warnings were suppressed. An initial full-gate lint +failure (SIM117 in the new test's nested context managers) was corrected before +the passing gate. `git diff --check`, 50 local documentation links/anchors and +whitespace checks for all six changed files also passed. + +Live Meta delivery, current account/version-specific protocol fixtures, +PostgreSQL against a live server, Google/JWT login and production credentials +were not tested. Process termination that prevents Python cleanup is outside +the cancellation guarantee; crash recovery still preserves uncertain attempts. + +## 7. Suggested improvements + +Continue with the already planned WA-06 pilot runbook and operational preflight, +including a concrete operator procedure for unknown/failed work, current account +contract verification and the identity integration prerequisite. Do not enable +an internal-information pilot until that trusted account-linking path exists. + +## 8. Final recommendation + +**Approve** the scoped delivery change with the lifecycle correction included. +The confirmed defect is fixed and the full gate passes. This review does not +activate a real account or certify a production rollout; the existing identity +and operational prerequisites still apply. diff --git a/docs/WHATSAPP_INTEGRATION_PLAN.md b/docs/WHATSAPP_INTEGRATION_PLAN.md new file mode 100644 index 00000000..57f2ec78 --- /dev/null +++ b/docs/WHATSAPP_INTEGRATION_PLAN.md @@ -0,0 +1,511 @@ +# WhatsApp integration plan + +- **Status:** WA-01 preparation, WA-02 durable channel storage/identity + boundary, WA-03 authenticated webhook intake, WA-04 turn worker/recovery, + and WA-05 outbound delivery/controls are implemented. Organization account + linking and live pilot setup remain pending. Intake, worker and sender are + separately disabled by default. +- **Date:** 2026-10-15. +- **Updated:** 2026-10-16. +- **Confirmed scope:** One organization-owned WhatsApp Business number. +- **Confirmed pilot:** An assistant that needs internal organization information. +- **Identity direction:** A generic integration, later using Google login and + an organization authentication server that issues JWTs. +- **Goal:** Let a person message that number, hold a persistent conversation with + an Extra agent system, and receive its answer in WhatsApp. + +The user approved starting implementation on 2026-10-15. +[ADR 0006](adr/0006-idempotent-conversation-turn-preparation.md) records the +WA-01 preparation contract implemented by +[task 0013](../tasks/0013-idempotent-conversation-turns.md). +[ADR 0007](adr/0007-durable-channel-identity-boundary.md) records the SQL +storage and client identity boundary implemented by +[task 0014](../tasks/0014-channel-storage-and-identity.md). +[ADR 0008](adr/0008-authenticated-whatsapp-intake.md) records authenticated +batch intake implemented by +[task 0015](../tasks/0015-whatsapp-webhook.md). +[ADR 0009](adr/0009-channel-worker-ownership-and-recovery.md) records worker +ownership and recovery implemented by +[task 0016](../tasks/0016-channel-turn-worker.md). Account preflight +remains pending: no real number, credentials, or deployment were configured. +[Task 0017](../tasks/0017-whatsapp-review-and-hardening.md) reviews these stages +and hardens caller-cancelled shutdown and persistable normalized input; see +the [implementation explanation and review](WHATSAPP_REVIEW.md). +[Task 0019](../tasks/0019-whatsapp-delivery-and-controls.md) implements delivery +and controls under [ADR 0010](adr/0010-whatsapp-outbound-delivery.md), with local +SQL/fake-provider verification. No real account was configured or contacted. +Subsequent architecture/contract changes still require scoped tasks and ADRs, +as required by [AGENTS.md](../AGENTS.md). + +## 1. Recommended first release + +Use Meta's **WhatsApp Cloud API** directly, with a channel adapter in +`agent_manager`. Start with one number, one configured agent system, individual +text conversations, and a single final answer per turn, split into ordered +messages when necessary. Configure the connection administratively; a customer +account-connection interface is unnecessary for this scope. + +The experience is: send a text message, receive an answer, then continue with +the previous conversation as context. A deterministic `שיחה חדשה` / `/new` +command starts a fresh Extra conversation. Unsupported attachments receive a +short explanation. A `נציג` / `/help` command gives the organization's real +support contact; an agent inbox and live operator takeover are later work. + +The confirmed pilot needs internal information. Complete the identity/access +work in section 5 before activating the channel. Select an +organization-specific assistant use case and check its +eligibility against the account's current Meta terms during setup; this plan +does not establish that every agent system may be distributed through WhatsApp. + +### Existing implementation to reuse + +| Existing component | Role in this integration | +| --- | --- | +| `agent_manager/api/app.py` | Startup composition: validate configuration and build the engine once. | +| `application/conversation_service.py` | Conversation ownership, history, turn preparation, execution, and final-answer persistence. | +| `domain/identity.py` | `Principal`, including externally resolved identities. | +| `domain/turn_preparation.py` | Optional atomic preparation capability and validated, digested retry keys (WA-01). | +| `domain/channels/` | Bounded normalized input, work states, durable repository port, and client identity resolver (WA-02). | +| `application/channels/preparation.py` | Fresh organization identity checks and recoverable manager turn preparation (WA-02). | +| `infrastructure/persistence/channels/` | Atomic SQL intake, sender claims, ordered reset, execution claims, and immutable reply staging (WA-02). | +| `api/routes/whatsapp.py`, `infrastructure/channels/whatsapp/` | Opt-in challenge/signature verification, bounded typed batch normalization, and durable intake (WA-03). | +| `domain/channels/webhooks.py`, SQL webhook adapter | Atomic callback receipts, normalized delivery observations, and safe quarantine metadata (WA-03). | +| `application/channels/{worker,processing}.py` | Bounded sender jobs, fresh execution identity, normal manager streaming, saved-answer staging, and conservative recovery (WA-04). | +| `domain/channels/ownership.py`, SQL ownership adapter | Exclusive connection supervisor; POSIX file lock for local SQLite or a dedicated PostgreSQL advisory-lock session (WA-04). | +| `application/channels/{delivery,controls}.py` | Original-recipient authorization, attempt policy, window/suppression checks, and deterministic controls (WA-05). | +| `infrastructure/channels/whatsapp/{sender,rendering}.py` | Bounded HTTPS submission and exact Unicode/grapheme-safe plain-text parts (WA-05). | +| SQL delivery/control adapters, migration 0010 | Immutable provider parts, pinned owners, CAS attempts, callback projection and suppression cutoffs (WA-05). | +| `infrastructure/persistence/` | Conversation storage, SQL run storage, and Alembic migrations. | +| `agent_engine/engine/engine.py` | Existing typed engine boundary. | +| `agent_engine/runtime/hooks/models.py` | Request-scoped `RunContext` and `AuthContext`. | + +The current code supports SQL persistence and defaults to a file-backed SQLite +database in `Settings`; explicit memory configurations are also possible. Do +not infer restart durability from the package's presence alone: validate the +actual deployment settings. SQL composition exposes a channel repository; +memory-mode composition exposes none. Session approvals, tool usage, and the tool +execution ledger have process-local implementations, so durable conversations +do not imply durable tool replay or approvals. + +## 2. Architecture and ownership + +```mermaid +flowchart TD + U[WhatsApp user] --> M[Meta Cloud API] + M --> W[HTTPS webhook: verify and normalize] + W --> I[(Durable inbox)] + I --> J[Worker: serialize each sender's conversation] + J --> P[Channel identity policy] + P --> C[ConversationService] + C --> E[Existing long-lived Engine] + C --> D[(Conversation and run storage)] + D --> O[(Durable outbox)] + O --> S[Cloud API sender] + S --> M + M --> U + W --> T[(Delivery status records)] +``` + +`agent_manager` owns the transport and conversation lifecycle. The provider +adapter normalizes WhatsApp payloads into typed channel events, and the +application layer calls `ConversationService`. It must not call an HTTP +endpoint on its own process or bypass the service and invoke `/invoke` directly. + +`agent_engine` has no WhatsApp imports, provider payloads, or channel storage. +It still receives typed messages/history and a fresh request context. Agent +YAML, the compiler, model providers, prompts, and engine public ports remain +unchanged. Organization-specific account lookup and permissions live in a +client-owned identity/access plugin, behind the implemented +`ChannelIdentityResolver` port. + +The implemented first worker shares one `agent-manager` process and application +service. It is separately enabled at startup and awaited during shutdown. +Bounded concurrent jobs let independent senders progress; binding claims retain +each sender's order. Database records are the work queue; an in-memory +task or FastAPI background callback is not the durable handoff. This avoids +introducing Redis, a broker, or a separate deployment for the initial number. + +Recovery requires exclusive ownership of the connection. Local SQLite uses a +POSIX lock beside the canonical database path; do not delete that lock file or +use network-mounted SQLite for the worker. PostgreSQL uses a dedicated session +advisory lock with backend-session checks and autocommit; transaction-pooling +proxies are unsuitable for that ownership connection. A lost owner stops claims +and cancels/awaits its jobs. An enabled stopped worker makes `/health` return +503 with `status: degraded`; worker-disabled health behavior stays unchanged. + +Shutdown retains ownership while engine cleanup finishes, including when the +lifespan caller is cancelled repeatedly. Ownership failure degrades health +before waiting for that cleanup. Normalized identifiers/text reject +NUL before SQL writes, so unpersistable authenticated items are quarantined +without discarding valid siblings in their callback. + +Use PostgreSQL for the production deployment. File-backed SQLite may support a +single-process local pilot with tested transaction semantics. Multiple manager +replicas and distributed execution are a separate milestone, since current +process-local runtime repositories are insufficient for safe failover. + +## 3. Incoming and outgoing message flow + +1. **Verify subscription (implemented).** `GET /channels/whatsapp/webhook` checks + `hub.mode` and the configured verification token, then returns the exact + challenge. The verification token is distinct from the app secret and API token. +2. **Verify notification (implemented).** `POST` on that path checks the signature + over the original request bytes using the app secret, with constant-time + comparison. Streamed bytes are bounded before parsing, even without a + content-length header. Invalid signatures are rejected before storage; + unsupported encoding/content types are rejected. Meta's archived official + reference describes the separate challenge and `X-Hub-Signature-256` + mechanisms. [Meta webhook reference](https://whatsapp.github.io/WhatsApp-Nodejs-SDK/api-reference/webhooks/start/) +3. **Normalize the full batch (implemented).** Iterate every entry, change, and message; do + not assume the event is at index zero. Check the configured business account + and receiving-number identifiers. Separate inbound messages, delivery + updates, and unsupported event types. Delivery updates never invoke an agent. + The current explicit contract uses `whatsapp_business_account` envelopes + and opaque message `from` identifiers. New account/BSUID payload variants + require verified fixtures before adoption. Meta's official + [Postman reference](https://www.postman.com/meta/whatsapp-business-platform/folder/tduohwq/webhook-payload-reference) + confirms the wire account/number envelope. +4. **Persist, then acknowledge (implemented).** Insert inbound messages and work state in one + transaction. Return `200` after commit, without waiting for an LLM. A duplicate + provider message returns `200` and reuses its existing work. A temporary + storage failure returns a retryable failure; never acknowledge unrecorded work. + Authenticated malformed items retain a body digest and bounded reason/position + metadata, preserving valid siblings without storing raw payloads or errors. + Conflicting provider-message input is quarantined without rebinding its + original target. Receipt, messages/bindings, delivery observations, and + quarantine metadata commit in one transaction. Storage failure returns 503; + the complete callback can be retried. Unauthenticated requests are rejected; + unrelated accounts/events are ignored without creating a conversation. +5. **Resolve the sender and conversation (implemented).** Use an opaque provider sender ID, + scoped to the configured connection and agent system. Do not assume it is + always a visible phone number or merge it with a web user by matching text. + Atomically get or create the active binding and its Extra conversation. +6. **Prepare one turn (implemented).** Persist a stable link between the inbound event and + the manager's user message/run. The manager loads bounded prior context and + the worker refreshes the original linked `Principal` again before the atomic + execution claim. Roles and delegated credentials remain request-scoped. +7. **Persist the answer (implemented).** Use the manager's normal completion path, then + create an outbox entry referring to that persisted assistant message. + An idempotent recovery scan repairs a crash between answer persistence and + outbox creation. A persisted answer is never regenerated to retry delivery. + The worker currently stages one exact response part, up to 65536 UTF-8 bytes. + Empty/missing/oversized answers stop for review. WA-05 freezes provider parts + separately, preserving this original logical answer. With the sender enabled, + controls and unsupported-media feedback use static replies without a model. +8. **Send and track (implemented, opt-in).** Preserve exact plain text/Unicode/RTL, + split at grapheme boundaries with paragraph preference, and cap each part at + 4096 UTF-16 units and each batch at 128 parts. Freeze the plan once; send + parts/turns in recipient order and persist each returned provider message ID. + API acceptance is distinct from delivered/read callbacks. Interrupted or + ambiguous submission is held for review; it is never blindly resent. + +Target a webhook acknowledgement below one second under the agreed pilot +load. This is an internal acceptance target, not a claimed Meta timeout. + +### Concurrency, ordering, and crash recovery + +- A database-backed claim serializes a sender's turns through answer/outbox + creation; a local lock alone does not establish ownership. Preserve a durable + receive sequence within that conversation. Provider timestamps remain metadata: + the adapter cannot guarantee the original order of late webhook arrivals. +- Preserve outbound part and turn order per recipient. Bound queue depth and + concurrency across recipients; apply model/time/tool budgets as usual. +- `/new` is queued in the same ordering mechanism. It changes the active + binding without deleting history or causing an older answer to attach to the + new conversation. WhatsApp's service window does not define conversation expiry. +- Without a key, `prepare_turn()` still creates fresh random IDs. WA-01 adds + an opt-in `idempotency_key`, backed by the optional + `IdempotentTurnRepository` capability. Receipt, user append, head selection, + and snapshot commit atomically. Run registration is reconciled on replay, + and conflicting owner/conversation/content is rejected. See ADR 0006. + Merely deduplicating webhook receipt does **not** replace that preparation + boundary. This capability does not itself claim or authorize execution. +- On restart, queued unprepared work may proceed; a prepared turn may proceed + only when the durable execution claim proves it has not started. Recover a + stored answer into the outbox. An interrupted/ambiguous execution goes to + `needs_review`, blocks later turns for that sender, and requires explicit + reconciliation. Do not automatically rerun tools, reclaim a live execution + because a lease expired, or infer that a `COMPLETED` run always has a stored + assistant answer. The engine and manager currently commit those separately. +- Retry a known rejected/transient provider send with capped exponential + backoff and jitter. On a timeout after submission, record `delivery_unknown`: + reconcile using verified provider correlation support/status callbacks where + available, or require operator review. Blind resends can duplicate messages. + Do not promise exactly-once external delivery. + +## 4. Storage and application seams + +Migration 0008 implements `channel_bindings`, `channel_inbox`, and +`channel_outbox`. The latter retains immutable logical reply parts. +Migration 0010 adds provider delivery batches/parts, sender preferences/window +metadata, and callback projection bookkeeping. Migration 0009 adds +`channel_webhook_receipts` and `channel_delivery_events` for atomic callback +intake and normalized observations. The generic turn-preparation receipt +is implemented as `conversation_turn_preparations` by migration 0007; its +input/owner/run binding comes from its atomically committed user message. + +| Record / port | Responsibility and key | +| --- | --- | +| Channel connection settings (implemented) | Explicit connection, receiving number/account, enable flags, request limits, separate webhook secrets, and separately required API version/access token/support contact for sending. | +| `channel_bindings` (implemented) | Unique `(connection_id, system_id, sender_id)` to active conversation, organization/user IDs, reset generation, and active work claim. | +| `channel_inbox` (implemented) | Unique `(connection_id, provider_message_id)`; normalized input, receive sequence, provider timestamp, work state, claim token, and conversation/run/message references. Retry attempts remain later worker work. | +| Turn-preparation receipt (implemented) | Unique hashed idempotency key linked to stable user-message/run IDs; same transaction as the user-message append. Callers namespace keys for their connection/provider. | +| `channel_outbox` (implemented) | Unique `(event_id, response_id, part_index)` with stable logical reply IDs, original text/order and creation time. Provider rendering does not mutate it. | +| `channel_delivery_batches` / `channel_delivery_parts` (implemented) | Immutable provider text/order and original recipient/user/org; CAS attempt token/counter, due time, provider ID, safe reason and delivery state. Partial/unknown/blocked work preserves recipient order. | +| `channel_webhook_receipts` (implemented) | Unique connection/body digest, original system, receive time, stable inbox/status references, and bounded safe quarantine metadata. No raw payload retention or automatic poison replay. | +| `channel_delivery_events` (implemented) | Deduplicated connection/message/recipient/status observations; projection joins the exact original recipient/provider ID and never regresses read/delivered proof. Early unmatched callbacks remain available for later matching. No agent input. | +| Channel identity port (implemented) | Resolve a channel sender into a fresh `Principal`; client plugins must still supply account linking and permissions. | +| Channel/delivery/control/sender ports (implemented) | Generic typed boundaries for durable channel work, immutable provider plans, attempt outcomes and static controls. Client identity remains separate. | + +Implemented inbox states are `queued`, `preparing`, `prepared`, `running`, +`answered`, `done`, `denied`, `failed`, and `needs_review`. Claims have no expiry +that could silently rerun execution. Provider part states distinguish `ready`, +`sending`, `retry`, `accepted`, `delivered`, `read`, `failed`, `delivery_unknown`, +`blocked_window`, `blocked_identity` and `suppressed`; batches may also be +`blocked_format`. A raw provider error is never inserted into the agent prompt. + +Use environment/settings for the connection, separate from agent YAML. +Implemented intake settings: + +| Environment setting | Behavior | +| --- | --- | +| `EXTRA_WHATSAPP_ENABLED` | Defaults to false; the webhook routes are absent until explicitly enabled. | +| `EXTRA_WHATSAPP_CONNECTION_ID` | Stable deployment-owned connection ID, default `whatsapp`; do not repurpose an existing ID for another account/system. | +| `EXTRA_WHATSAPP_ACCOUNT_ID` | Required receiving account ID when enabled. | +| `EXTRA_WHATSAPP_PHONE_NUMBER_ID` | Required receiving phone-number ID when enabled. | +| `EXTRA_WHATSAPP_APP_SECRET` | Required app secret for signed notifications. | +| `EXTRA_WHATSAPP_VERIFY_TOKEN` | Required subscription token; must differ from the app secret. | +| `EXTRA_WHATSAPP_MAX_BODY_BYTES` | Defaults to 262144 bytes; configurable from 1024 to 1048576. | + +Enabling requires persistent SQL and migration 0009. The normal `agent-manager` +startup applies migrations by default; deployments that disable migration must +apply it separately. The target system is the validated startup system name. +The two secrets are excluded from settings dumps/representations and input +values are hidden in validation error messages. Values come from deployment +secrets; scrub subscription query strings from proxy/access logs. No real +settings or secrets were configured by these tasks. Intake records queued work; +an explicitly enabled worker can execute it and stage an answer. Outbound +settings below are required only when the sender is explicitly enabled. + +Implemented worker settings: + +| Environment setting | Behavior | +| --- | --- | +| `EXTRA_WHATSAPP_WORKER_ENABLED` | Defaults to false; enabling also requires intake and durable SQL. | +| `EXTRA_WHATSAPP_ORGANIZATION_ID` | Required organization for current principal validation. | +| `EXTRA_WHATSAPP_IDENTITY_RESOLVER` | Required trusted `module:Class` implementing `ChannelIdentityResolver`, with a zero-argument constructor. | +| `EXTRA_WHATSAPP_WORKER_CONCURRENCY` | Defaults to 4; between 1 and 16 independent sender jobs. | +| `EXTRA_WHATSAPP_WORKER_POLL_SECONDS` | Defaults to 0.5; positive and at most 60. | +| `EXTRA_WHATSAPP_WORKER_TURN_SECONDS` | Defaults to 120; positive and at most 3600, covering identity/preparation/execution. | +| `EXTRA_WHATSAPP_WORKER_DRAIN_SECONDS` | Defaults to 15; positive and at most 60 before remaining jobs are cancelled. | + +The resolver is instantiated once at startup and its async `resolve(address)` +is called for current authorization. It returns `None` for an unlinked/revoked +sender. Its optional async `close()` releases client resources at shutdown; +the default is a no-op. Both methods must be async, and plugin load/cleanup +errors are sanitized. No default organization principal is supplied. The +customer still implements account linking and current permission/credential +lookup; setting a plugin path alone does not implement Google login or JWT +verification. Bounded polling scans preparing/prepared/running work, and review +states stay blocked. Shutdown may await cleanup beyond the drain budget: +ownership cannot be released while engine-owned work is still closing. +No additional schema migration beyond 0009 is needed for WA-04. + +Implemented sender settings (migration 0010 required): + +| Environment setting | Behavior | +| --- | --- | +| `EXTRA_WHATSAPP_SENDER_ENABLED` | Defaults to false. Requires the enabled worker and its trusted identity integration. | +| `EXTRA_WHATSAPP_API_VERSION` | Required explicit `vN.0` Graph version supported by the actual account; no version is chosen automatically. | +| `EXTRA_WHATSAPP_ACCESS_TOKEN` | Required separate secret, excluded from settings representations/dumps and distinct from webhook secrets. | +| `EXTRA_WHATSAPP_SUPPORT_CONTACT` | Required real public escalation contact; at most 512 UTF-8 bytes. | + +One owned HTTP client sends to the fixed HTTPS Graph host, with redirects, +environment proxies and transport-level retries disabled. Response bytes and +request duration are bounded. Known pre-submission connection/pool failures, +structured 429/5xx rejections and explicit transient 400 rejections can retry; +attempts are capped at four with exponential backoff/jitter and a 300-second +delay cap. Other 4xx failures stop. Ambiguous I/O, malformed success or lost +acceptance persistence becomes `delivery_unknown`, never an automatic resend. +Current account/version-specific error codes still need verified pilot fixtures. + +The same exclusive supervisor runs one outbound submission at a time, while +engine jobs retain their existing bounded concurrency. Shutdown cancels/joins +sends and commits their uncertain state before releasing ownership. Callback +projection runs in bounded pages and matches the connection, system, provider +ID and original recipient. No guessed callback-correlation field is used; +unknown submissions without a verified provider ID require operator review. + +With sending enabled, exact `/new`/`שיחה חדשה`, `/help`/`נציג`, `/stop`/`הפסק` +and `/start`/`המשך` bypass the model. Stop is applied atomically at intake, +even if an earlier model run is active. A sequence cutoff suppresses old input +and replies permanently; start permits subsequent messages but never grants +employee access or resurrects old replies. Already-submitted network requests +cannot be recalled. Reset keeps history and changes the binding generation +once. Static help/media/start/reset feedback obeys suppression and the response +window. Failed/unknown/review work still needs an operator; start does not +reconcile uncertain delivery or execution. + +Only verified customer-message kinds update the window. System/unknown kinds +are quarantined and cannot open it. Migration backfill uses known text/control +kinds conservatively because old unsupported rows lack the original media type. + +## 5. Identity and protected resources + +The webhook signature authenticates the provider notification. It does not +prove that the sender is an employee or may access organization data. The +business-number credential authorizes transport only and must never become a +user's host/MCP access token. + +For public information, create a channel-scoped `Principal.external(...)` with +no organization roles or privileged credentials. Keep its stable sender mapping +server-side. Roles, tenant selection, and target conversation never come from +message text, a displayed name, or LLM output. + +For internal information, introduce a client-owned account-linking plugin: +issue a short-lived, single-use browser link bound to this sender, authenticate +through the organization's existing login, confirm the binding, and store a +revocable association. Resolve permissions afresh for each turn and use an +appropriate delegated credential. Until that exists, the channel cannot expose +internal tools or documents. Do not reuse `link_anonymous()` as though a +WhatsApp identity were an existing anonymous web session. + +The user selected a generic identity integration, with Google login and a +JWT-issuing authentication server to be implemented later. That trusted server +will authenticate the user; the client integration must verify its JWT and +resolve the revocable sender/account association into `Principal`. The channel +does not interpret message text as a JWT or grant access based on a Google +email alone. Login, JWT verification, and account-link persistence are outside +WA-02; the implemented port deliberately has no provider dependency. + +WA-02 requires that resolver at preparation, rejects anonymous or +wrong-organization principals, and prevents a binding from silently changing +owners. It resolves current permissions for every turn without storing the +resolved credentials or roles in channel rows. WA-04 refreshes identity after +preparation immediately before the execution claim, and again before staging a +persisted internal answer. Before delivering delayed internal replies, the +enabled sender verifies that the original recipient still has the same authorized +account link; a revoked or changed link must block delivery, never retarget an +old answer to another user. + +The current code projects `Principal` into `RunContext.auth_context`, and the +engine already projects non-secret identity into `GraphState["run_context"]`. +Existing engine tests cover context-dependent allow/deny, streaming, and +resolver errors. Older project documentation reports this bridge as missing; +that statement must not be used as a reason to rewrite working engine code. +This does not establish a trusted identity for WhatsApp senders. Before enabling +the internal pilot, implement and test the channel identity/account-linking +plugin and its complete access/tool enforcement path, including denial for +unlinked, revoked, and cross-organization users. A public pilot would instead +need an operator-reviewed public +deployment profile with an explicit allowlist of nodes, resolvers, tools, and +MCPs. Startup validation rejects configurations outside that profile; absence +of a `protected` flag is not evidence that data is public. Security remains +enforced in access/tool/data adapters, not by prompts. See +[SIDECAR_CONTEXT_AUTH.md](SIDECAR_CONTEXT_AUTH.md). + +## 6. WhatsApp behavior and operations + +Free-form replies are permitted inside the 24-hour window after the latest +user message. Outside it, approved templates are required. Check the window +when sending, including retries; an outbound message or status callback does +not reopen it. The MVP blocks expired-window replies for review instead of +inventing a template. Provide the configured support contact for escalation, +and honor opt-out requests by suppressing pending and subsequent sends until +the applicable permission is restored. [WhatsApp messaging policy](https://business.whatsapp.com/policy) + +Track the newest verified inbound timestamp without moving it backwards for +late events; reject implausible future timestamps for window calculations. +Proactive campaigns, template authoring, and consent collection for those +campaigns are deferred. User support instructions and any data-retention notice +should describe the actual organization and deployment. + +Before the pilot, an administrator must confirm number ownership, account +setup, appropriate messaging credentials, webhook subscription, HTTPS routing, +and billing readiness. If the number already uses the WhatsApp Business app, +verify current coexistence or migration eligibility before changing it; this +plan does not promise simultaneous app/API operation or history import. Pin +an account-supported Graph API version explicitly rather than copying a +version from an old SDK example. + +Persist conversations, the queue, and the outbox on durable storage. Define +retention/deletion for normalized messages, identities, and quarantine data; +raw webhook bodies should be short-lived and access-restricted. Redact tokens +and sender identifiers from ordinary logs. Monitor inbox age, run duration, +outbox age, signature failures, provider rejections, unknown delivery, and +work requiring review. Correlate internal event, conversation, run, and provider +message IDs without logging message bodies. + +Disabling the connection must pause intake/execution/sending according to an +explicit runbook while preserving pending records. Drain workers on shutdown; +do not rely on process lifetime to retain accepted work. The deployment must +start `agent-manager`; the current image's default `agentctl serve` command +does not provide the conversation service or this proposed channel. + +## 7. Implementation sequence + +These are new work items, not completion claims or replacements for the +historical numbered task files. Before coding each item, create a task with +its exact file allowlist and acceptance criteria. Every meaningful behavior +ships with tests, and each implementation task must pass `make check`. + +| Item | Scope / proposed file boundary | Acceptance criteria | +| --- | --- | --- | +| WA-00: Design and account preflight | Proposed ADR under `docs/adr/`, task definitions under `tasks/`, this plan. | Record account/number eligibility, API version, intended assistant use case, support contact, selected safe agent config, and approved ownership/transaction contracts. | +| WA-01: Idempotent manager preparation — implemented | Task 0013; optional domain port, manager service, memory/SQL adapters, migration 0007, and manager tests. | Same key yields the same turn; conflicting identity/content is rejected; receipt and user append are atomic; crash boundaries reconcile run registration without a second turn. Existing clients still pass. | +| WA-02: Channel storage and identity boundary — implemented | Task 0014; `agent_manager/domain/channels/`, `application/channels/`, `infrastructure/persistence/channels/`, migration 0008, corresponding tests. | Typed bounded events/ports; atomic deduplication and claims; ordered reset; immutable reply staging; fresh same-organization identity; restart and rollback. Customer account linking remains a deployment prerequisite. | +| WA-03: Cloud API intake boundary — implemented | Task 0015; `api/routes/whatsapp.py`, `infrastructure/channels/whatsapp/`, settings/composition, domain/SQL callback port, migration 0009, and tests. | Challenge and byte-level signature verification, account checks, complete batch parsing, separate status observations, safe quarantine, atomic commit-before-ack, and no agent work. Disabled by default. | +| WA-04: Turn worker and recovery — implemented | Task 0016; channel processor/supervisor, active-work scan, authorized service response lookup, client identity loader, ownership adapter, settings/lifespan, and tests. | Per-sender order and independent concurrency; fresh identity; one execution claim; saved-answer recovery; ambiguous work held for review; approvals cancelled; streams awaited before releasing ownership; engine reused. | +| WA-05: Outbound delivery and controls — implemented | Task 0019; sender/rendering adapters, delivery/control ports and services, SQL plans/projection, migration 0010, settings/supervisor wiring and tests. | Ordered Unicode/grapheme-safe splitting; accepted/delivered distinction; bounded retry; unknown-delivery review; fresh original-owner/window/opt-out checks; unsupported media/reset/help/stop/start without model calls. Disabled by default, verified locally with a fake provider. | +| WA-06: Pilot packaging and operation | New WhatsApp example/runbook, docs index, deployment command/configuration and fake-provider end-to-end tests. | Documented administrator setup and rollback; durable restart rehearsal; metrics/redaction; full fake-provider round trip; then an explicitly authorized real-number smoke test. | + +WA-00 → WA-01 → WA-02 → WA-03 → WA-04 → WA-05 → WA-06 is the delivery order. +The identity/account-linking prerequisite is required for the confirmed internal +pilot. Reuse the engine's existing context propagation and introduce any needed +client-plugin work as a separate scoped task, not a runtime rewrite. + +### Required test matrix + +| Area | Cases to prove | +| --- | --- | +| Provider authenticity | Valid/invalid challenge, missing/bad signature, byte-level body changes, oversized payload, wrong receiving account. | +| Inbox | Multiple messages per callback, duplicate and concurrent duplicate callbacks, unsupported events, status-only callback, failed commit, quarantined poison event. | +| Identity | Two senders cannot share histories; same sender on another connection is isolated; message text cannot choose user/org/roles; WhatsApp token never reaches business tools. | +| Turns | Context retained across turns; idempotent prepare conflict; branch/head conflict; ordered resets; concurrent senders remain independent; budgets and failures are recorded. | +| Recovery | Crash before/after inbox commit, preparation commit, execution claim, answer persistence, outbox creation, provider acceptance, and delivery-ID persistence. | +| Sending | Rate limiting/transient failure, permanent rejection, ambiguous timeout, duplicate/out-of-order statuses, multi-part partial delivery, service-window expiry during retry. | +| Product controls | Hebrew/RTL/emoji preserved, unsupported audio/image feedback, opt-out cancels queued sends, support command returns real contact, disable/shutdown preserves accepted work. | +| Existing runtime behavior | No WhatsApp dependencies in the engine, one engine build per process, request context isolated, web/API/CLI tests remain green. | + +Unit and component tests use a fake engine, controlled clock, fake identity +plugin, fake Cloud API transport, and local test storage. No live Meta, LLM, +or MCP calls belong in the automated gate. The real account smoke test is a +separate deployment activity. + +## 8. Later milestones and remaining decisions + +Later scope: media and voice transcription, interactive approval flows, +an operator inbox, approved +template notifications, additional numbers, customer self-service onboarding, +and distributed workers. In particular, a `pending_approval` result must never +be auto-approved or treated as an empty final answer: keep the first-release +agent configuration free of approval-dependent actions. If an unexpected +approval occurs, use the manager's normal cancellation path, record the +outcome, and offer the support contact without exposing tool arguments. If +cancellation cannot be confirmed, stop that sender's work for review. + +Inputs still needed before channel activation: whether this is an existing +Business-app number, the first agent/config, the authentication server's JWT +contract and account-linking integration, the actual support contact, and the +pilot hosting/database environment. Internal information access and the +Google-login/JWT direction are confirmed. These do not block provider-independent +storage or the provider boundary; resolve them before internal activation. + +Some current Meta developer reference pages returned throttling/errors during +research. Meta's official Postman reference and collection were readable and +verify the original envelope/text/status contract used by WA-03; the archived +official SDK reference supports challenge/signature authentication and is not +installed. Recheck account-specific provider payloads, sender +identifier formats, account models, send limits, current AI-provider eligibility, +and permissions against the account's current documentation in WA-00. Do not +install the archived Node SDK into this Python application. diff --git a/docs/WHATSAPP_REVIEW.md b/docs/WHATSAPP_REVIEW.md new file mode 100644 index 00000000..4a39222d --- /dev/null +++ b/docs/WHATSAPP_REVIEW.md @@ -0,0 +1,227 @@ +# WhatsApp implementation explanation and review + +Review date: 2026-10-16. Scope: the uncommitted WA-01–WA-04 implementation, +tasks 0013–0016, reviewed and hardened in +[task 0017](../tasks/0017-whatsapp-review-and-hardening.md), plus the bounded +test-resource follow-up in [task 0018](../tasks/0018-sql-test-resource-cleanup.md). + +## 1. Summary + +The implemented path receives authenticated notifications, stores them before +acknowledging, resolves the sender through a required customer identity plugin, +and runs the existing Extra conversation service. It persists the assistant's +answer and stages an immutable outgoing reply. Actual provider sending, +Google/JWT account linking, and production setup remain unfinished. + +```mermaid +flowchart TD + A[Signed WhatsApp notification] --> B[Verify signature, target and bounds] + B --> C[(Durable inbox and duplicate receipts)] + C --> D[Worker: ordered per sender] + D --> E[Fresh organization identity via customer plugin] + E --> F[Existing ConversationService and shared Engine] + F --> G[(Saved answer and outgoing reply)] + G -. Planned WA-05 .-> H[Send through Meta] + I[Google login and JWT account linking: planned] -.-> E +``` + +| Stage | What exists | Why it matters | +| --- | --- | --- | +| WA-01 | Optional idempotent turn preparation, original history and run IDs, atomic receipts in memory/SQL. | Retrying queue bookkeeping does not add a second user turn. Preparation itself is not permission to execute again. | +| WA-02 | SQL inbox, sender/conversation binding, execution claim, immutable outbox, generic identity resolver port. | Serializes a sender's messages, keeps different senders separate, and requires an organizational identity before internal work. | +| WA-03 | Opt-in GET/POST webhook, challenge verification, HMAC over original bounded bytes, typed batches and safe quarantine. | HTTP success follows a complete durable commit. Invalid siblings can be isolated; delivery observations do not become agent input. | +| WA-04 | Opt-in supervised worker, exclusive connection ownership, fresh identity, existing manager streaming, stored-answer recovery. | Uses the engine built at startup; uncertain executions stop for review instead of rerunning tools. | + +The worker requires a trusted `module:Class` identity implementation and the +configured organization. This is an extension point, not a deployed login +service. A WhatsApp sender ID or a Meta signature never grants employee roles. +The webhook and worker default off. Migrations 0007–0009 are required for the +corresponding SQL features; this review adds no migration. + +## 2. Blocking issues + +Three concrete issues were reproduced and fixed before the review recommendation. + +### P1 — Cancelling shutdown released ownership before engine cleanup + +Location: [worker.py](../src/agent_manager/application/channels/worker.py), +`ChannelWorker.__aexit__`. + +Before: cancellation of the caller awaiting shutdown entered its `finally` +and closed the ownership stack while the shielded driver was still draining. +A second supervisor could acquire the same connection while the old engine +stream remained active. The previous slow-drain tests cancelled the driver, +so they did not cover cancellation of the lifespan caller itself. + +After: the entire drain-and-release sequence runs in an owned shutdown task. +The caller shields and joins that task through repeated cancellation, then +propagates cancellation. Ownership remains held until stream cleanup and +durable interruption bookkeeping finish. Existing bounded drain behavior is +preserved; cleanup can exceed that budget to maintain exclusivity. + +The regression test uses the actual local SQLite POSIX lock and a blocked fake +engine finalizer. Both single- and repeated-cancellation cases failed before +the change because a competing owner entered early, then passed after it. + +### P2 — Unpersistable text could poison a complete webhook transaction + +Location: [models.py](../src/agent_manager/domain/channels/models.py), +`bounded_identifier` and `bounded_text`. + +Before: a signed JSON message or status containing U+0000 passed normalized +validation. SQLite accepts it, so the existing storage tests did not expose +the PostgreSQL failure. PostgreSQL character columns cannot store NUL; a write +would roll back valid siblings and yield retryable HTTP 503 for the same poison +batch. This storage restriction is documented in the +[official PostgreSQL character-type reference](https://www.postgresql.org/docs/17/datatype-character.html). +The batch failure consequence follows from our atomic intake transaction; it +was not tested against a live PostgreSQL server. + +After: the shared normalized channel boundary rejects NUL in identifiers and +text before persistence. Existing normalization converts that validation +failure into safe quarantine metadata and preserves valid message/status +siblings. Outbox staging rejects an invalid part before writing any part or +releasing its sender. Valid Unicode, line breaks, and exact text remain intact; +input is not silently rewritten. + +Nine new failing cases demonstrated missing rejection: three inbound fields, +three status fields, two normalized-domain cases, and atomic reply staging. +They passed after the two boundary validators were corrected. + +### P2 — Ownership failure remained healthy during slow cleanup + +Location: [worker.py](../src/agent_manager/application/channels/worker.py), +`ChannelWorker._serve` and its public `running` property. + +Before: after ownership verification failed, the driver awaited child cleanup +before its task became done. During slow cleanup, `running` remained true and +the existing health route could still return HTTP 200. This concealed a failed +supervisor from readiness monitoring even though it could no longer claim work. + +After: a fatal driver exception sets the stop signal before awaiting cleanup. +The public liveness property and the existing `/health` mapping therefore report +failure immediately, while shutdown continues to await child cleanup safely. +A regression holding the engine finalizer open failed before this correction +and passed afterwards. The HTTP response contract itself is unchanged. + +No unresolved blocking finding remains within the implemented WA-01–WA-04 scope. +This conclusion does not establish readiness for a live internal-information +pilot; its missing components are listed below. + +## 3. Non-blocking issues + +These are remaining delivery stages, not implemented features: + +- WA-05: send the outbox through Meta, preserve reply/part order, distinguish + accepted from delivered, handle retries and uncertain delivery, and recheck + authorization immediately before delayed delivery. +- Complete the trusted Google/JWT login and revocable sender/account mapping. + Test actual organization access and tool/data permissions through that plugin. +- Implement reset/help/opt-out behavior and unsupported-media feedback; storage + reset support alone is not a user-facing command. Approvals currently cause + cancellation or review, with no WhatsApp approval experience. +- WA-06/account preflight: actual account compatibility, administrator setup, + operational review/recovery procedures, and a controlled full round trip. + +An item in `needs_review` deliberately retains its sender claim. There is no +operator review UI or automatic ambiguous-run replay. The outbox currently +contains saved replies, with no provider delivery guarantee. + +The full gate also exposed a separate test-resource defect, corrected in task +0018: repository-contract fixtures and SQL conversation-creation helpers owned +engines without disposing them. Test fixtures now close their engines in +`finally`, including when assertions or setup fail. Separate engine instances +are retained for the existing concurrency tests; product code is unchanged. + +## 4. Architecture concerns + +Verdict: conforms within scope. Transport, persistence, worker supervision, and +conversation lifecycle belong to `agent_manager`; the engine has no WhatsApp +imports. Startup builds the engine once, while principals and execution context +remain per request. The worker calls the existing service rather than its own +HTTP endpoints. YAML validation/compilation and public engine, plugin, and +conversation HTTP contracts are unchanged by this review. + +The corrections enforce existing decisions in ADRs 0007–0009; a new contract +ADR is unnecessary. ADR 0009 now explicitly describes cancellation of the +shutdown caller. SQLite ownership requires local POSIX storage. PostgreSQL +ownership requires a dedicated persistent session, not transaction pooling. + +Older project-wide status prose says that identity projection into the engine +is missing. Current code already projects `Principal` into `RunContext`, then +non-secret identity into graph state, with access tests. That older prose is +not evidence that WhatsApp account linking exists; see the +[plan's identity section](WHATSAPP_INTEGRATION_PLAN.md#5-identity-and-protected-resources). +No engine rewrite was needed for this integration review. + +## 5. Security concerns + +Reviewed boundaries: original-byte signature checking before parsing, target +account/number isolation, bounded input, no browser-auth dependency on webhook +routes, fresh same-organization identity before execution and answer staging, +binding ownership continuity, and sanitized plugin/storage failures. Channel +rows do not persist resolved credentials or role claims. Prompt text is not +used as an authorization boundary. + +Internal deployment still requires a real client identity/access implementation +and enforcement at the tool/data layer. The future sender must reauthorize the +original recipient rather than redirect an old reply after account relinking. +Reviewing generic boundaries and fake credentials is not verification of the +customer's eventual login, permissions, or Meta account. + +## 6. Testing gaps + +The new regression cases exercise real local file SQLite, actual POSIX locks, +public manager/channel methods, controlled cancellation, and fake engine and +identity adapters. Existing suites cover receipt concurrency/rollback/restart, +owner and organization isolation, HTTP signatures and limits, malformed batch +isolation, migrations, sender ordering, answer recovery, approvals, and startup. + +No live PostgreSQL server, Meta account, model, MCP server, Google login, JWT +issuer, production database, or real phone number was used. PostgreSQL ownership +has controlled connection-contract tests; backend deployment verification and +the complete outbound round trip remain pending. The NUL restriction was +checked against the official database reference, with local boundary tests. + +Final `make check` passed: **1,327 tests**, 24 baseline dependency-warning +instances, formatting for 339 files, lint, mypy for 335 files, and generated-stub +checks. Whitespace and local documentation links/anchors passed. New coverage +in this review totals twelve regression cases. + +Early full gates also reported intermittent aiosqlite worker-thread exceptions +(`Event loop is closed`), sometimes during unrelated later tests. Creation-site +diagnostics traced unclosed engines to the two older test modules corrected in +task 0018. A deterministic local diagnostic retaining connection objects showed +38 of 42 connections still open before that correction, and zero of 42 after +it; all 61 tests in those modules passed. No warning filter or dependency change +was used to conceal the defect. The final full-gate result is recorded in tasks +0017/0018; the final run contained no closed-loop aiosqlite thread warning. + +## 7. Suggested improvements + +Keep the next implementation focused on WA-05 outbound delivery and controls, +including a fake-provider full round trip. Build customer account linking as a +separate trusted integration before enabling internal access. Keep the existing +review-stop behavior for uncertain executions until an operator recovery design +is implemented; do not add an expiring lease that reruns tool effects. + +Files changed by this review: + +- `src/agent_manager/application/channels/worker.py` +- `src/agent_manager/domain/channels/models.py` +- `tests/agent_manager/test_channel_worker.py` +- `tests/agent_manager/test_channel_storage.py` +- `tests/agent_manager/test_whatsapp_normalization.py` +- `tests/agent_manager/test_repository_contract.py` and + `tests/agent_manager/test_sql_conversation_creation.py` (task 0018 cleanup). +- This review, `docs/README.md`, `docs/WHATSAPP_INTEGRATION_PLAN.md`, and + `docs/adr/0009-channel-worker-ownership-and-recovery.md`. +- `tasks/0017-whatsapp-review-and-hardening.md`, + `tasks/0018-sql-test-resource-cleanup.md`, and `tasks/README.md`. + +## 8. Final recommendation + +Approve the implemented stages with the three corrections, regression coverage, +test-resource cleanup, and the passing full gate recorded in task 0017. Provider sending, +trusted account linking, and pilot deployment remain subsequent work. No commit, +PR, account setup, or deployment was performed in this review. diff --git a/docs/adr/0007-durable-channel-identity-boundary.md b/docs/adr/0007-durable-channel-identity-boundary.md new file mode 100644 index 00000000..e149b946 --- /dev/null +++ b/docs/adr/0007-durable-channel-identity-boundary.md @@ -0,0 +1,73 @@ +# ADR 0007 — Durable channel work uses freshly resolved organization identity + +- **Status:** Accepted for implementation under the user's approved WhatsApp plan and continuation request. +- **Date:** 2026-10-15. + +## Context + +WhatsApp messages can be duplicated or delivered while a previous turn is +running. The sender's channel identifier is not an organization login. The +confirmed deployment must access internal information, making anonymous or +phone-number-derived organization privileges unacceptable. + +## Decision + +Keep typed channel events and storage in `agent_manager`, independent of the +engine and provider payloads. Introduce SQL inbox, binding, and outbox tables +through migration 0008. A connection/provider-message key deduplicates input; +the original target system, sender, kind, content, and provider timestamp cannot +be rebound. A binding is scoped by connection, system, and opaque sender ID. + +Each binding holds one active inbox claim. Atomic conditional updates serialize +its turns while allowing other bindings to progress. Preparing, prepared, +running, and review states keep that claim. Execution has a separate one-way +prepared-to-running claim. No automatic lease expiry or recovery reruns work. +A claimed reset advances the binding generation once; prior conversations are +retained. Staging one immutable ordered reply releases the sender for its next +turn, while delivery remains a later subsystem. + +A client-owned `ChannelIdentityResolver` resolves the sender to a fresh +`Principal` for each preparation. The generic application verifies a nonanonymous +identity for the configured organization, and an existing binding cannot change +owners implicitly. Resolver failures and revoked/unlinked users fail closed. +Do not persist resolved credentials or role claims in channel rows, and do not +promote message text into identity fields. Normalized user text is retained as +message content. The returned principal remains request-scoped and reaches the +normal manager preparation and engine context boundary. + +The selected future integration is Google login through a generic +JWT-issuing authentication server. JWT verification and revocable account +linking belong in that trusted client integration; this task implements the +provider-neutral port, without a login flow or JWT parser. + +Use deterministic conversation IDs per binding generation and task 0013's +idempotency key per inbox event. Separate commits between conversation creation, +preparation, and queue bookkeeping are therefore recoverable without creating a +second conversation/user message. This does not authorize executing twice. + +## Consequences / alternatives + +The identity port is implemented by the customer's trusted integration; it is +not a login implementation or a blanket organization grant. Account-linking UI, +worker reauthorization before execution and delayed internal delivery, and +actual provider delivery remain later tasks. Reauthorization must preserve the +original recipient/owner rather than redirect old answers after relinking. +An unlinked message is denied without creating a manager user, +conversation, or turn. A rejection after preparation instead requires review. + +SQL is required for durable channel work. Memory-mode manager deployments +continue to work and expose no channel repository. PostgreSQL uses the same +SQL adapter, but this task's local behavioral tests use SQLite and do not claim +live PostgreSQL deployment verification. + +In-memory locks/caches were rejected because they do not establish durable +ownership. Persisting a resolved principal or credentials was rejected because +permissions can change between turns and secrets must not enter queue rows. +Rebinding a conversation after a phone/account change was rejected because it +would hand the old owner's history to a new principal. + +## Enforcement + +Tests cover SQL duplicates/concurrency/restart/rollback, stale claim tokens, +single execution claims, ordered reset, immutable reply staging, owner and +organization isolation, resolver errors/revocation, and secret-free persistence. diff --git a/docs/adr/0008-authenticated-whatsapp-intake.md b/docs/adr/0008-authenticated-whatsapp-intake.md new file mode 100644 index 00000000..060d3088 --- /dev/null +++ b/docs/adr/0008-authenticated-whatsapp-intake.md @@ -0,0 +1,73 @@ +# ADR 0008 — Authenticate and commit webhook batches before acknowledging + +- **Status:** Accepted under the approved WhatsApp plan and explicit WA-03 continuation. +- **Date:** 2026-10-15. + +## Context + +Provider callbacks are batched, duplicated, and retried. Authentic notifications +may contain malformed messages alongside valid siblings. A successful HTTP +response must not lose accepted input, and a transport signature grants no +organization identity or tool privileges. + +## Decision + +Add opt-in GET/POST `/channels/whatsapp/webhook` to `agent_manager`. GET checks +the verification token and returns the exact challenge. POST bounds streamed +bytes and verifies `X-Hub-Signature-256` using the separate app secret before +parsing. Neither endpoint uses browser authentication or interprets user JWTs. +Existing conversation API and engine contracts remain unchanged. + +Settings require durable SQL and explicit account, number, and connection IDs. +The receiving system comes from the validated startup specification, never +from callback text. Separate provider Pydantic models normalize messages into +the existing channel domain. The initial verified envelope contract is +`whatsapp_business_account`, `entry[].id`, and +`changes[].value.metadata.phone_number_id`; message `from` stays opaque. +Unknown identity/account variants cannot implicitly gain a compatible mapping. + +Migration 0009 adds authenticated webhook receipts and normalized delivery +events. A connection/body digest deduplicates callbacks. One transaction writes +the receipt, inbox inputs/bindings, delivery events, and bounded quarantine +reason/position metadata. Reuse the existing inbox insert helper. Conflicting +provider-message input is quarantined without altering its original target. +Database failure rolls the whole batch back and returns HTTP 503. HTTP 200 +means durable intake only; it makes no execution or delivery claim. + +Quarantine stores a digest and generated reason/position codes, without raw +webhook bodies, validation input, message snippets, profile data, or raw +provider errors. Valid normalized text remains in the inbox. This trades raw +payload forensic replay for reduced retention; operator investigation requires +provider-side diagnostics or controlled fixtures. Status callbacks persist +separately and never enter the agent queue; delivery projection is later work. + +## Consequences / alternatives + +The webhook is absent by default, and explicit enabling starts intake only. +There is no execution worker or sender in this task. A later worker must use +the mandatory client identity boundary before internal work. + +Per-message commits and in-process background tasks were rejected: they cannot +atomically establish receipt of a whole callback. Rejecting every malformed +signed event was rejected because it causes poison retries. Unsigned events +are rejected before storage; oversized bodies are rejected rather than stored. + +Current Meta developer pages returned HTTP 429 during research. The archived +[official webhook reference](https://whatsapp.github.io/WhatsApp-Nodejs-SDK/api-reference/webhooks/start/) +supports the challenge/signature contract, and the +[official webhook types](https://github.com/WhatsApp/WhatsApp-Nodejs-SDK/blob/main/src/types/webhooks.ts) +document the original envelope/message fields. Meta's +[official Postman reference](https://www.postman.com/meta/whatsapp-business-platform/folder/tduohwq/webhook-payload-reference) +and [collection documentation](https://www.postman.com/meta/whatsapp-business-platform/documentation/wlk6lh4/whatsapp-cloud-api) +were readable and verify the wire key `phone_number_id`, text input, and +separate status callbacks. The archived SDK itself is not used. +Current account-model/BSUID contracts and account eligibility +must be rechecked with the actual account before deployment. No API version is +invented for intake; outbound version selection remains a later prerequisite. + +## Enforcement + +Tests exercise byte-level signatures, streamed limits, token challenges, +target isolation, complete batches, malformed siblings, safe quarantine, +atomic rollback, concurrent duplicates, restart, schema migration, startup +composition, and absence of manager/engine execution during callbacks. diff --git a/docs/adr/0009-channel-worker-ownership-and-recovery.md b/docs/adr/0009-channel-worker-ownership-and-recovery.md new file mode 100644 index 00000000..a24b5bdf --- /dev/null +++ b/docs/adr/0009-channel-worker-ownership-and-recovery.md @@ -0,0 +1,104 @@ +# ADR 0009 — Own channel recovery and never replay uncertain execution + +- **Status:** Accepted under the approved WhatsApp plan and explicit WA-04 continuation. +- **Date:** 2026-10-16. + +## Context + +Inbox deduplication alone does not authorize execution. Manager preparation, +engine completion, response persistence, and outbox staging have separate +commit boundaries. Identity can be revoked while a message waits. Recovery +must not mistake a live worker or a completed run without an answer for work +that is safe to execute again. + +## Decision + +Keep the worker in the manager application, using the existing service and +durable channel/run ports. Extend the channel port with bounded, cursor-based +active-work scanning. Add an authorized service lookup for the deterministic +persisted assistant response; transports never derive its message ID or query +unowned conversation history themselves. + +A supervised worker is separately opt-in and requires a configured organization +and trusted `module:Class` implementation of `ChannelIdentityResolver`. Load +and instantiate it once at startup; customer plugins own current account links, +credentials, and lookup resources. Resolver methods receive the opaque sender +address and return request-scoped principals. No Google/JWT implementation or +blanket organization identity is invented. Reauthorize after preparation, +immediately before claiming execution, and before recovering an internal answer. +The future sender must reauthorize again before delayed external delivery. + +When an enabled supervisor stops, `/health` returns HTTP 503 with `status: +degraded` and the existing system field. Default worker-disabled deployments +retain their existing health response. This exposes supervisor failure to +readiness monitoring while preserving already accepted inbox records. +An ownership failure marks the supervisor stopped before awaiting child cleanup, +so a slow engine finalizer cannot keep readiness healthy after ownership loss. + +One supervisor owns each connection before scanning active work. Local SQLite +uses a nonblocking POSIX file lock beside the canonical database path, released +by closing the descriptor. PostgreSQL uses a dedicated session advisory lock +and verifies the backend session while supervising; ownership loss aborts the +worker and cancels child tasks. Neither lock expires into a new execution claim. +See [Python flock](https://docs.python.org/3.12/library/fcntl.html) and +[PostgreSQL session advisory locks](https://www.postgresql.org/docs/17/explicit-locking.html#ADVISORY-LOCKS). +SQLite network filesystems/Windows and distributed tool replay remain unsupported. + +Bound active jobs and polling; prioritize recoverable preparing/prepared/running +inbox items, then claim queued work. Per-sender binding ownership remains the +serialization boundary. Each active item is scheduled at most once locally; +the SQL prepared-to-running claim authorizes at most one engine invocation. + +Recovery rules: + +| Inbox / evidence | Action | +| --- | --- | +| Queued | Claim and prepare after current identity verification. | +| Preparing / prepared | Reuse WA-01 preparation, refresh identity, and claim execution once. | +| Persisted assistant response matching this conversation/run/user message | Stage that exact response with stable IDs; never regenerate it. | +| Running without a persisted response, including a completed run record | Hold for review; no automatic execution or tool replay. | +| Answered / done / denied / failed / review | Do not reopen automatically. | + +On a staging/storage failure, retain running work so a later scan can recover +the stored answer. Identity denial after preparation instead holds review and +does not expose that answer. Empty or answers exceeding the current staging +part limit require review; provider-specific formatting/splitting is WA-05. +Unsupported media invokes no model and is recorded as failed; feedback is later +product work. Unexpected approval is cancelled through the manager's normal +owner-bound path; cancellation must be confirmed and its stream closed before +releasing the sender. + +Shutdown stops new claims, drains existing tasks for a configured duration, +then cancels and awaits remaining streams before closing the manager and its +engine. An interrupted execution is held for review; preparation can be +reconciled using its existing receipt. Logs contain internal event IDs and +safe reason/type codes, never content, credentials, or raw plugin errors. +Cancellation cleanup is shielded and awaited even if it exceeds the drain +budget. The ownership lock is released only afterwards. +Cancellation of the lifespan caller itself also shields and joins the complete +shutdown task through repeated cancellation before propagating cancellation; +it cannot close the ownership stack while the driver is still draining. +Plugin lookup and cleanup methods are validated as async; cleanup errors are +logged by type without preventing manager/engine shutdown. + +## Alternatives / consequences + +Lease-based reruns and process-local sender locks were rejected. A persistent +owner row without a liveness boundary cannot safely distinguish restart from +another live process. Session/OS ownership provides that boundary while SQL +claims continue to prevent duplicate execution. This is still a single-owner +channel deployment, not distributed failover for the engine's process-local +approvals or tool ledger. Advisory-lock ownership requires a direct/session- +persistent PostgreSQL connection, not transaction-pooling middleware. + +The internal repository/service additions are recorded here; existing +conversation HTTP, YAML, engine, and resolver signatures stay compatible. The +health status extension applies only to an enabled failed/stopping worker. +Enabling a worker without +a customer identity implementation fails closed. No outgoing transport is added. + +## Enforcement + +Behavior tests use local SQL and fake engine/identity adapters, exercising +fresh-context isolation, duplicate/restart boundaries, saved-answer recovery, +sticky ambiguous work, exclusive ownership, shutdown, and startup composition. diff --git a/docs/adr/0010-whatsapp-outbound-delivery.md b/docs/adr/0010-whatsapp-outbound-delivery.md new file mode 100644 index 00000000..c1ba41ed --- /dev/null +++ b/docs/adr/0010-whatsapp-outbound-delivery.md @@ -0,0 +1,100 @@ +# ADR 0010 — Persist provider delivery separately from logical answers + +- **Status:** Accepted under the approved WhatsApp plan and explicit WA-05 request. +- **Date:** 2026-10-16. + +## Context + +WA-04 stores immutable logical replies. Provider rendering, retries and receipt +callbacks have different commit boundaries. An accepted send is not evidence +of delivery; a transport timeout is not evidence of rejection. Delayed internal +replies must not escape after account revocation or opt-out. + +## Decision + +Keep logical outbox rows intact, including replies staged before this migration. +Migration 0010 adds delivery batches and immutable provider parts, and sender +preference/window metadata. A bounded planner freezes parts once per event; +provider IDs and attempts live on those parts. No model is called for delivery. +UTF-16 units are a conservative application bound (4096 by default), with full +grapheme segmentation via an explicit regex dependency and paragraph preference. +Rendered bodies remain plain text; no Markdown-to-HTML conversion, truncation, +normalization or bidi rewriting occurs. Unrenderable/oversized batches stop. + +The existing exclusive supervisor also owns the outbound pump. One network +submission is in flight at a time in this first single-number implementation; +engine jobs retain bounded independent-sender concurrency. Each part is claimed +durably before submission. Later parts/turns wait for prior acceptance; unknown, +failed or blocked batches stop that recipient while other recipients progress. +Startup/recovery marks abandoned sending attempts unknown, never ready again. +HTTP retries inside the client are disabled. Only a known rejection or failure +before submission may retry, with capped exponential backoff, jitter and attempts. +The adapter recognizes structured 429/5xx rejections and a structured 400 whose +`is_transient` flag is explicitly true; it does not guess a current Meta error +code catalog from third-party references. Other client errors are terminal. +Ambiguous I/O, malformed success, cancellation or lost acceptance persistence +requires review. No exactly-once external delivery is promised. + +Before every attempt resolve a fresh identity for the original immutable +recipient/owner and configured organization, then recheck persisted suppression +and service-window state. Static control replies contain no internal data and +do not require a linked employee. They still obey suppression/window rules. +The HTTP adapter sends only to the fixed HTTPS Graph host with an explicit +deployment-selected API version; tokens are secret settings, not YAML, rows, +URLs, prompts or logs. There is no default production token or chosen API version. + +Callbacks retain their existing authenticated intake. Projection joins exact +connection/system/provider-ID/recipient, including an early saved callback. +Read/delivered evidence does not regress on late sent/failed observations. +If verified delivery later supersedes an earlier failed callback, recompute the +batch from its parts so remaining parts can proceed without resending that part. +Accepted IDs never trigger automatic resends after delivery failure. +Unknown sends without a verified ID cannot be reconciled from guessed metadata. + +Exact commands are normalized at the provider edge: /new or שיחה חדשה, +/help or נציג, /stop or הפסק, /start or המשך. Stop is durable at intake, +before acknowledgement, even if a model is still running. A sequence cutoff +permanently suppresses previous pending input/replies; start does not resurrect +them and does not grant organization access. Controls use the ordinary sender +queue, create no model runs and retain history on reset. Unsupported media +receives a static explanation without downloading data. + +## Protocol evidence and limitations + +Meta's [official archived text reference](https://whatsapp.github.io/WhatsApp-Nodejs-SDK/api-reference/messages/text/) +and [message types](https://github.com/WhatsApp/WhatsApp-Nodejs-SDK/blob/main/src/types/messages.ts) +support the text body/preview flag, recipient, and returned message ID contract. +The [official messaging policy](https://business.whatsapp.com/policy) confirms +the 24-hour response window, opt-out and escalation requirements. Current Meta +developer message/error pages returned HTTP 429 during implementation research. +The application cap is conservative; the operator must verify the configured +version/account's actual contract and eligibility during WA-00/WA-06. The +archived SDK is not installed. No unverified callback correlation field or +account variant is enabled, and no current Graph version is guessed. +System/unknown event kinds cannot open the service window. Existing unsupported +rows are excluded from the conservative window backfill because their original +provider message type was not retained. + +## Consequences / alternatives + +A new provider layer avoids mutating already-staged answers or coupling generic +manager completion to WhatsApp. A broker/distributed sender is unnecessary for +this pilot. Expiring send leases, blind timeout retries and regenerating answers +were rejected because they can duplicate messages or tool effects. + +Internal channel ports and additive settings/schema are extended under this +approved stage. Existing engine, YAML, conversation HTTP and identity resolver +signatures remain compatible. Sender is separately opt-in and cannot run without +the existing supervised worker. Customer account linking and live activation +remain separate prerequisites. + +Downgrade preserves the previous logical inbox/outbox schema but removes new +delivery evidence. A live rollback must preserve/restore that evidence and +reconcile any submitted attempts before re-enabling sends; dropping tables and +replanning old replies is not a safe delivery recovery procedure. + +## Enforcement + +Local SQL and fake HTTP/model/identity tests cover immutable planning, Unicode +roundtrip/bounds, recipient order, CAS/restart, callback order, failure taxonomy, +fresh identity/suppression/window checks, controls, shutdown and migration. diff --git a/tasks/0014-channel-storage-and-identity.md b/tasks/0014-channel-storage-and-identity.md new file mode 100644 index 00000000..51d51402 --- /dev/null +++ b/tasks/0014-channel-storage-and-identity.md @@ -0,0 +1,90 @@ +# Task 0014 — Durable channel storage and organization identity (WA-02) + +## Goal / context + +Implement the next stage of the approved WhatsApp integration plan after task +0013: a provider-independent inbox, sender/conversation bindings, execution +claims, reply staging, and a client-owned organization identity boundary. +The confirmed first deployment needs internal organization information. +The customer identity integration is intentionally deferred; its selected +direction is Google login with a generic JWT-issuing authentication server. +This task implements the mandatory provider-neutral resolver boundary only. + +## Read first + +`AGENTS.md`, `.ai/README.md`, project-architecture, architecture-review, +senior-python-engineering, sidecar-auth-context, testing, documentation and +code-review skills; the WhatsApp plan and ADRs 0006/0007. + +## Scope / files allowed to change + +- This task, `tasks/README.md`, `docs/README.md`, `docs/WHATSAPP_INTEGRATION_PLAN.md`, + and new `docs/adr/0007-durable-channel-identity-boundary.md`. +- New `src/agent_manager/domain/channels/{__init__,models,repository,identity}.py`. +- New `src/agent_manager/application/channels/{__init__,preparation}.py`. +- New `src/agent_manager/infrastructure/persistence/channels/{__init__,tables,repository,replies}.py`. +- `src/agent_manager/infrastructure/persistence/tables.py`, new migration + `0008_channel_storage.py`, and `src/agent_manager/composition.py`. +- New `tests/agent_manager/test_channel_storage.py`, + `tests/agent_manager/test_channel_identity.py`, and + `tests/agent_manager/test_channel_migration.py`. + +## Requirements / acceptance criteria + +- [x] Typed events, states, ports, and explicit bounded input validation. +- [x] SQL inbox receipt and binding creation are atomic; duplicate provider + messages reuse one event and conflicting input is rejected without mutation. +- [x] Sender bindings are isolated by connection/system/sender. Ordered claims + serialize each sender while independent senders can progress concurrently. +- [x] A prepared turn receives one atomic execution claim. Ambiguous/review + work cannot be silently reclaimed or skipped; no lease expiry reruns it. +- [x] Reset operates only on the current claimed control event, advances the + binding generation once, and preserves prior conversation history. +- [x] Reply parts commit together, keep stable keys/order, and reject conflicting + restaging. This task does not send them or claim external delivery. +- [x] A client identity resolver is mandatory for internal preparation. Missing, + anonymous, revoked, wrong-organization, and changed-owner identities fail + closed before a new conversation/turn; resolved credentials/roles are never + persisted in channel rows. +- [x] Current permissions/credentials are resolved for each turn. A crash + between manager preparation and recording queue IDs reuses WA-01's receipt. +- [x] Migration and file-SQLite restart tests pass. Composition exposes durable + channel storage only with SQL; process-memory mode has no channel backend. +- [x] `make check` passes; no engine, YAML, HTTP, or provider API changes. + +## Out of scope + +Meta webhook parsing/signatures, background workers, actual sending/delivery, +automatic replay of interrupted execution, login/account-linking browser UI, +customer-specific identity lookup, live account setup, and production deployment. + +## Commands / report + +Run focused tests while developing, then `make check` and `git diff --check`. +Report using AGENTS.md section 9, clearly distinguishing this storage/identity +boundary from a live WhatsApp connection. + +## Implementation result + +Completed WA-02 on 2026-10-15. Migration 0008 adds normalized inbox, sender +bindings, and immutable reply parts. SQL composition exposes the repository; +memory mode exposes none. The mandatory identity port validates a freshly +resolved organization principal before preparing an internal manager turn. + +Validation: `make check` passed (1,204 tests, including 45 new channel cases; +21 warnings from existing dependency-warning categories). Formatting, lint, +and type checking passed. Tests cover concurrent intake/preparation/reset/reply +staging, the delayed-reset race, restart, failed-transaction rollback, migration +upgrade/downgrade, revoked/malformed identities, and sanitized error handling. + +Self-review found and corrected two concurrency gaps: preparation bookkeeping +now reuses a concurrently committed receipt, and a delayed duplicate reset +cannot clear a newer conversation binding. Each has a regression test. No +blocking review findings remain within this task's scope. + +No live Meta, model, MCP, or production database was accessed. PostgreSQL is +supported by the SQL adapter but was not exercised against a live server. +Account linking, Google login/JWT verification, worker reauthorization, +webhooks, and actual sending remain subsequent work. The internal channel +cannot be activated with this storage boundary alone. Next: WA-03, the +disabled-by-default provider/webhook boundary. diff --git a/tasks/0015-whatsapp-webhook.md b/tasks/0015-whatsapp-webhook.md new file mode 100644 index 00000000..e79ff49e --- /dev/null +++ b/tasks/0015-whatsapp-webhook.md @@ -0,0 +1,105 @@ +# Task 0015 — Authenticated durable WhatsApp intake (WA-03) + +## Goal / context + +Implement the next stage of the approved WhatsApp plan: an opt-in Cloud API +webhook for one configured account/number and the manager's compiled system. +The user explicitly requested this stage after the WA-03 description. + +## Read first + +AGENTS.md, .ai/README.md, project-architecture, senior-python-engineering, +testing, architecture-review, documentation, code-review; the WhatsApp plan +and ADR 0008. Verify protocol details against official Meta references. + +## Scope / files allowed to change + +- This task, tasks/README.md, docs/README.md, docs/WHATSAPP_INTEGRATION_PLAN.md, + and new docs/adr/0008-authenticated-whatsapp-intake.md. +- New domain/channels/webhooks.py under src/agent_manager/. +- New infrastructure/channels/__init__.py and + infrastructure/channels/whatsapp/{__init__,config,security,payloads,normalization,webhook}.py. +- New infrastructure/persistence/channels/{intake,webhooks}.py; existing + channels/repository.py and channels/tables.py, persistence/tables.py, and + new migration versions/0009_webhook_receipts.py. +- config.py, composition.py, api/app.py, and new api/routes/whatsapp.py. +- New tests/agent_manager/test_whatsapp_webhook.py, + test_whatsapp_normalization.py, test_webhook_storage.py, test_webhook_migration.py. + +All source paths above are relative to src/agent_manager/. Extract the existing +inbox insert transaction helper so webhook batches reuse its deduplication +without copying it or committing individual messages early. + +## Requirements / acceptance criteria + +- [x] Disabled by default. Explicit settings require distinct secret values, + configured account/number/connection IDs, and durable SQL. Secrets are hidden + from settings representations and validation messages. +- [x] GET /channels/whatsapp/webhook validates subscription mode/token and + returns the exact plain-text challenge, with no host-login dependency. +- [x] POST verifies HMAC-SHA256 over bounded original bytes before parsing or + writing; malformed/missing/ambiguous signatures fail closed. +- [x] Typed parsing processes all entries/changes/messages for the configured + account and number; unrelated targets are ignored. Sender IDs stay opaque. +- [x] Text/Unicode is preserved; unsupported media is normalized without + downloading it. Delivery events are separate from agent input. +- [x] Malformed authenticated items are durably quarantined with safe reason + codes and positions, preserving valid siblings. No raw payload/error text or + provider secrets enter receipts, logs, or agent prompts. +- [x] Inbox messages, bindings, normalized delivery events, quarantine metadata, + and receipt commit atomically before HTTP 200. Failure returns retryable 503; + duplicate/concurrent callbacks reuse durable receipts and immutable input. +- [x] SQL restart and migration upgrade/downgrade preserve prior channel data. +- [x] App startup wires intake once alongside the existing engine; webhooks + never create manager conversations, execute agents, or send provider requests. +- [x] Focused tests, make check, git diff --check, and doc links pass. + +## Out of scope + +Turn/delivery workers, sending or delivery projections, account linking, +Google/JWT integration, support/reset interpretation, media retrieval, live +Meta/account setup, and deployment. New Meta account/BSUID payload contracts +require verified account fixtures before being enabled; do not guess them. + +## Commands / final report + +Write behavior tests first, run focused tests during implementation, then +make check and git diff --check. Use AGENTS.md section 9's report; distinguish +local verified intake from a live WhatsApp round trip. + +## Implementation result + +Completed WA-03 on 2026-10-15. The optional endpoints are composed once at +startup. Strict Pydantic boundary models normalize only the configured target; +domain batches/receipts remain provider independent. SQL migration 0009 adds +receipt and delivery-observation tables. The existing inbox insertion was +extracted unchanged in purpose into a transaction helper shared by both SQL +adapters, keeping WA-02 duplicate/conflict behavior intact. + +`make check` passed: 1,268 tests including 64 new cases; formatting, lint, and +type checking passed (326 source/test files checked by mypy). The 24 warnings +come from existing dependency-warning categories, including three additional +Alembic configuration warnings exercised by the new migration test. +`git diff --check` and 43 local documentation links also passed. + +Tests use real local file-SQLite, a fake engine, and injected storage failures. +They cover signatures and byte tampering, exact challenge responses, bounded +streaming, full batches, malformed sibling isolation, separate status storage, +safe quarantine, concurrent duplicates, failed writes/commit rollback, restart, +and migration upgrade/downgrade. Startup builds the engine once and callbacks +create no manager conversations or agent runs. + +Self-review corrected malformed-collection coupling (valid messages and status +siblings now survive independently), missing-entry quarantine, excessive +content-length conversion, and unpersistable startup system IDs, with focused +regression cases. No blocking review findings remain within this scope. + +No real account, secrets, webhook subscription, provider sends, model calls, +MCP calls, or production database were used. PostgreSQL is supported by the +adapter but was not verified against a live server. Current account-model/BSUID +variants remain explicit compatibility work before deployment. Quarantine +retains safe metadata rather than raw bodies and cannot replay raw payloads. + +Next: WA-04 turn worker and recovery, including fresh organization identity +checks before execution and reconciliation of ambiguous runs. Google/JWT +account linking and outgoing delivery remain separate subsequent tasks. diff --git a/tasks/0016-channel-turn-worker.md b/tasks/0016-channel-turn-worker.md new file mode 100644 index 00000000..771168f2 --- /dev/null +++ b/tasks/0016-channel-turn-worker.md @@ -0,0 +1,115 @@ +# Task 0016 — Channel turn worker and conservative recovery (WA-04) + +## Goal / context + +Implement the approved plan's next stage: a supervised, explicitly enabled +manager worker using durable channel claims and the existing conversation +service. The confirmed pilot accesses internal organization information; +Google/JWT account linking remains deferred behind the generic identity port. + +## Read first + +AGENTS.md, .ai/README.md; project-architecture, senior-python-engineering, +testing, architecture-review, runtime-engine, sidecar-auth-context, +documentation, code-review; the WhatsApp plan and ADRs 0006–0009. + +## Scope / files allowed to change + +- This task, tasks/README.md, docs/README.md, docs/WHATSAPP_INTEGRATION_PLAN.md, + and new docs/adr/0009-channel-worker-ownership-and-recovery.md. +- Under src/agent_manager/: new application/channels/{processing,worker,worker_options}.py; + existing application/channels/preparation.py and application/conversation_service.py. +- New domain/channels/ownership.py; existing domain/channels/{repository,identity}.py. +- New infrastructure/channels/identity_loader.py and + infrastructure/persistence/channels/ownership.py; existing persistence/channels/repository.py. +- config.py, composition.py, api/app.py. +- New tests/agent_manager/test_channel_worker.py, + test_channel_worker_lifecycle.py, test_channel_worker_ownership.py. + +No schema migration is needed: bounded active-work scanning uses existing +states/claims. Add a service lookup for an authorized persisted run response; +do not reproduce manager response-ID logic in the worker. + +## Requirements / acceptance criteria + +- [x] Worker defaults off; enabling requires intake, durable SQL, an organization, + and a trusted module:class identity resolver. Invalid plugin configuration + fails startup without reflecting plugin exception text or credentials. +- [x] Exclusive connection ownership prevents concurrent recovery/supervisors: + POSIX file lock for local SQLite; dedicated PostgreSQL session advisory lock. + Ownership loss stops claims and cancels active work; no expiring execution lease. + An enabled worker that has stopped makes /health return 503/degraded. +- [x] Bounded concurrency allows independent senders to progress while durable + binding claims preserve each sender's order. Engine is reused from startup. +- [x] Preparation and execution use fresh external same-organization identities; + owner changes/revocation fail closed. Updated roles/credentials reach only the + request context, never channel persistence or ordinary logs. +- [x] Preparing/prepared work recovers via WA-01 receipts; an atomic execution + claim allows one engine invocation. Terminal run state without a response is + not permission to restart an already claimed execution. +- [x] Persisted assistant responses are recovered and staged with stable IDs, + without regeneration. Running work with no persisted response stops for review. + A failed outbox write can retry staging from the saved answer, never rerun tools. +- [x] Pending approvals are never approved: cancel through the normal manager + path; unconfirmed cancellation requires review. Empty/missing/oversized + answers require review and are not silently truncated. Unsupported media + causes no model work; product feedback/control commands remain WA-05. +- [x] Shutdown stops claiming, drains bounded work, then cancels/awaits remaining + streams before manager/engine/repositories close. Interrupted execution stays + blocked for review; safe preparation remains recoverable. +- [x] Tests cover ordering/concurrency, ownership, identity refresh/revocation, + restart/crash boundaries, answer recovery, failure/approval/timeout/shutdown, + and app lifespan. No live Meta/model/MCP or production database calls. +- [x] make check, git diff --check, and documentation links pass. + +## Out of scope + +Provider sending/delivery projection, WhatsApp-specific rendering/splitting, +support/reset/opt-out product controls, login/account-linking implementation, +automatic reexecution of ambiguous runs, distributed tool/approval recovery, +live account setup, and deployment. SQLite ownership requires local POSIX +storage; network-mounted SQLite and Windows locks are not enabled here. + +## Commands / report + +Test first; run focused cases while developing, then make check and +git diff --check. Report using AGENTS.md section 9, distinguishing staged +answers from actual provider delivery and a plugin port from deployed identity. + +## Implementation result + +Completed WA-04 on 2026-10-16. The opt-in worker shares the startup engine and +normal conversation service. It loads a mandatory client identity plugin once, +refreshes authorization before execution/staging, owns one connection lifetime, +and processes bounded independent-sender jobs. Existing HTTP conversation, +YAML, and engine contracts remain unchanged. The enabled-worker health extension +and internal repository/service additions are recorded in ADR 0009. + +Fresh validation: `make check` passed with 1,315 tests, including 47 new worker +cases; formatting, lint, and mypy passed (335 source/test files). The 24 existing +dependency warnings are unchanged from task 0015. `git diff --check` and 47 +local documentation links passed. No schema change beyond migration 0009 was +needed or applied to a production database. + +Coverage includes HTTP-to-worker history continuity with one engine build, +fresh credentials, revoked/malformed identity, bounded active scanning, stable +preparation receipts, all terminal-run states without an answer, reopened SQL +answer recovery, staging failure, independent senders/order, exclusive local +ownership, PostgreSQL session-change contracts, approval cancellation, missing/ +oversized responses, execution timeout, shutdown and ownership loss. + +Self-review found and fixed early sender release during approval stream cleanup, +missing failed-run metadata on a stream without a final response, repeated +cancellation interrupting slow cleanup, and a claim returning after shutdown +starting new work. Regression tests failed before each correction and passed +afterwards. Plugin cleanup failures are sanitized; failed supervisors report +503/degraded instead of healthy. No blocking findings remain within this scope. + +Tests used file-backed SQLite and fake engine/identity adapters. PostgreSQL +ownership used a controlled connection double, not a live database; SQLite OS +locks were exercised locally. No Meta, model, MCP, deployed identity, real +number, or production system was contacted. Internal activation still requires +the customer's trusted account-linking/permission implementation. Google/JWT, +provider sending, delivery projection, support/reset/opt-out behavior, media +feedback, and deployment remain subsequent tasks. Next: WA-05 outbound delivery +and controls. diff --git a/tasks/0017-whatsapp-review-and-hardening.md b/tasks/0017-whatsapp-review-and-hardening.md new file mode 100644 index 00000000..797bcb78 --- /dev/null +++ b/tasks/0017-whatsapp-review-and-hardening.md @@ -0,0 +1,82 @@ +# Task 0017 — Review and harden implemented WhatsApp stages + +## Goal / context + +The user requested an explanation of WA-01–WA-04, a code review, and fixes for +gaps in the implemented behavior. Review the existing integration as a whole; +keep outbound delivery and Google/JWT account linking in their planned stages. + +## Read first + +AGENTS.md, .ai/README.md; project-architecture, code-review, +architecture-review, senior-python-engineering, testing, documentation; +tasks 0013–0016, the WhatsApp plan, and ADRs 0006–0009. Apply the installed +Python review, systematic debugging, test-driven development, and verification +skills to findings and their regression tests. + +## Scope / files allowed to change + +- This task and tasks/README.md. +- docs/README.md, docs/WHATSAPP_INTEGRATION_PLAN.md, + docs/adr/0009-channel-worker-ownership-and-recovery.md, + and new docs/WHATSAPP_REVIEW.md (explanation and structured review). +- src/agent_manager/application/channels/worker.py. +- src/agent_manager/domain/channels/models.py. +- tests/agent_manager/test_channel_worker.py, + test_channel_storage.py, and test_whatsapp_normalization.py. + +Fix the cancellation path that releases ownership before active cleanup, and +reject normalized strings that PostgreSQL cannot store before a webhook batch +enters its transaction. Report failed ownership immediately while cleanup is +still running. No public API, plugin, YAML, or schema change is needed. +Preserve all previous work and keep fixes narrow. + +## Requirements / acceptance criteria + +- [x] Review architecture, identity, intake transactions, retries, execution, + response recovery, and lifecycle; document concrete findings and limitations. +- [x] Cancellation of shutdown, including repeated cancellation, cannot release + worker ownership before engine tasks and durable bookkeeping have finished. +- [x] A failed supervisor reports unhealthy immediately, including while slow + engine cancellation cleanup is still running. +- [x] NUL-containing identifiers/text are rejected at the normalized channel + boundary; one invalid message/status does not discard valid webhook siblings. +- [x] Regression tests fail before each fix and pass afterwards; use local SQL, + OS locks, fake engine/identity, and no live Meta/model/MCP/production database. +- [x] Explain implemented stages and explicitly distinguish staged answers from + actual WhatsApp sending and the identity port from deployed account linking. +- [x] make check, git diff --check, and changed documentation links pass. + +## Out of scope + +WA-05 sending, provider reply formatting, product controls, WA-06 pilot setup, +Google/JWT account linking, a review/admin UI, distributed tool replay, live +PostgreSQL/Meta tests, deployment, commits, and unrelated refactoring. + +## Commands / expected report + +Run targeted regression tests, then make check and git diff --check. Record +the review in docs/WHATSAPP_REVIEW.md using the review workflow's eight sections. +The user-facing task report follows AGENTS.md section 9 and explains the current +product flow, fixed findings, remaining work, and verification limits. + +## Implementation result + +Completed on 2026-10-16. The review reproduced and fixed three product gaps: +caller-cancelled shutdown releasing ownership early (P1), NUL-containing input +passing the SQL-compatible channel boundary (P2), and healthy readiness while +a failed supervisor awaited slow cleanup (P2). Twelve new regression cases +failed before their respective corrections and passed afterwards. See +[the explanation and structured review](../docs/WHATSAPP_REVIEW.md). + +The full gate also exposed existing test-owned engine leaks. These were kept +in the separate, bounded task 0018: its connection diagnostic improved from +38 open connections out of 42 to zero, with all 61 existing cases preserved. + +Final `make check` passed: 1,327 tests, 24 baseline warning instances, formatting +for 339 files, lint, mypy for 335 files, and generated-stub checks. No aiosqlite +worker-thread warning remained in that final run. `git diff --check` and local +documentation links/anchors passed. No public contract, schema, engine, provider +integration, deployed identity, or production system was changed. The three +fixes conform to the existing architecture; outbound delivery/account linking +and the live pilot remain subsequent work. diff --git a/tasks/0019-whatsapp-delivery-and-controls.md b/tasks/0019-whatsapp-delivery-and-controls.md new file mode 100644 index 00000000..408be339 --- /dev/null +++ b/tasks/0019-whatsapp-delivery-and-controls.md @@ -0,0 +1,180 @@ +# Task 0019 — WhatsApp outbound delivery and controls (WA-05) + +## Goal / authorization + +Implement the next recommended stage explicitly requested by the user after +the WA-01–WA-04 review: provider sending, ordered durable delivery, current +recipient authorization, retry/uncertain-outcome handling, and text controls. +The internal-information pilot still requires the customer's identity plugin. + +## Read first + +AGENTS.md, .ai/README.md; project-architecture, senior-python-engineering, +architecture-review, testing, sidecar-auth-context, documentation, code-review; +the WhatsApp plan, ADRs 0006–0010, and existing channel implementation/tests. +Use test-driven development and verification-before-completion. Apply Unicode +roundtrip/bounds properties without introducing a new property-test library. + +## Scope / files allowed to change + +- This task, tasks/README.md, docs/README.md, docs/WHATSAPP_INTEGRATION_PLAN.md, + and new docs/adr/0010-whatsapp-outbound-delivery.md. +- pyproject.toml: explicitly declare regex for Unicode grapheme segmentation. +- Under src/agent_manager/: new domain/channels/{delivery,controls}.py, + application/channels/{delivery,controls}.py, + infrastructure/channels/whatsapp/{sender,rendering}.py, + infrastructure/persistence/channels/{delivery,delivery_projection,controls}.py, + infrastructure/persistence/migrations/versions/0010_channel_delivery.py. +- Existing domain/channels/models.py, application/channels/{worker,processing}.py, + infrastructure/channels/whatsapp/normalization.py, + infrastructure/persistence/channels/{tables,intake,repository}.py, + infrastructure/persistence/tables.py, composition.py, config.py. +- New tests/agent_manager/test_whatsapp_rendering.py, test_whatsapp_sender.py, + test_channel_delivery.py, test_channel_delivery_storage.py, + test_channel_controls.py, test_delivery_migration.py, + test_whatsapp_delivery_lifecycle.py. Existing test_whatsapp_normalization.py, + test_channel_worker_lifecycle.py only for compatible setup as needed; + test_webhook_migration.py to seed/read its historical 0008 schema without + using a current adapter that now requires migration 0010. + +Keep modules focused; logical outbox answers remain immutable. New delivery +tables hold provider rendering/attempt state and pinned original ownership. +The existing reset repository also reconciles a concurrently committed reset +receipt if it commits between reading the inbox and sender binding; full +validation exposed this race while extending the same reset behavior. +No engine, YAML, existing HTTP request, or client identity signature change. + +## Acceptance criteria + +- [x] Sender is off by default; enabling requires the existing worker, explicit + API version, separate secret token, support contact, and persistent SQL. +- [x] Plain-text rendering preserves exact Unicode/RTL text and grapheme + clusters; bounded parts prefer paragraph boundaries and never truncate. +- [x] Durable immutable part planning handles existing logical outbox rows; + CAS claims preserve per-recipient part/turn order across restart. +- [x] Before each attempt, enforce original linked account, organization, + current opt-out state and a verified inbound 24-hour window. +- [x] One bounded HTTP client is owned by startup. Acceptance/provider IDs are + persisted separately from delivered/read; callbacks project monotonically, + including callbacks arriving before the acceptance commit. +- [x] Known rejected/transient attempts use bounded backoff/jitter. Ambiguous + submission/cancellation/crash becomes delivery_unknown and is never resent + automatically. Partial/blocked/unknown batches retain recipient ordering. +- [x] Exact /new, /help, /stop, /start and Hebrew aliases execute without a model. + Reset preserves history; stop applies at intake and old pending replies never + reappear after start. Unsupported media gets static text, without downloads. +- [x] Worker ownership/shutdown covers sends and engine jobs; disabled behavior + remains compatible. Secrets, bodies, sender IDs and raw provider errors do not + enter ordinary logs or receipts. +- [x] Tests use local SQL and fake provider/identity/engine only, cover failures, + ordering, authorization, windows, controls, migrations and app lifecycle. + make check, git diff --check and documentation links pass. + +## Out of scope + +Customer Google/JWT account linking, live number/account setup, live sends, +templates/campaigns, media downloads, account/BSUID variants, operator review UI, +automatic replay/reconciliation without a verified provider ID, and deployment. + +## Commands / report + +Write failing tests first, run focused tests, then make check and +git diff --check. Report using AGENTS.md section 9; distinguish local verified +sending behavior from actual account delivery and the remaining WA-06 pilot. + +## Implementation result + +Completed WA-05 on 2026-10-16. Logical answers remain immutable; a separate SQL +planner freezes provider parts and original ownership. The existing connection +supervisor owns the bounded outbound pump and HTTP client. Attempts reauthorize +the original recipient, enforce the service window and suppression, and persist +acceptance separately from authenticated delivered/read observations. Uncertain +submission is never automatically replayed. Static controls bypass the engine. + +Validation: final `make check` passed with **1,420 tests**, including **93 new +cases** compared with the 1,327-test baseline. Formatting passed for 356 files, +lint passed, mypy passed for 352 files, and generated-stub checks passed. +The 27 warnings are the prior 24 warning instances plus three Alembic warnings +from the added migration upgrade/downgrade coverage; no new warning category or +warning suppression was introduced. `git diff --check`, document whitespace, +and 57 local documentation links/anchors passed. + +Tests cover exact Unicode/grapheme roundtrips and bounds; original-recipient +authorization and revocation between parts; immutable planning, recipient/part +order and fairness; retries/budget/window expiry; ambiguous send/cancellation/ +lost acceptance commits; early/out-of-order callbacks and later delivery proof; +stop/start cutoffs, reset concurrency/history, static media/help replies; +real local ownership during send cleanup; reopened storage; historical migration +preservation; and signed HTTP intake through the worker to a fake provider with +one engine build. regex is now an explicit runtime dependency; no property-test +library or archived Meta SDK was added. + +### Self-review / corrections + +- Query filtering now excludes blocked preceding parts before the candidate + limit, so one recipient's multipart retry cannot starve independent recipients. +- An earlier unplanned answer blocks later delivery for that recipient; a + missing internal conversation owner cannot silently become a public reply. +- Both reset paths reconcile a receipt committed between inbox/binding reads; + a delayed duplicate cannot clear a newer conversation. Deterministic + regression tests failed before the fixes and passed afterwards. +- Callback aggregation now resumes remaining parts after verified delivery + supersedes an earlier failed observation, without resending the accepted part. +- System/unknown provider events cannot extend the response window, and invalid + provider bodies fail before I/O. Ownership loss before sending remains fatal + immediately, rather than being swallowed by scheduling-error handling. +- The new HTTP fixture initially selected another example's shared plugin + namespace, causing two existing CLI validation tests to fail in the full run. + It now uses the canonical example consistently with existing HTTP fixtures; + the combined lifecycle/CLI checks and final full gate pass. + +Architecture verdict: conforms; no remaining blocker within this scoped stage. +New domain ports and additive settings/schema are documented by ADR 0010. +Engine, YAML, conversation HTTP and client identity signatures stay compatible. +Identity and data/tool enforcement remain client-owned; no organization identity +or access token is synthesized from WhatsApp message text. + +### Actual files changed in WA-05 + +Created: + +- This task and `docs/adr/0010-whatsapp-outbound-delivery.md`. +- `src/agent_manager/domain/channels/{delivery,controls}.py`. +- `src/agent_manager/application/channels/{delivery,controls}.py`. +- `src/agent_manager/infrastructure/channels/whatsapp/{sender,rendering}.py`. +- `src/agent_manager/infrastructure/persistence/channels/{delivery,delivery_projection,controls}.py`. +- `src/agent_manager/infrastructure/persistence/migrations/versions/0010_channel_delivery.py`. +- `tests/agent_manager/test_whatsapp_rendering.py`, `test_whatsapp_sender.py`, + `test_channel_delivery.py`, `test_channel_delivery_storage.py`, + `test_channel_controls.py`, `test_delivery_migration.py`, + `test_whatsapp_delivery_lifecycle.py`. + +Modified: + +- `tasks/README.md`, `docs/README.md`, `docs/WHATSAPP_INTEGRATION_PLAN.md`, + and `pyproject.toml`. +- `src/agent_manager/{config,composition}.py`. +- `src/agent_manager/domain/channels/models.py` and + `src/agent_manager/application/channels/{processing,worker}.py`. +- `src/agent_manager/infrastructure/channels/whatsapp/normalization.py` and + `src/agent_manager/infrastructure/persistence/channels/{intake,tables,repository}.py`. +- `tests/agent_manager/test_whatsapp_normalization.py` and `test_webhook_migration.py`. + +No files were deleted. Previous uncommitted work was preserved. + +### Limits / next task + +No live Meta, model, MCP, Google login/JWT issuer or production database was +contacted. PostgreSQL remains unverified against a live server. Sender defaults +off; migration 0010 and explicit version/token/support/identity settings are +required before activation. The current account's API contract, error-code +fixtures and eligibility must be verified during preflight; current Meta +developer pages returned HTTP 429. No API version or account compatibility was +invented. Downgrade removes delivery evidence and requires a controlled live +rollback procedure before resuming sends. + +Unknown/failed/blocked work requires operator reconciliation; there is no review +UI, automatic unknown-ID correlation, template/campaign sending, or live number +setup. Google/JWT account linking remains a separate trusted client integration +and is mandatory before an internal-information pilot. Next: WA-06 pilot +packaging/runbook and operational verification. No commit, PR or deployment. diff --git a/tasks/0020-whatsapp-delivery-review.md b/tasks/0020-whatsapp-delivery-review.md new file mode 100644 index 00000000..4b2b9beb --- /dev/null +++ b/tasks/0020-whatsapp-delivery-review.md @@ -0,0 +1,82 @@ +# Task 0020 — Review and harden WhatsApp delivery lifecycle + +## Goal / authorization + +The user explicitly requested a code review and fixes for any gaps after WA-05. +Review delivery authorization, ordering, callbacks, controls, recovery and +shutdown without extending the approved integration scope. + +## Read first + +AGENTS.md, .ai/README.md; project-architecture, code-review, +senior-python-engineering, testing, documentation, architecture-review, +sidecar-auth-context; task 0019, ADRs 0007–0010 and the implementation/tests. +Use systematic debugging, test-driven development and verification before +completion. The existing public contracts and immutable logical answers stay +compatible. + +## Scope / files allowed to change + +- This task and tasks/README.md. +- New docs/WHATSAPP_DELIVERY_REVIEW.md and docs/README.md. +- src/agent_manager/composition.py: channel client cleanup only. +- tests/agent_manager/test_whatsapp_delivery_lifecycle.py: deterministic + regression coverage for cancellation during client resource cleanup. + +The suspected gap is cancellation after the worker has drained but while its +HTTP/identity clients are closing. Reproduce before changing implementation; +preserve the existing worker ownership/drain ordering and sanitized errors. +Record any unrelated findings without silently expanding the change. + +## Acceptance criteria + +- [x] Review architecture, recipient security, delivery state/ordering, exact + controls, retry/recovery and lifecycle; record a structured eight-part review. +- [x] Reproduce any fixed behavior with a failing regression first. +- [x] Cancellation during either channel client close cannot skip cleanup or + finish application shutdown before both clients finish; repeated cancellation + remains safe and cancellation is propagated afterwards. +- [x] Ordinary close errors remain sanitized and do not skip the other client. +- [x] Sender-disabled behavior and existing engine/HTTP/plugin contracts remain + compatible; no request state is added to the engine. +- [x] Focused tests, make check, git diff --check and documentation links pass. + +## Out of scope + +Google/JWT account linking, production activation, provider protocol expansion, +new delivery/review APIs, operator UI, PostgreSQL deployment and live sends. +No schema, YAML, engine, conversation HTTP or identity signature change. + +## Commands / expected report + +Run failing regressions, focused lifecycle tests and make check. Record exact +results and remaining limits in the review; final response follows AGENTS.md +section 9. Preserve all pre-existing uncommitted work. + +## Implementation result + +The review confirmed one P2 lifecycle defect: cancellation while HTTP or +identity close awaited could abandon cleanup, and HTTP cancellation could skip +identity close. Three regression cases failed before the fix. Composition now +owns and shields a cleanup task, joins it through repeated cancellation and +propagates cancellation only afterwards; identity close runs in finally. +Worker drain and ownership behavior, ordinary sanitized close errors, public +contracts and data storage are unchanged. No additional actionable defect was +confirmed in the inspected WA-05 paths. + +Verification: 129 focused tests passed. Final make check passed: format 356 +files, lint, mypy 352 files, **1,423 tests / 27 warnings**, generated-stub check. +Warning instances match the 1,420-test baseline; none were suppressed. The first +full-gate attempt caught SIM117 in the new test, fixed before the final pass. +git diff --check, 50 local documentation links/anchors and whitespace checks +for all six changed files passed. + +Created this task and docs/WHATSAPP_DELIVERY_REVIEW.md. Modified composition.py, +test_whatsapp_delivery_lifecycle.py, docs/README.md and tasks/README.md. No files +were deleted and previous uncommitted work was preserved. The structured review +records architecture/security checks, the fixed finding, evidence and limits. +No new ADR or migration is needed for restoring the existing cleanup guarantee. + +Next: WA-06 pilot packaging and operational preflight; the Google/JWT account +linking prerequisite remains separate. No live provider/model/MCP/production +database was contacted, and there was no commit, PR, activation or deployment. diff --git a/tasks/README.md b/tasks/README.md index aeb30db0..11827899 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -22,5 +22,13 @@ finishing, Expected final report.** | 0010 | [Docker deployment](0010-docker-deployment.md) | 0009 | | 0011 | [Observability & tracing](0011-observability-and-tracing.md) | 0004 | | 0012 | [Tests & quality gates](0012-tests-and-quality-gates.md) | all | +| 0013 | [Idempotent conversation turns](0013-idempotent-conversation-turns.md) | conversation lifecycle; WhatsApp WA-01 | +| 0014 | [Channel storage and identity](0014-channel-storage-and-identity.md) | 0013; WhatsApp WA-02 | +| 0015 | [Authenticated WhatsApp webhook](0015-whatsapp-webhook.md) | 0014; WhatsApp WA-03 | +| 0016 | [Channel turn worker and recovery](0016-channel-turn-worker.md) | 0015; WhatsApp WA-04 | +| 0017 | [WhatsApp review and hardening](0017-whatsapp-review-and-hardening.md) | 0013–0016 | +| 0018 | [SQL test resource cleanup](0018-sql-test-resource-cleanup.md) | 0017 review verification | +| 0019 | [WhatsApp outbound delivery and controls](0019-whatsapp-delivery-and-controls.md) | 0013–0018; WhatsApp WA-05 | +| 0020 | [WhatsApp delivery review](0020-whatsapp-delivery-review.md) | 0019 | Before starting any task, read `AGENTS.md` and the skill(s) the task lists. From b13f7eee56f94448c61788f3d0ea4114658336c0 Mon Sep 17 00:00:00 2001 From: Asaf Varon Date: Fri, 16 Oct 2026 15:47:45 +0300 Subject: [PATCH 6/7] feat(manager): log channel status and webhook rejections The worker logs one `channel status:` line per configured interval with queue counts, the oldest unfinished and unsent ages, and unknown, failed, and blocked reply counts. The read is bounded to five seconds so monitoring cannot stall scheduling or a graceful drain. Every rejected webhook request logs a fixed reason. No line contains message text, sender IDs, tokens, or provider IDs. Adds EXTRA_WHATSAPP_STATUS_LOG_SECONDS (default 60). Co-Authored-By: Claude Opus 5.5 --- src/agent_manager/api/routes/whatsapp.py | 26 +- .../application/channels/worker.py | 45 ++++ .../application/channels/worker_options.py | 2 + src/agent_manager/composition.py | 12 +- src/agent_manager/config.py | 1 + src/agent_manager/domain/channels/status.py | 29 +++ .../persistence/channels/status.py | 104 ++++++++ tests/agent_manager/test_channel_status.py | 230 ++++++++++++++++++ tests/agent_manager/test_whatsapp_webhook.py | 36 +++ 9 files changed, 474 insertions(+), 11 deletions(-) create mode 100644 src/agent_manager/domain/channels/status.py create mode 100644 src/agent_manager/infrastructure/persistence/channels/status.py create mode 100644 tests/agent_manager/test_channel_status.py diff --git a/src/agent_manager/api/routes/whatsapp.py b/src/agent_manager/api/routes/whatsapp.py index 32d21aa2..978c2e61 100644 --- a/src/agent_manager/api/routes/whatsapp.py +++ b/src/agent_manager/api/routes/whatsapp.py @@ -18,6 +18,12 @@ router = APIRouter(prefix="/channels/whatsapp", tags=["whatsapp"]) +def _rejected(status_code: int, detail: str, reason: str) -> HTTPException: + """Log a fixed reason only; request content never reaches ordinary logs.""" + logger.warning("whatsapp webhook rejected: reason=%s", reason) + return HTTPException(status_code=status_code, detail=detail) + + def _webhook(request: Request) -> WhatsAppWebhook: webhook = getattr(request.app.state, "whatsapp_webhook", None) if webhook is None: @@ -30,7 +36,7 @@ async def verify_subscription(request: Request) -> PlainTextResponse: webhook = _webhook(request) names = ("hub.mode", "hub.verify_token", "hub.challenge") if any(len(request.query_params.getlist(name)) != 1 for name in names): - raise HTTPException(status_code=403, detail="invalid webhook subscription") + raise _rejected(403, "invalid webhook subscription", "subscription") try: challenge = subscription_challenge( request.query_params.get("hub.mode"), @@ -39,7 +45,7 @@ async def verify_subscription(request: Request) -> PlainTextResponse: expected=webhook.config.verify_token.get_secret_value(), ) except WebhookAuthenticationError: - raise HTTPException(status_code=403, detail="invalid webhook subscription") from None + raise _rejected(403, "invalid webhook subscription", "subscription") from None return PlainTextResponse(challenge) @@ -48,16 +54,16 @@ async def receive_notification(request: Request) -> dict[str, str]: webhook = _webhook(request) signatures = request.headers.getlist("X-Hub-Signature-256") if len(signatures) != 1: - raise HTTPException(status_code=401, detail="invalid webhook signature") + raise _rejected(401, "invalid webhook signature", "signature") if request.headers.get("content-encoding", "identity") != "identity": - raise HTTPException(status_code=415, detail="encoded webhook bodies are unsupported") + raise _rejected(415, "encoded webhook bodies are unsupported", "encoding") if request.headers.get("content-type", "").split(";", 1)[0].lower() != "application/json": - raise HTTPException(status_code=415, detail="webhook requires application/json") + raise _rejected(415, "webhook requires application/json", "content_type") body = await _bounded_body(request, webhook.config.max_body_bytes) try: await webhook.receive(body, signatures[0], datetime.now(UTC)) except WebhookAuthenticationError: - raise HTTPException(status_code=401, detail="invalid webhook signature") from None + raise _rejected(401, "invalid webhook signature", "signature") from None except Exception as exc: # The failure boundary must never reflect payloads, identifiers, or DB # exception text into provider responses or ordinary logs. @@ -73,15 +79,15 @@ async def _bounded_body(request: Request, limit: int) -> bytes: if len(lengths) > 1 or ( lengths and (len(lengths[0]) > 20 or not lengths[0].isascii() or not lengths[0].isdecimal()) ): - raise HTTPException(status_code=400, detail="invalid content length") + raise _rejected(400, "invalid content length", "length") if lengths and int(lengths[0]) > limit: - raise HTTPException(status_code=413, detail="webhook body too large") + raise _rejected(413, "webhook body too large", "size") body = bytearray() try: async for chunk in request.stream(): if len(body) + len(chunk) > limit: - raise HTTPException(status_code=413, detail="webhook body too large") + raise _rejected(413, "webhook body too large", "size") body.extend(chunk) except ClientDisconnect: - raise HTTPException(status_code=400, detail="incomplete webhook body") from None + raise _rejected(400, "incomplete webhook body", "incomplete") from None return bytes(body) diff --git a/src/agent_manager/application/channels/worker.py b/src/agent_manager/application/channels/worker.py index 002f2d60..3bf8fb8f 100644 --- a/src/agent_manager/application/channels/worker.py +++ b/src/agent_manager/application/channels/worker.py @@ -6,6 +6,7 @@ import logging from collections import deque from contextlib import AsyncExitStack +from datetime import UTC, datetime from types import TracebackType from agent_manager.application.channels.delivery import ChannelDeliveryService @@ -17,8 +18,10 @@ ChannelWorkerOwnershipLost, ) from agent_manager.domain.channels.repository import ChannelRepository +from agent_manager.domain.channels.status import ChannelStatusReader logger = logging.getLogger(__name__) +STATUS_READ_TIMEOUT_SECONDS = 5.0 class ChannelWorker: @@ -30,12 +33,15 @@ def __init__( options: ChannelWorkerOptions, *, delivery: ChannelDeliveryService | None = None, + status: ChannelStatusReader | None = None, ) -> None: self._repository = repository self._processor = processor self._ownership = ownership self._options = options self._delivery = delivery + self._status = status + self._status_due = 0.0 self._stack = AsyncExitStack() self._stop = asyncio.Event() self._jobs: dict[str, asyncio.Task[None]] = {} @@ -110,6 +116,8 @@ async def _serve(self) -> None: raise except Exception as exc: logger.warning("channel scheduling failed: type=%s", type(exc).__name__) + if self._status is not None and not self._stop.is_set(): + await self._log_status(self._status) stop_waiter = asyncio.create_task(self._stop.wait()) try: await asyncio.wait( @@ -163,6 +171,39 @@ async def _schedule(self) -> None: if item.event_id not in self._jobs: self._jobs[item.event_id] = asyncio.create_task(self._processor.process(item)) + async def _log_status(self, reader: ChannelStatusReader) -> None: + due = asyncio.get_running_loop().time() + if due < self._status_due: + return + self._status_due = due + self._options.status_seconds + try: + # Monitoring must never stall scheduling or a graceful drain. + async with asyncio.timeout(STATUS_READ_TIMEOUT_SECONDS): + status = await reader.read( + self._options.connection_id, self._options.system_id, datetime.now(UTC) + ) + except Exception as exc: + logger.warning("channel status unavailable: type=%s", type(exc).__name__) + return + logger.info( + "channel status: connection=%s inbox_queued=%d inbox_running=%d " + "inbox_answered=%d inbox_oldest_s=%s needs_review=%d inbox_failed=%d " + "outbox_pending=%d outbox_oldest_s=%s delivery_unknown=%d " + "delivery_failed=%d delivery_blocked=%d", + self._options.connection_id, + status.inbox_queued, + status.inbox_running, + status.inbox_answered, + _seconds(status.inbox_oldest_seconds), + status.inbox_needs_review, + status.inbox_failed, + status.outbox_pending, + _seconds(status.outbox_oldest_seconds), + status.delivery_unknown, + status.delivery_failed, + status.delivery_blocked, + ) + def _reap(self) -> None: for event_id, task in tuple(self._jobs.items()): if task.done(): @@ -189,3 +230,7 @@ async def _drain(self, *, immediate: bool) -> None: def _observe_failure(task: asyncio.Task[None]) -> None: if not task.cancelled() and task.exception() is not None: logger.error("channel supervisor stopped: type=%s", type(task.exception()).__name__) + + +def _seconds(value: float | None) -> str: + return "none" if value is None else str(int(value)) diff --git a/src/agent_manager/application/channels/worker_options.py b/src/agent_manager/application/channels/worker_options.py index fafd3f26..79ac7ef7 100644 --- a/src/agent_manager/application/channels/worker_options.py +++ b/src/agent_manager/application/channels/worker_options.py @@ -14,6 +14,7 @@ class ChannelWorkerOptions: poll_seconds: float = 0.5 turn_seconds: float = 120 drain_seconds: float = 15 + status_seconds: float = 60 def __post_init__(self) -> None: bounded_identifier(self.connection_id, limit=128) @@ -25,5 +26,6 @@ def __post_init__(self) -> None: 0 < self.poll_seconds <= 60 and 0 < self.turn_seconds <= 3600 and 0 < self.drain_seconds <= 60 + and 0 < self.status_seconds <= 3600 ): raise ValueError("channel worker time bounds must be finite and positive") diff --git a/src/agent_manager/composition.py b/src/agent_manager/composition.py index 4ffda5d2..609fbd36 100644 --- a/src/agent_manager/composition.py +++ b/src/agent_manager/composition.py @@ -32,6 +32,7 @@ from agent_manager.domain.channels.delivery import DeliveryRepository from agent_manager.domain.channels.identity import ChannelIdentityResolver from agent_manager.domain.channels.repository import ChannelRepository +from agent_manager.domain.channels.status import ChannelStatusReader from agent_manager.domain.channels.webhooks import WebhookRepository from agent_manager.infrastructure.auth.anonymous_identity_source import AnonymousIdentitySource from agent_manager.infrastructure.auth.claim_mapping import ClaimMapping @@ -50,6 +51,7 @@ from agent_manager.infrastructure.persistence.channels.delivery import SqlDeliveryRepository from agent_manager.infrastructure.persistence.channels.ownership import SqlChannelWorkerOwnership from agent_manager.infrastructure.persistence.channels.repository import SqlChannelRepository +from agent_manager.infrastructure.persistence.channels.status import SqlChannelStatusReader from agent_manager.infrastructure.persistence.channels.webhooks import SqlWebhookRepository from agent_manager.infrastructure.persistence.database import create_db_engine, session_factory from agent_manager.infrastructure.persistence.memory_repository import MemoryRepository @@ -72,6 +74,7 @@ class ApplicationRepositories: webhooks: WebhookRepository | None = None deliveries: DeliveryRepository | None = None controls: ControlRepository | None = None + status: ChannelStatusReader | None = None def build_whatsapp_webhook( @@ -123,6 +126,7 @@ async def channel_worker_lifespan( poll_seconds=settings.extra_whatsapp_worker_poll_seconds, turn_seconds=settings.extra_whatsapp_worker_turn_seconds, drain_seconds=settings.extra_whatsapp_worker_drain_seconds, + status_seconds=settings.extra_whatsapp_status_log_seconds, ) identities = load_channel_identity_resolver(settings.extra_whatsapp_identity_resolver) sender: WhatsAppSender | None = None @@ -169,7 +173,12 @@ async def channel_worker_lifespan( settings.effective_database_url, options.connection_id ) async with ChannelWorker( - repositories.channels, processor, ownership, options, delivery=delivery + repositories.channels, + processor, + ownership, + options, + delivery=delivery, + status=repositories.status, ) as worker: yield worker finally: @@ -305,6 +314,7 @@ async def application_repositories( webhooks=SqlWebhookRepository(sessions), deliveries=SqlDeliveryRepository(sessions), controls=SqlControlRepository(sessions), + status=SqlChannelStatusReader(sessions), ) finally: await db_engine.dispose() diff --git a/src/agent_manager/config.py b/src/agent_manager/config.py index 98ccb451..e7592ccb 100644 --- a/src/agent_manager/config.py +++ b/src/agent_manager/config.py @@ -176,6 +176,7 @@ class Settings(BaseSettings): default=120, gt=0, le=3600, allow_inf_nan=False ) extra_whatsapp_worker_drain_seconds: float = Field(default=15, gt=0, le=60, allow_inf_nan=False) + extra_whatsapp_status_log_seconds: float = Field(default=60, gt=0, le=3600, allow_inf_nan=False) extra_whatsapp_sender_enabled: bool = False extra_whatsapp_api_version: str | None = None extra_whatsapp_access_token: SecretStr | None = Field(default=None, repr=False, exclude=True) diff --git a/src/agent_manager/domain/channels/status.py b/src/agent_manager/domain/channels/status.py new file mode 100644 index 00000000..4528e984 --- /dev/null +++ b/src/agent_manager/domain/channels/status.py @@ -0,0 +1,29 @@ +"""Operational counts for one connection, without message content or identifiers.""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from datetime import datetime + + +@dataclass(frozen=True) +class ChannelStatus: + inbox_queued: int + inbox_running: int + inbox_answered: int + inbox_needs_review: int + inbox_failed: int + # Seconds since the oldest unfinished inbound message was received. + inbox_oldest_seconds: float | None + # Unsent reply parts; one reply can be split into several parts. + outbox_pending: int + # Seconds since the inbound message behind the oldest unsent part arrived. + outbox_oldest_seconds: float | None + # Replies, not parts, whose delivery ended in each outcome. + delivery_unknown: int + delivery_failed: int + delivery_blocked: int + + +class ChannelStatusReader(ABC): + @abstractmethod + async def read(self, connection_id: str, system_id: str, now: datetime) -> ChannelStatus: ... diff --git a/src/agent_manager/infrastructure/persistence/channels/status.py b/src/agent_manager/infrastructure/persistence/channels/status.py new file mode 100644 index 00000000..3b7ebfff --- /dev/null +++ b/src/agent_manager/infrastructure/persistence/channels/status.py @@ -0,0 +1,104 @@ +"""Grouped counts over channel queues; never selects text or sender identifiers.""" + +from collections.abc import Iterable +from datetime import datetime + +from sqlalchemy import func +from sqlalchemy.ext.asyncio import async_sessionmaker +from sqlmodel import col, select +from sqlmodel.ext.asyncio.session import AsyncSession + +from agent_manager.domain.channels.models import InboxState +from agent_manager.domain.channels.status import ChannelStatus, ChannelStatusReader +from agent_manager.domain.pagination import ensure_utc +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelDeliveryBatchRow as Batch, +) +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelDeliveryPartRow as Part, +) +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelInboxRow as Inbox, +) + +_RUNNING = (InboxState.PREPARING, InboxState.PREPARED, InboxState.RUNNING) +_UNFINISHED = (InboxState.QUEUED, *_RUNNING, InboxState.ANSWERED) +_UNSENT = ("ready", "retry", "sending") + + +class SqlChannelStatusReader(ChannelStatusReader): + def __init__(self, sessions: async_sessionmaker[AsyncSession]) -> None: + self._sessions = sessions + + async def read(self, connection_id: str, system_id: str, now: datetime) -> ChannelStatus: + async with self._sessions() as session: + inbox = ( + await session.exec( + select( + Inbox.state, + func.count(), + func.min(Inbox.received_at), + ) + .where( + col(Inbox.connection_id) == connection_id, + col(Inbox.system_id) == system_id, + ) + .group_by(col(Inbox.state)) + ) + ).all() + parts = ( + await session.exec( + select( + Part.state, + func.count(), + func.min(Inbox.received_at), + ) + .join( + Batch, + col(Batch.event_id) == col(Part.event_id), + ) + .join( + Inbox, + col(Inbox.event_id) == col(Batch.event_id), + ) + .where( + col(Batch.connection_id) == connection_id, + col(Batch.system_id) == system_id, + ) + .group_by(col(Part.state)) + ) + ).all() + # Outcomes count replies: a reply mirrors its part's failure state. + replies = ( + await session.exec( + select(Batch.state, func.count()) + .where( + col(Batch.connection_id) == connection_id, + col(Batch.system_id) == system_id, + ) + .group_by(col(Batch.state)) + ) + ).all() + inbox_counts = {state: count for state, count, _ in inbox} + part_counts = {state: count for state, count, _ in parts} + reply_counts = dict(replies) + return ChannelStatus( + inbox_queued=inbox_counts.get(InboxState.QUEUED, 0), + inbox_running=sum(inbox_counts.get(state, 0) for state in _RUNNING), + inbox_answered=inbox_counts.get(InboxState.ANSWERED, 0), + inbox_needs_review=inbox_counts.get(InboxState.NEEDS_REVIEW, 0), + inbox_failed=inbox_counts.get(InboxState.FAILED, 0), + inbox_oldest_seconds=_age(now, (o for s, _, o in inbox if s in _UNFINISHED)), + outbox_pending=sum(part_counts.get(state, 0) for state in _UNSENT), + outbox_oldest_seconds=_age(now, (o for s, _, o in parts if s in _UNSENT)), + delivery_unknown=reply_counts.get("delivery_unknown", 0), + delivery_failed=reply_counts.get("failed", 0), + delivery_blocked=sum( + count for state, count in reply_counts.items() if state.startswith("blocked_") + ), + ) + + +def _age(now: datetime, oldest: Iterable[datetime | None]) -> float | None: + times = [value for value in map(ensure_utc, oldest) if value is not None] + return max(0.0, (now - min(times)).total_seconds()) if times else None diff --git a/tests/agent_manager/test_channel_status.py b/tests/agent_manager/test_channel_status.py new file mode 100644 index 00000000..d3703e37 --- /dev/null +++ b/tests/agent_manager/test_channel_status.py @@ -0,0 +1,230 @@ +"""Operational status: bounded counts and ages, logged without message identifiers.""" + +import asyncio +import logging +import time +from collections.abc import AsyncIterator +from dataclasses import replace +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any, cast + +import pytest +from pydantic import ValidationError +from sqlmodel import SQLModel + +from agent_manager.application.channels.processing import ChannelTurnProcessor +from agent_manager.application.channels.worker import ChannelWorker +from agent_manager.config import Settings +from agent_manager.domain.channels.repository import ChannelRepository +from agent_manager.domain.channels.status import ChannelStatus, ChannelStatusReader +from agent_manager.infrastructure.persistence.channels.status import SqlChannelStatusReader +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelDeliveryBatchRow, + ChannelDeliveryPartRow, + ChannelInboxRow, +) +from agent_manager.infrastructure.persistence.database import create_db_engine, session_factory +from tests.agent_manager.test_channel_worker import NOW, OPTIONS, Ownership + +SENDER = "+15550001111" + + +@pytest.fixture +async def reader(tmp_path: Path) -> AsyncIterator[tuple[SqlChannelStatusReader, Any]]: + database = create_db_engine(f"sqlite+aiosqlite:///{tmp_path / 'status.db'}") + async with database.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + sessions = session_factory(database) + try: + yield SqlChannelStatusReader(sessions), sessions + finally: + await database.dispose() + + +def inbox(event_id: str, state: str, age: int, *, connection: str = "c") -> ChannelInboxRow: + return ChannelInboxRow( + event_id=event_id, + connection_id=connection, + system_id="s", + sender_id=SENDER, + provider_message_id=f"wamid.{event_id}", + kind="text", + content="private message text", + received_at=NOW - timedelta(seconds=age), + state=state, + ) + + +def delivery( + event_id: str, state: str, sequence: int, *, parts: int = 1, connection: str = "c" +) -> list[ChannelDeliveryBatchRow | ChannelDeliveryPartRow]: + return [ + ChannelDeliveryBatchRow( + event_id=event_id, + connection_id=connection, + system_id="s", + sender_id=SENDER, + sequence=sequence, + state=state, + ), + *( + ChannelDeliveryPartRow( + delivery_id=f"d-{event_id}-{index}", + event_id=event_id, + connection_id=connection, + part_index=index, + text="private reply text", + state=state, + ) + for index in range(parts) + ), + ] + + +async def test_reader_counts_attention_states_and_ages_for_one_connection(reader): + status_reader, sessions = reader + async with sessions() as session, session.begin(): + session.add_all( + [ + inbox("queued", "queued", 100), + inbox("preparing", "preparing", 50), + inbox("answered", "answered", 30), + inbox("unknown", "done", 1000), + inbox("failed-send", "done", 900), + inbox("blocked", "done", 800), + inbox("review", "needs_review", 700), + inbox("failed", "failed", 600), + inbox("other", "queued", 5000, connection="other"), + ] + ) + async with sessions() as session, session.begin(): + session.add_all( + [ + # Pending work counts parts; outcomes count replies. + *delivery("answered", "ready", 3, parts=2), + *delivery("unknown", "delivery_unknown", 4), + *delivery("failed-send", "failed", 5, parts=2), + *delivery("blocked", "blocked_window", 6), + *delivery("other", "ready", 1, connection="other"), + ] + ) + + status = await status_reader.read("c", "s", NOW) + + assert status == ChannelStatus( + inbox_queued=1, + inbox_running=1, + inbox_answered=1, + inbox_needs_review=1, + inbox_failed=1, + inbox_oldest_seconds=100.0, + outbox_pending=2, + outbox_oldest_seconds=30.0, + delivery_unknown=1, + delivery_failed=1, + delivery_blocked=1, + ) + + +async def test_empty_connection_reports_zero_counts_and_no_ages(reader): + status_reader, _ = reader + status = await status_reader.read("c", "s", NOW) + assert status == ChannelStatus(0, 0, 0, 0, 0, None, 0, None, 0, 0, 0) + + +class IdleRepository: + async def list_active(self, *args: Any, **kwargs: Any) -> list[Any]: + return [] + + async def claim_next(self, *args: Any, **kwargs: Any) -> None: + return None + + +class FixedReader(ChannelStatusReader): + def __init__(self, status: ChannelStatus | Exception) -> None: + self.status = status + self.reads = 0 + + async def read(self, connection_id: str, system_id: str, now: datetime) -> ChannelStatus: + self.reads += 1 + if isinstance(self.status, Exception): + raise self.status + return self.status + + +async def run_worker(reader: ChannelStatusReader, *, status_seconds: float) -> bool: + worker = ChannelWorker( + cast(ChannelRepository, IdleRepository()), + cast(ChannelTurnProcessor, None), + Ownership(), + replace(OPTIONS, status_seconds=status_seconds), + status=reader, + ) + async with worker: + await asyncio.sleep(0.15) + return worker.running + + +STATUS = ChannelStatus(2, 1, 0, 1, 0, 42.9, 3, None, 1, 0, 2) + + +async def test_worker_logs_one_bounded_status_line_per_interval(caplog): + caplog.set_level(logging.INFO, logger="agent_manager.application.channels.worker") + reader = FixedReader(STATUS) + + assert await run_worker(reader, status_seconds=60) + + lines = [r.getMessage() for r in caplog.records if r.getMessage().startswith("channel status:")] + assert lines == [ + "channel status: connection=connection inbox_queued=2 inbox_running=1 " + "inbox_answered=0 inbox_oldest_s=42 needs_review=1 inbox_failed=0 " + "outbox_pending=3 outbox_oldest_s=none delivery_unknown=1 " + "delivery_failed=0 delivery_blocked=2" + ] + assert reader.reads == 1 + + +async def test_short_interval_logs_repeatedly(caplog): + caplog.set_level(logging.INFO, logger="agent_manager.application.channels.worker") + reader = FixedReader(STATUS) + assert await run_worker(reader, status_seconds=0.02) + assert reader.reads >= 2 + + +async def test_status_failure_is_logged_by_type_and_never_stops_the_worker(caplog): + caplog.set_level(logging.INFO, logger="agent_manager.application.channels.worker") + reader = FixedReader(RuntimeError(f"database said {SENDER}")) + + assert await run_worker(reader, status_seconds=60) + + assert "channel status unavailable: type=RuntimeError" in caplog.text + assert SENDER not in caplog.text + + +class HangingReader(ChannelStatusReader): + async def read(self, connection_id: str, system_id: str, now: datetime) -> ChannelStatus: + await asyncio.Event().wait() + raise AssertionError("unreachable") + + +async def test_hanging_status_read_neither_stalls_the_worker_nor_its_shutdown(caplog, monkeypatch): + caplog.set_level(logging.INFO, logger="agent_manager.application.channels.worker") + monkeypatch.setattr( + "agent_manager.application.channels.worker.STATUS_READ_TIMEOUT_SECONDS", 0.05 + ) + started = time.monotonic() + + assert await run_worker(HangingReader(), status_seconds=60) + + # Without a bound, shutdown waits drain_seconds + 1 and then cancels. + assert time.monotonic() - started < OPTIONS.drain_seconds + 1 + assert "channel status unavailable: type=TimeoutError" in caplog.text + + +@pytest.mark.parametrize("seconds", [0, -1, 3601, float("inf")]) +def test_status_interval_is_bounded(seconds): + with pytest.raises(ValueError): + replace(OPTIONS, status_seconds=seconds) + with pytest.raises(ValidationError): + Settings.from_values(extra_whatsapp_status_log_seconds=seconds) diff --git a/tests/agent_manager/test_whatsapp_webhook.py b/tests/agent_manager/test_whatsapp_webhook.py index 79816430..3a112b4a 100644 --- a/tests/agent_manager/test_whatsapp_webhook.py +++ b/tests/agent_manager/test_whatsapp_webhook.py @@ -6,6 +6,7 @@ import hashlib import hmac import json +import logging from collections.abc import Iterator from datetime import UTC, datetime from pathlib import Path @@ -381,3 +382,38 @@ def test_duplicate_subscription_parameters_are_rejected(channel): ("hub.challenge", "two"), ] assert client.get(ENDPOINT, params=params).status_code == 403 + + +@pytest.mark.parametrize( + ("request_kind", "reason"), + [ + ("bad_signature", "signature"), + ("missing_signature", "signature"), + ("content_type", "content_type"), + ("encoding", "encoding"), + ("subscription", "subscription"), + ], +) +def test_rejections_log_a_fixed_reason_without_request_content( + channel, caplog, request_kind, reason +): + client, _, _ = channel + caplog.set_level(logging.WARNING, logger="agent_manager.api.routes.whatsapp") + body = encoded(payload(change(message(text={"body": "private question"})))) + forged = "sha256=" + "0" * 64 + if request_kind == "subscription": + params = {"hub.mode": "subscribe", "hub.verify_token": "wrong-token", "hub.challenge": "c"} + client.get(ENDPOINT, params=params) + else: + headers = { + "bad_signature": {"Content-Type": "application/json", "X-Hub-Signature-256": forged}, + "missing_signature": {"Content-Type": "application/json"}, + "content_type": {**signed(body), "Content-Type": "text/plain"}, + "encoding": {**signed(body), "Content-Encoding": "gzip"}, + }[request_kind] + client.post(ENDPOINT, content=body, headers=headers) + + rejected = [r.getMessage() for r in caplog.records if "webhook rejected" in r.getMessage()] + assert rejected == [f"whatsapp webhook rejected: reason={reason}"] + for private in ("private question", forged, "wrong-token", APP_SECRET): + assert private not in caplog.text From 675c8741abb19dea4c5e6cbf67f7a6fa95d34b51 Mon Sep 17 00:00:00 2001 From: Asaf Varon Date: Fri, 16 Oct 2026 15:47:51 +0300 Subject: [PATCH 7/7] feat(examples): add WhatsApp pilot example, runbook, and rehearsal The runbook covers preflight, deployment, staged enablement and pausing, health and status logs, restart recovery, rollback, retention, and the real-number smoke-test checklist. The example runs agent-manager on durable storage with a tool-free assistant and an allowlist identity resolver; its port is bound to loopback because the web conversation API is anonymous by default. A fake-provider rehearsal proves that paused intake survives restarts, is answered and sent exactly once, projects a delivered callback, and is not resent after a further restart. Co-Authored-By: Claude Opus 5.5 --- docs/README.md | 10 +- docs/WHATSAPP_INTEGRATION_PLAN.md | 9 +- docs/WHATSAPP_RUNBOOK.md | 239 ++++++++++++++++++ examples/whatsapp-pilot/.env.example | 34 +++ examples/whatsapp-pilot/.gitignore | 1 + examples/whatsapp-pilot/README.md | 42 +++ examples/whatsapp-pilot/agents.yml | 23 ++ examples/whatsapp-pilot/docker-compose.yml | 28 ++ examples/whatsapp-pilot/prompts/assistant.md | 8 + examples/whatsapp-pilot/senders.example.json | 7 + examples/whatsapp-pilot/whatsapp_identity.py | 52 ++++ tasks/0021-whatsapp-pilot-packaging.md | 105 ++++++++ tasks/README.md | 1 + .../test_whatsapp_pilot_rehearsal.py | 192 ++++++++++++++ 14 files changed, 744 insertions(+), 7 deletions(-) create mode 100644 docs/WHATSAPP_RUNBOOK.md create mode 100644 examples/whatsapp-pilot/.env.example create mode 100644 examples/whatsapp-pilot/.gitignore create mode 100644 examples/whatsapp-pilot/README.md create mode 100644 examples/whatsapp-pilot/agents.yml create mode 100644 examples/whatsapp-pilot/docker-compose.yml create mode 100644 examples/whatsapp-pilot/prompts/assistant.md create mode 100644 examples/whatsapp-pilot/senders.example.json create mode 100644 examples/whatsapp-pilot/whatsapp_identity.py create mode 100644 tasks/0021-whatsapp-pilot-packaging.md create mode 100644 tests/agent_manager/test_whatsapp_pilot_rehearsal.py diff --git a/docs/README.md b/docs/README.md index b2ba5f59..39bd9a9b 100644 --- a/docs/README.md +++ b/docs/README.md @@ -37,9 +37,13 @@ implemented, so protected-node access control is not actually enforced today. channel for one organization-owned number, with durable delivery, conversation reuse, identity boundaries, and scoped implementation tasks. Generic turn preparation, SQL channel storage, authenticated webhook intake, an - opt-in turn worker with conservative recovery, and opt-in provider delivery - with text controls are implemented. Customer account linking and live pilot - packaging remain planned. + opt-in turn worker with conservative recovery, opt-in provider delivery + with text controls, and pilot packaging are implemented. Customer account + linking, data retention, and the real-number smoke test remain planned. +- [WhatsApp pilot runbook](WHATSAPP_RUNBOOK.md) — preflight, deployment, + staged enablement and pausing, health and status logs, restart recovery, + rollback, retention, and the smoke-test checklist + ([task 0021](../tasks/0021-whatsapp-pilot-packaging.md)). [ADR 0006](adr/0006-idempotent-conversation-turn-preparation.md) records the implemented provider-independent preparation prerequisite for channels. diff --git a/docs/WHATSAPP_INTEGRATION_PLAN.md b/docs/WHATSAPP_INTEGRATION_PLAN.md index 57f2ec78..a0810212 100644 --- a/docs/WHATSAPP_INTEGRATION_PLAN.md +++ b/docs/WHATSAPP_INTEGRATION_PLAN.md @@ -2,9 +2,10 @@ - **Status:** WA-01 preparation, WA-02 durable channel storage/identity boundary, WA-03 authenticated webhook intake, WA-04 turn worker/recovery, - and WA-05 outbound delivery/controls are implemented. Organization account - linking and live pilot setup remain pending. Intake, worker and sender are - separately disabled by default. + WA-05 outbound delivery/controls, and WA-06 pilot packaging (runbook, + example, status logs, fake-provider rehearsal) are implemented. Organization + account linking, data retention, and the authorized real-number smoke test + remain pending. Intake, worker and sender are separately disabled by default. - **Date:** 2026-10-15. - **Updated:** 2026-10-16. - **Confirmed scope:** One organization-owned WhatsApp Business number. @@ -457,7 +458,7 @@ ships with tests, and each implementation task must pass `make check`. | WA-03: Cloud API intake boundary — implemented | Task 0015; `api/routes/whatsapp.py`, `infrastructure/channels/whatsapp/`, settings/composition, domain/SQL callback port, migration 0009, and tests. | Challenge and byte-level signature verification, account checks, complete batch parsing, separate status observations, safe quarantine, atomic commit-before-ack, and no agent work. Disabled by default. | | WA-04: Turn worker and recovery — implemented | Task 0016; channel processor/supervisor, active-work scan, authorized service response lookup, client identity loader, ownership adapter, settings/lifespan, and tests. | Per-sender order and independent concurrency; fresh identity; one execution claim; saved-answer recovery; ambiguous work held for review; approvals cancelled; streams awaited before releasing ownership; engine reused. | | WA-05: Outbound delivery and controls — implemented | Task 0019; sender/rendering adapters, delivery/control ports and services, SQL plans/projection, migration 0010, settings/supervisor wiring and tests. | Ordered Unicode/grapheme-safe splitting; accepted/delivered distinction; bounded retry; unknown-delivery review; fresh original-owner/window/opt-out checks; unsupported media/reset/help/stop/start without model calls. Disabled by default, verified locally with a fake provider. | -| WA-06: Pilot packaging and operation | New WhatsApp example/runbook, docs index, deployment command/configuration and fake-provider end-to-end tests. | Documented administrator setup and rollback; durable restart rehearsal; metrics/redaction; full fake-provider round trip; then an explicitly authorized real-number smoke test. | +| WA-06: Pilot packaging and operation — implemented except the smoke test | Task 0021; [runbook](WHATSAPP_RUNBOOK.md), `examples/whatsapp-pilot/`, `channel status:` and webhook-rejection log lines, and a fake-provider restart rehearsal. | Documented administrator setup and rollback; durable restart rehearsal; metrics/redaction; full fake-provider round trip; then an explicitly authorized real-number smoke test. | WA-00 → WA-01 → WA-02 → WA-03 → WA-04 → WA-05 → WA-06 is the delivery order. The identity/account-linking prerequisite is required for the confirmed internal diff --git a/docs/WHATSAPP_RUNBOOK.md b/docs/WHATSAPP_RUNBOOK.md new file mode 100644 index 00000000..683d7155 --- /dev/null +++ b/docs/WHATSAPP_RUNBOOK.md @@ -0,0 +1,239 @@ +# WhatsApp pilot runbook + +How an administrator deploys, operates, pauses, and rolls back the WhatsApp +channel for one organization-owned number. It describes the implemented +behavior from the [integration plan](WHATSAPP_INTEGRATION_PLAN.md) (WA-01 to +WA-06). The deployable example is in +[`examples/whatsapp-pilot/`](../examples/whatsapp-pilot/README.md). + +No real account or number has been activated yet. The real-number smoke test +(§10) needs explicit authorization from the organization. + +## 1. Preflight + +Confirm all of these before enabling anything: + +- [ ] The organization owns the number. If it already uses the WhatsApp Business + app, check current coexistence or migration eligibility in Meta's + documentation first. This channel does not import history. +- [ ] Business account ID and phone number ID for the Cloud API. +- [ ] Meta app secret, used to verify webhook signatures. +- [ ] A long-lived system-user access token with messaging permission. It must + differ from the app secret and the verify token. +- [ ] A Graph API version the account supports, pinned explicitly (`vNN.0`). +- [ ] A public HTTPS URL that routes to `/channels/whatsapp/webhook` on the + manager, and a webhook subscription to the `messages` field. +- [ ] Billing is ready for the account. +- [ ] A real support contact for `/help` and escalation. +- [ ] Identity: who may use the pilot and how senders are linked (§3). +- [ ] Durable storage: a SQLite file on a persistent volume for one host, or + PostgreSQL. +- [ ] A retention decision for stored messages (§9). + +## 2. Deploy + +Run the conversation manager, not the engine API. The image's default +`agentctl serve` does not host conversations or the channel. + +```bash +make build +``` + +Then, from `examples/whatsapp-pilot/` with a filled-in `.env`: + +```bash +docker compose up -d +``` + +- **Expose only the webhook.** The manager also serves the web conversation API, + which accepts anonymous visitors by default. Anyone who reaches that API can + talk to the assistant without the WhatsApp identity check. The compose file + therefore binds port 8100 to `127.0.0.1`. Your HTTPS reverse proxy on the + same host should forward only `/channels/whatsapp/webhook` to it. If the proxy + runs in a container, attach it to the compose network instead of publishing + the port. +- `agent-manager` applies database migrations at startup (`--migrate`, the + default). Migrations 0007–0010 add the channel tables. +- Storage defaults to SQLite on the `pilot-data` volume. `EXTRA_DB_BACKEND` and + `EXTRA_DB_URL` in `.env` override it, for example for PostgreSQL. +- The queue and outbox live in the database. Keep its volume across restarts. +- Only one instance may run the worker for a connection. A second instance with + the worker enabled fails at startup with `ChannelWorkerAlreadyRunning`. + Other instances may run intake only. +- `docker compose stop` sends SIGTERM. The worker drains active turns for up to + `EXTRA_WHATSAPP_WORKER_DRAIN_SECONDS`, so the compose file allows 30 seconds + before the process is killed. + +## 3. Identity + +The worker resolves every sender through the configured identity plugin +(`module:Class`) at preparation, before execution, and before each send. +Message text never chooses the user, organization, or roles, and the principal's +organization must equal `EXTRA_WHATSAPP_ORGANIZATION_ID`. + +The example plugin, `whatsapp_identity:AllowlistIdentityResolver`, reads a JSON +allowlist on every lookup. Keys are the sender IDs exactly as Meta sends them in +`from` (international digits without `+`). Removing an entry revokes access on +the next message. Possessing the phone is the only proof, so use it only for a +closed pilot. Replace it with the organization's account-linking resolver before +the assistant reaches internal information. + +## 4. Configuration + +All settings are environment variables, read at startup. Changing one requires +a restart. + +| Variable | Stage | Meaning | +| --- | --- | --- | +| `EXTRA_WHATSAPP_ENABLED` | 1 | Mount the webhook and store incoming messages. | +| `EXTRA_WHATSAPP_CONNECTION_ID` | 1 | Internal name of this number (default `whatsapp`). | +| `EXTRA_WHATSAPP_ACCOUNT_ID` | 1 | Business account ID; entries for other accounts are ignored. | +| `EXTRA_WHATSAPP_PHONE_NUMBER_ID` | 1 | Receiving and sending number ID. | +| `EXTRA_WHATSAPP_APP_SECRET` | 1 | Verifies `X-Hub-Signature-256`. | +| `EXTRA_WHATSAPP_VERIFY_TOKEN` | 1 | Answers Meta's subscription challenge. | +| `EXTRA_WHATSAPP_MAX_BODY_BYTES` | 1 | Webhook body limit (default 262144). | +| `EXTRA_WHATSAPP_WORKER_ENABLED` | 2 | Run the turn worker. Requires stage 1. | +| `EXTRA_WHATSAPP_ORGANIZATION_ID` | 2 | Organization every sender must belong to. | +| `EXTRA_WHATSAPP_IDENTITY_RESOLVER` | 2 | Identity plugin, `module:Class`. | +| `EXTRA_WHATSAPP_WORKER_CONCURRENCY` | 2 | Senders processed in parallel (1–16, default 4). | +| `EXTRA_WHATSAPP_WORKER_POLL_SECONDS` | 2 | Queue polling interval (default 0.5). | +| `EXTRA_WHATSAPP_WORKER_TURN_SECONDS` | 2 | Time limit per turn (default 120). | +| `EXTRA_WHATSAPP_WORKER_DRAIN_SECONDS` | 2 | Shutdown drain limit (default 15). | +| `EXTRA_WHATSAPP_STATUS_LOG_SECONDS` | 2 | Interval of `channel status:` lines (default 60). | +| `EXTRA_WHATSAPP_SENDER_ENABLED` | 3 | Send answers. Requires stage 2. | +| `EXTRA_WHATSAPP_API_VERSION` | 3 | Pinned Graph API version, `vNN.0`. | +| `EXTRA_WHATSAPP_ACCESS_TOKEN` | 3 | System-user token for sending. | +| `EXTRA_WHATSAPP_SUPPORT_CONTACT` | 3 | Contact returned by `/help`. | + +Startup fails if a stage is enabled without its prerequisites or required +values, or if storage is in-memory. Secrets are excluded from settings output. + +## 5. Enablement order and pause states + +Enable one stage at a time, and check health and logs before the next: + +| State | Flags on | New messages | Pending work | +| --- | --- | --- | --- | +| Off | none | Webhook returns 404; nothing is stored. | Preserved. | +| Intake | 1 | Verified and stored as `queued`. | Waits. No status lines yet (they come from the worker). | +| Answering | 1, 2 | Answered and saved, not sent. | Answers wait as `answered`. | +| Live | 1, 2, 3 | Answered and sent. | Waiting answers are sent. | + +Moving between states never deletes accepted work. To pause, set the flags for +the target state and restart. While intake is off, Meta's own retry policy +applies to callbacks it could not deliver; check its current documentation +before a long outage. + +Free-form replies are allowed only within 24 hours of the user's latest +message. An answer that waits longer than that is blocked rather than sent, and +is counted in `delivery_blocked`. + +## 6. Health and monitoring + +`GET /health` returns 200 with `"status": "ok"`. When the worker is enabled but +has stopped, for example after losing exclusive ownership, it returns 503 with +`"status": "degraded"`. + +The worker logs one line per `EXTRA_WHATSAPP_STATUS_LOG_SECONDS`: + +```text +channel status: connection=whatsapp inbox_queued=0 inbox_running=1 inbox_answered=0 + inbox_oldest_s=4 needs_review=0 inbox_failed=0 outbox_pending=0 outbox_oldest_s=none + delivery_unknown=0 delivery_failed=0 delivery_blocked=0 +``` + +| Field | Meaning | +| --- | --- | +| `inbox_queued`, `inbox_running`, `inbox_answered` | Messages waiting, in progress, or answered but not yet sent. | +| `inbox_oldest_s` | Age of the oldest unfinished message; `none` when there is none. | +| `needs_review`, `inbox_failed` | Turns held for an administrator, or failed. | +| `outbox_pending`, `outbox_oldest_s` | Unsent reply parts (a long reply is split into several), and how long ago their message arrived. | +| `delivery_unknown` | Replies with an uncertain send outcome; never retried automatically. | +| `delivery_failed` | Replies the provider rejected. | +| `delivery_blocked` | Replies blocked by identity, the 24-hour window, or format. | + +`needs_review`, `inbox_failed`, and the `delivery_*` fields are totals since +the database was created, so alert when they increase. Lines contain no message +text, sender IDs, tokens, or provider message IDs. + +Every rejected webhook request logs `whatsapp webhook rejected: reason=`, +where the reason is `signature`, `subscription`, `encoding`, `content_type`, +`length`, `size`, or `incomplete`. A burst of `signature` rejections means a +wrong app secret or forged traffic. + +Suggested alerts: health 503; `inbox_oldest_s` or `outbox_oldest_s` above a few +minutes; any increase in `needs_review`, `delivery_unknown`, or +`delivery_failed`; repeated `reason=signature`; `webhook intake failed` or +`channel supervisor stopped` warnings. + +## 7. Restarts and recovery + +A restart preserves every accepted message. On the next start the worker: + +- continues `queued` and prepared messages; +- sends an answer that was saved but not yet sent, without generating it again; +- holds a turn that was running without a saved answer for review instead of + running it twice; +- marks a send interrupted mid-request as `delivery_unknown` for review instead + of risking a duplicate message. + +Rehearse this before the pilot, as the automated +`test_whatsapp_pilot_rehearsal.py` does with a fake provider: + +1. Start with intake only and send a test message: it is stored as `queued`. +2. Restart with all three stages: it is answered and sent once. +3. Restart again: nothing is sent twice, and the status line shows no pending + work. + +## 8. Rollback + +To stop the channel but keep the current version, disable the flags (§5) and +restart. Data is kept. + +To run an application version older than the channel migrations, downgrade the +schema first. **This deletes all channel data** (queue, bindings, outbox, +delivery history) while keeping conversations. Back up the database and stop +the manager. Then open a shell in the same image, volume, and settings; the +image's entrypoint would otherwise pass `sh` to `agentctl`: + +```bash +docker compose run --rm --entrypoint sh manager +``` + +Inside that shell: + +```bash +cd "$(python -c 'import agent_manager, pathlib; print(pathlib.Path(agent_manager.__file__).parent / "infrastructure/persistence/migrations")')" +``` + +```bash +alembic -c alembic.ini downgrade 0006 +``` + +Without Docker, run the last two commands in the manager's Python environment +with the same `EXTRA_DB_URL`. + +## 9. Data and retention + +Stored channel data: normalized message text and sender IDs, reply text, sender +bindings and opt-out state, delivery observations, and webhook receipts (body +digest and issue codes, not raw bodies). Ordinary logs exclude message text, +sender IDs, and tokens. + +There is no automatic deletion of channel data yet. Agree on a retention period +with the organization, and restrict database access, before the pilot. Automated +retention is a follow-up task. + +## 10. Real-number smoke test + +Only with explicit authorization, on the pilot number, with one allowlisted +tester: + +1. Deploy with intake only. Confirm the subscription challenge succeeds in Meta. +2. Send a message: a `queued` row exists and no signature rejections are logged. +3. Enable the worker and restart: an answer is saved, not sent. +4. Enable sending and restart: the tester receives the answer once. +5. Send `/help`: the support contact arrives without a model call. +6. Send a message from a non-allowlisted number: it is marked `denied` and no + answer is sent. +7. Stop and start: no duplicate messages. Record the results and the API version. diff --git a/examples/whatsapp-pilot/.env.example b/examples/whatsapp-pilot/.env.example new file mode 100644 index 00000000..6c34bc86 --- /dev/null +++ b/examples/whatsapp-pilot/.env.example @@ -0,0 +1,34 @@ +# Copy to .env in this directory. .env is gitignored; never commit real values. + +# Model provider for agents.yml. +OPENAI_API_KEY=replace-with-your-model-key + +# Storage: docker-compose.yml sets EXTRA_DB_URL to a file on a durable volume. +# For PostgreSQL instead: +# EXTRA_DB_BACKEND=postgres +# EXTRA_DB_URL=postgresql://user:password@host/database + +# Stage 1 — webhook intake. Messages are stored; nothing is answered yet. +EXTRA_WHATSAPP_ENABLED=false +EXTRA_WHATSAPP_CONNECTION_ID=whatsapp +EXTRA_WHATSAPP_ACCOUNT_ID=replace-with-business-account-id +EXTRA_WHATSAPP_PHONE_NUMBER_ID=replace-with-phone-number-id +EXTRA_WHATSAPP_APP_SECRET=replace-with-meta-app-secret +EXTRA_WHATSAPP_VERIFY_TOKEN=replace-with-a-random-verify-token + +# Stage 2 — worker. Requires stage 1. Answers are saved but not sent. +EXTRA_WHATSAPP_WORKER_ENABLED=false +EXTRA_WHATSAPP_ORGANIZATION_ID=replace-with-organization-id +EXTRA_WHATSAPP_IDENTITY_RESOLVER=whatsapp_identity:AllowlistIdentityResolver +# Path inside the container; docker-compose.yml sets this to /workspace/senders.json. +# WHATSAPP_PILOT_SENDERS_FILE=./senders.json + +# Stage 3 — sending. Requires stage 2. +EXTRA_WHATSAPP_SENDER_ENABLED=false +# Pin the Graph API version your account supports (format vNN.0). +EXTRA_WHATSAPP_API_VERSION=replace-with-pinned-version +EXTRA_WHATSAPP_ACCESS_TOKEN=replace-with-system-user-access-token +EXTRA_WHATSAPP_SUPPORT_CONTACT=replace-with-support-contact + +# Optional: seconds between `channel status:` log lines (default 60). +# EXTRA_WHATSAPP_STATUS_LOG_SECONDS=60 diff --git a/examples/whatsapp-pilot/.gitignore b/examples/whatsapp-pilot/.gitignore new file mode 100644 index 00000000..6f731aa2 --- /dev/null +++ b/examples/whatsapp-pilot/.gitignore @@ -0,0 +1 @@ +senders.json diff --git a/examples/whatsapp-pilot/README.md b/examples/whatsapp-pilot/README.md new file mode 100644 index 00000000..101993a0 --- /dev/null +++ b/examples/whatsapp-pilot/README.md @@ -0,0 +1,42 @@ +# WhatsApp pilot + +A deployable starting point for the WhatsApp channel: one tool-free assistant, +a sender allowlist, and the conversation manager on durable storage. Read the +[runbook](../../docs/WHATSAPP_RUNBOOK.md) before enabling anything. + +| File | Purpose | +| --- | --- | +| `agents.yml`, `prompts/assistant.md` | The assistant. It has no tools, so it never waits for an approval that WhatsApp cannot answer. | +| `whatsapp_identity.py` | `AllowlistIdentityResolver`: links listed senders to pilot users. For a closed pilot only. | +| `senders.example.json` | Allowlist format, keyed by the sender ID Meta sends in `from`. | +| `.env.example` | Every setting, with all three stages off. | +| `docker-compose.yml` | Runs `agent-manager` with the database on a durable volume. | + +## Start + +```bash +cp .env.example .env +``` + +```bash +cp senders.example.json senders.json +``` + +Fill in `.env` and `senders.json`, build the image from the repository root +with `make build`, then: + +```bash +docker compose up -d +``` + +Turn on the stages one at a time, in this order, restarting after each: +intake (`EXTRA_WHATSAPP_ENABLED`), worker (`EXTRA_WHATSAPP_WORKER_ENABLED`), +sending (`EXTRA_WHATSAPP_SENDER_ENABLED`). + +The manager listens on `127.0.0.1:8100` only. It also serves the web +conversation API, which accepts anonymous visitors by default, so do not publish +the port. Configure your HTTPS reverse proxy to forward only +`https:///channels/whatsapp/webhook` to it. + +`senders.json` and `.env` contain personal data and secrets. Keep them out of +version control. diff --git a/examples/whatsapp-pilot/agents.yml b/examples/whatsapp-pilot/agents.yml new file mode 100644 index 00000000..11f17307 --- /dev/null +++ b/examples/whatsapp-pilot/agents.yml @@ -0,0 +1,23 @@ +system: + name: "WhatsApp Pilot Assistant" + +defaults: + model: + provider: openai + name: gpt-4o-mini + temperature: 0.2 + +# No tools: a pending approval cannot be answered over WhatsApp, so the pilot +# agent must never need one. +execution: + max_iterations: 4 + +agents: + pilot_assistant: + name: "Pilot Assistant" + description: "Answers pilot users' questions in short plain-text messages." + prompts: + system: prompts/assistant.md + +graph: + pilot_assistant: diff --git a/examples/whatsapp-pilot/docker-compose.yml b/examples/whatsapp-pilot/docker-compose.yml new file mode 100644 index 00000000..959cd5d1 --- /dev/null +++ b/examples/whatsapp-pilot/docker-compose.yml @@ -0,0 +1,28 @@ +# The image's default `agentctl serve` does not host conversations or the +# WhatsApp channel, so this service runs `agent-manager` instead. +# Build the image first from the repository root: `make build`. +services: + manager: + image: ghcr.io/extra-org/extra:local + command: ["agent-manager", "--config", "/workspace/agents.yml", "--host", "0.0.0.0", "--port", "8100"] + env_file: + - .env + environment: + # A value in .env (for example PostgreSQL) takes precedence over this file. + EXTRA_DB_URL: ${EXTRA_DB_URL:-sqlite+aiosqlite:////data/pilot.db} + WHATSAPP_PILOT_SENDERS_FILE: /workspace/senders.json + volumes: + - .:/workspace + # The queue and outbox live here; keep this volume across restarts. + - pilot-data:/data + ports: + # Loopback only. The manager also serves the web conversation API, which + # is anonymous by default; expose just the webhook path through a proxy. + - "127.0.0.1:8100:8100" + restart: unless-stopped + # Longer than the worker drain (EXTRA_WHATSAPP_WORKER_DRAIN_SECONDS, 15 s + # by default), so a stop lets active turns finish before the process exits. + stop_grace_period: 30s + +volumes: + pilot-data: diff --git a/examples/whatsapp-pilot/prompts/assistant.md b/examples/whatsapp-pilot/prompts/assistant.md new file mode 100644 index 00000000..26d7ca89 --- /dev/null +++ b/examples/whatsapp-pilot/prompts/assistant.md @@ -0,0 +1,8 @@ +You are a pilot assistant that people reach over WhatsApp. + +- Reply in the language the person writes in. Hebrew and English are both common. +- Keep replies short: a few sentences of plain text. WhatsApp does not render + Markdown tables or headings. +- If you do not know something, say so. Do not guess facts about the organization. +- Never ask for passwords, one-time codes, or payment details. +- When the person needs a human, tell them to send /help for the support contact. diff --git a/examples/whatsapp-pilot/senders.example.json b/examples/whatsapp-pilot/senders.example.json new file mode 100644 index 00000000..cb89c3cb --- /dev/null +++ b/examples/whatsapp-pilot/senders.example.json @@ -0,0 +1,7 @@ +{ + "15550001111": { + "user_id": "pilot-user-1", + "display_name": "Pilot User", + "roles": ["employee"] + } +} diff --git a/examples/whatsapp-pilot/whatsapp_identity.py b/examples/whatsapp-pilot/whatsapp_identity.py new file mode 100644 index 00000000..c698fd4e --- /dev/null +++ b/examples/whatsapp-pilot/whatsapp_identity.py @@ -0,0 +1,52 @@ +"""Pilot identity: an administrator-managed allowlist of WhatsApp senders. + +Possessing the phone number is the only proof. Use this for a closed pilot and +replace it with the organization's account-linking resolver before internal +information is exposed. +""" + +import asyncio +import json +import os +from pathlib import Path + +from agent_manager.domain.channels.identity import ( + ChannelIdentityResolver, + ChannelIdentityUnavailable, +) +from agent_manager.domain.channels.models import ChannelAddress +from agent_manager.domain.identity import Principal + + +class AllowlistIdentityResolver(ChannelIdentityResolver): + def __init__(self) -> None: + self._path = Path(os.environ["WHATSAPP_PILOT_SENDERS_FILE"]) + self._organization_id = os.environ["EXTRA_WHATSAPP_ORGANIZATION_ID"] + + async def resolve(self, address: ChannelAddress) -> Principal | None: + # Read on every lookup, so removing a sender revokes access immediately. + try: + text = await asyncio.to_thread(self._path.read_text, encoding="utf-8") + senders = json.loads(text) + except (OSError, ValueError) as exc: + raise ChannelIdentityUnavailable("sender allowlist is unreadable") from exc + entry = senders.get(address.sender_id) if isinstance(senders, dict) else None + if not isinstance(entry, dict): + return None + user_id, roles = entry.get("user_id"), entry.get("roles", []) + display_name = entry.get("display_name") + # A malformed entry denies access rather than granting partial roles. + if ( + not isinstance(user_id, str) + or not user_id.strip() + or not isinstance(roles, list) + or not all(isinstance(role, str) for role in roles) + or not (display_name is None or isinstance(display_name, str)) + ): + return None + return Principal.external( + user_id, + display_name=display_name, + roles=tuple(roles), + organization_id=self._organization_id, + ) diff --git a/tasks/0021-whatsapp-pilot-packaging.md b/tasks/0021-whatsapp-pilot-packaging.md new file mode 100644 index 00000000..83f55879 --- /dev/null +++ b/tasks/0021-whatsapp-pilot-packaging.md @@ -0,0 +1,105 @@ +# Task 0021 — WhatsApp pilot packaging and operation (WA-06) + +## Goal / authorization + +Implement the final planned stage requested by the user after the WA-05 review: +an administrator runbook, a deployable pilot example, operational monitoring, +and fake-provider evidence for the complete round trip and durable restart. +The user chose structured log lines for monitoring instead of a new HTTP or CLI +interface. The real-number smoke test remains a separately authorized +deployment activity. + +## Read first + +AGENTS.md; project-architecture, senior-python-engineering, testing, +documentation, code-review; the WhatsApp plan §6–§7, ADRs 0006–0010, task +0019/0020, and the existing channel implementation and tests. + +## Scope / files allowed to change + +- This task, tasks/README.md, docs/README.md, docs/WHATSAPP_INTEGRATION_PLAN.md, + and new docs/WHATSAPP_RUNBOOK.md. +- New examples/whatsapp-pilot/: README, agent config and prompt, an allowlist + identity resolver plugin, `.env.example`, `senders.example.json`, and a + `docker-compose.yml` that starts `agent-manager` on durable storage. +- Under src/agent_manager/: new domain/channels/status.py and + infrastructure/persistence/channels/status.py; existing + application/channels/{worker,worker_options}.py, api/routes/whatsapp.py, + composition.py, config.py. +- New tests/agent_manager/test_channel_status.py and + test_whatsapp_pilot_rehearsal.py; existing test_whatsapp_webhook.py for + rejection-log assertions. + +## Acceptance criteria + +1. The worker logs one bounded `channel status:` line per configured interval + with inbox counts by attention state, the oldest active inbox age, pending + delivery parts and their oldest age, and unknown, failed, and blocked + delivery counts. No message text, sender identifier, token, or provider + message ID appears in it. A failed status read is logged and never stops + the worker. +2. Every rejected webhook request logs `whatsapp webhook rejected:` with a + fixed reason (signature, subscription, encoding, content type, length, + size) and no request content. +3. The interval is configurable and validated; the default needs no setting. +4. A fake-provider rehearsal proves: intake while the worker is disabled keeps + the message queued; a restart with the worker and sender enabled on the same + database answers and sends it exactly once; a signed `delivered` status + callback is projected; a further restart sends nothing again. +5. The runbook documents preflight, configuration, the actual enablement order + and pause states, deployment command and migrations, health, monitoring, + restart, rollback, retention, and the manual smoke-test checklist, matching + the implemented behavior. +6. The example configuration validates, contains no secrets, and keeps the + agent free of approval-dependent tools. +7. `make check` passes. + +## Out of scope + +Real Meta credentials or traffic, the smoke test itself, Google/JWT account +linking, a metrics endpoint or dependency, templates, media, and any schema +migration. + +## Commands / report + +`make check`; `agentctl validate` for the example; report per AGENTS.md §9. + +## Implementation result + +- `SqlChannelStatusReader` reads grouped counts and oldest ages without + selecting text or identifiers. The worker bounds each read to five seconds + and logs the result each `EXTRA_WHATSAPP_STATUS_LOG_SECONDS` (default 60); a + failed read logs its type. +- The webhook route logs a fixed reason for every rejection. +- `examples/whatsapp-pilot/` validates offline. Its allowlist resolver denies + malformed entries, re-reads the file per lookup, and fails closed when the + file is unreadable. +- The rehearsal test proves paused intake, a single answer and send after + restart, a projected `delivered` callback, and no resend on a later restart. +- The runbook's rollback command was rehearsed on a temporary database: + downgrading to 0006 removes the channel tables and keeps conversations. + +### Self-review / corrections + +- The example published port 8100 on all interfaces. The manager's web + conversation API is anonymous by default, so that bypassed the WhatsApp + identity check. The port is now bound to loopback, and the runbook requires a + proxy that forwards only the webhook path. +- Compose `environment` overrode `EXTRA_DB_URL` from `.env`, pairing a + PostgreSQL backend with a SQLite URL. The value now defaults only when unset. +- The rollback commands did not work inside the image, whose entrypoint passes + other commands to `agentctl`. The runbook now opens a shell with + `--entrypoint sh`. +- The status read had no bound inside the scheduling loop. A stalled database + could delay scheduling and turn a graceful drain into cancellation. It now + times out after five seconds. +- Delivery outcomes counted parts while blocked replies counted replies. All + outcome fields now count replies; `outbox_pending` still counts parts. + +### Limits / next task + +- No automatic retention for channel data exists. Proposed next task: a + retention job with a configurable period for channel rows. +- The status query scans delivery history without retention; acceptable at + pilot volume. +- The real-number smoke test and organization account linking remain pending. diff --git a/tasks/README.md b/tasks/README.md index 11827899..816351bf 100644 --- a/tasks/README.md +++ b/tasks/README.md @@ -30,5 +30,6 @@ finishing, Expected final report.** | 0018 | [SQL test resource cleanup](0018-sql-test-resource-cleanup.md) | 0017 review verification | | 0019 | [WhatsApp outbound delivery and controls](0019-whatsapp-delivery-and-controls.md) | 0013–0018; WhatsApp WA-05 | | 0020 | [WhatsApp delivery review](0020-whatsapp-delivery-review.md) | 0019 | +| 0021 | [WhatsApp pilot packaging](0021-whatsapp-pilot-packaging.md) | 0020; WhatsApp WA-06 | Before starting any task, read `AGENTS.md` and the skill(s) the task lists. diff --git a/tests/agent_manager/test_whatsapp_pilot_rehearsal.py b/tests/agent_manager/test_whatsapp_pilot_rehearsal.py new file mode 100644 index 00000000..6937f264 --- /dev/null +++ b/tests/agent_manager/test_whatsapp_pilot_rehearsal.py @@ -0,0 +1,192 @@ +"""Pilot rehearsal: paused intake survives restarts and is answered and sent exactly once.""" + +import asyncio +import logging +import time +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from datetime import UTC, datetime +from pathlib import Path + +import pytest +from fastapi.testclient import TestClient +from sqlmodel import SQLModel + +from agent_engine.core.validator import SystemSpecValidator +from agent_engine.parsers.yaml.parser import YAMLParser +from agent_manager.api.app import create_app +from agent_manager.config import Settings +from agent_manager.domain.channels.delivery import SendOutcome +from agent_manager.domain.channels.identity import ChannelIdentityUnavailable +from agent_manager.domain.channels.models import ChannelAddress +from agent_manager.infrastructure.channels.identity_loader import load_channel_identity_resolver +from agent_manager.infrastructure.persistence.channels.tables import ( + ChannelDeliveryPartRow, + ChannelInboxRow, +) +from agent_manager.infrastructure.persistence.database import create_db_engine +from tests.agent_manager.test_whatsapp_delivery_lifecycle import outbound_values +from tests.agent_manager.test_whatsapp_webhook import ( + ENDPOINT, + StartupEngine, + change, + encoded, + message, + payload, + saved_rows, + settings_values, + signed, +) + +CONFIG = Path(__file__).parents[2] / "examples/enterprise-knowledge-assistant/agents.yaml" +SENDER = "opaque-sender" + + +class Provider: + def __init__(self) -> None: + self.calls: list[tuple[str, str]] = [] + self.closed = False + + async def send(self, recipient: str, text: str) -> SendOutcome: + self.calls.append((recipient, text)) + return SendOutcome("accepted", f"wamid.reply-{len(self.calls)}") + + async def close(self) -> None: + self.closed = True + + +def wait_until(condition: Callable[[], bool], seconds: float = 3) -> None: + deadline = time.monotonic() + seconds + while not condition(): + assert time.monotonic() < deadline, "rehearsal step did not complete in time" + time.sleep(0.01) + + +def post(client: TestClient, value: dict) -> None: + body = encoded(value) + assert client.post(ENDPOINT, content=body, headers=signed(body)).status_code == 200 + + +@pytest.fixture +def url(tmp_path: Path) -> str: + url = f"sqlite+aiosqlite:///{tmp_path / 'pilot.db'}" + + async def schema() -> None: + database = create_db_engine(url) + try: + async with database.begin() as connection: + await connection.run_sync(SQLModel.metadata.create_all) + finally: + await database.dispose() + + asyncio.run(schema()) + return url + + +def test_paused_intake_is_answered_once_after_restart_and_delivery_is_projected( + url: str, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + engine = StartupEngine() + providers: list[Provider] = [] + + def provider_factory(*args: object, **kwargs: object) -> Provider: + providers.append(Provider()) + return providers[-1] + + monkeypatch.setattr("agent_manager.api.app.LangGraphEngine", lambda *a, **kw: engine) + monkeypatch.setattr("agent_manager.composition.WhatsAppSender", provider_factory) + # Startup replaces logging handlers; keep pytest's capture in place. + monkeypatch.setattr("agent_manager.api.app.configure_logging", lambda: None) + caplog.set_level(logging.INFO, logger="agent_manager.application.channels.worker") + + @contextmanager + def deployment(values: dict) -> Iterator[TestClient]: + app = create_app(str(CONFIG), Settings.from_values(database_url=url, **values)) + with TestClient(app) as client: + assert client.get("/health").status_code == 200 + yield client + + # Intake only: the worker is paused, so the accepted message waits durably. + with deployment(settings_values()) as client: + post(client, payload(change(message("question", text={"body": "hello"})))) + assert [row.state for row in saved_rows(url, ChannelInboxRow)] == ["queued"] + assert engine.prompts == [] and providers == [] + + # Restart with the worker and sender: answered once, then a signed callback lands. + full = outbound_values(extra_whatsapp_status_log_seconds=0.05) + with deployment(full) as client: + wait_until(lambda: providers[0].calls == [(SENDER, "answer:hello")]) + delivered = { + "id": "wamid.reply-1", + "recipient_id": SENDER, + "status": "delivered", + "timestamp": str(int(datetime.now(UTC).timestamp())), + } + post(client, payload(change(statuses=[delivered]))) + wait_until( + lambda: ( + [part.state for part in saved_rows(url, ChannelDeliveryPartRow)] == ["delivered"] + ) + ) + assert providers[0].closed + + # A further restart finds nothing to redo. + with deployment(full): + time.sleep(0.3) + assert providers[1].calls == [] + assert engine.prompts == ["hello"] + assert [part.state for part in saved_rows(url, ChannelDeliveryPartRow)] == ["delivered"] + + status_lines = [ + record.getMessage() + for record in caplog.records + if record.getMessage().startswith("channel status:") + ] + assert status_lines + for private in (SENDER, "hello", "answer:hello", "wamid."): + assert not any(private in line for line in status_lines) + + +EXAMPLE = Path(__file__).parents[2] / "examples/whatsapp-pilot" + + +def test_pilot_example_config_passes_startup_validation() -> None: + spec = YAMLParser().parse(str(EXAMPLE / "agents.yml")) + assert SystemSpecValidator().validate(spec, EXAMPLE) == [] + + +async def test_pilot_allowlist_resolver_links_revokes_and_fails_closed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + senders = tmp_path / "senders.json" + senders.write_text((EXAMPLE / "senders.example.json").read_text(), encoding="utf-8") + monkeypatch.setenv("WHATSAPP_PILOT_SENDERS_FILE", str(senders)) + monkeypatch.setenv("EXTRA_WHATSAPP_ORGANIZATION_ID", "org") + # The manager adds the config directory to sys.path at startup. + monkeypatch.syspath_prepend(str(EXAMPLE)) + resolver = load_channel_identity_resolver("whatsapp_identity:AllowlistIdentityResolver") + listed = ChannelAddress("whatsapp", "system", "15550001111") + + principal = await resolver.resolve(listed) + assert principal is not None + assert (principal.external_id, principal.organization_id, principal.roles) == ( + "pilot-user-1", + "org", + ("employee",), + ) + assert await resolver.resolve(ChannelAddress("whatsapp", "system", "15559999999")) is None + + for malformed in ( + '{"user_id": "u", "roles": "employee"}', + '{"user_id": " "}', + '{"user_id": 7}', + ): + senders.write_text(f'{{"15550001111": {malformed}}}', encoding="utf-8") + assert await resolver.resolve(listed) is None + + senders.write_text("{}", encoding="utf-8") + assert await resolver.resolve(listed) is None + + senders.write_text("not json", encoding="utf-8") + with pytest.raises(ChannelIdentityUnavailable): + await resolver.resolve(listed)