diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ab2086..9055977 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ All notable changes to the E2A framework documents and reference code. +## v1.7.1 — Durable outbox stores (design note MP-05) + +### Added +- **`e2a.port_contract.check_outbox_store`:** 11 scenarios every `BaseOutboxStore` adapter passes: + - round trip; oldest first; the limit; + - mark once; an unknown id; + - fencing (a stale sweeper cannot mark); + - a race in which exactly one sweeper wins. +- **e2a-adapters 1.7.1, the transactional outbox in each database's own transaction.** A record commits with the business change or not at all: + - **`DynamoDBOutboxStore`** puts `transact_item(record)` in your `TransactWriteItems`. Pending records are one Query in id order (ids sort by time). `mark_published` deletes the record and advances the fence in one transaction. + - **`FirestoreOutboxStore`** takes `add(record, transaction=tx)` (or `batch=`). Pending is ordered by `created_at` with no composite index; `mark_published` is one transaction. + - **`CosmosOutboxStore`** keeps the record in the business document's partition: `batch_operation(record, partition_key)` goes into your transactional batch. Pending is one cross-partition query. The fence and the delete are two conditional writes, because they are in different partitions; a stale double-mark is harmless because consumers deduplicate. + - All three pass the contract on DynamoDB Local and the Firestore and Cosmos DB emulators. +- **The cloud profiles bind `outbox_store`** (key `outbox_name`, default `default`). + +### Changed +- **`PubSubBroker` imports its SDK on first use** (MP-05 follow-up): a profile binds and verifies without google-cloud-pubsub installed. +- **`param_dropped` is logged only for a temperature someone set** (keyword, config, profile or environment), not for the registry default (MP-04 follow-up). A model declaring `sampling: unsupported` no longer warns on every call. + ## v1.7.0 — Messaging, identity, secrets and the cloud profiles (design note MP-05) Cloud infrastructure behind E2A's ports, selected by configuration. The contracts are vendor-neutral and diff --git a/README.md b/README.md index 1a52cb0..1503db7 100644 --- a/README.md +++ b/README.md @@ -12,16 +12,16 @@ It focuses on the boundary where probabilistic reasoning meets deterministic ent ## Install - pip install "e2a-framework @ git+https://github.com/subhamviky/e2a-framework.git@v1.7.0" + pip install "e2a-framework @ git+https://github.com/subhamviky/e2a-framework.git@v1.7.1" -Poetry: `e2a-framework = { git = "https://github.com/subhamviky/e2a-framework.git", tag = "v1.7.0" }`. +Poetry: `e2a-framework = { git = "https://github.com/subhamviky/e2a-framework.git", tag = "v1.7.1" }`. The framework is the `e2a` package (`from e2a import BaseAgent, BaseWorkflow, Steps, Step`); the release gate is `from e2a.release import ProjectReleaseGate`; the `e2a_base` module re-exports the same names. Since v1.3 hooks use the v1.2 signature (`state, ctx, …`). **Production adapters (e2a-adapters, since v1.4.0):** DynamoDB (v1.4.0), Firestore and Cosmos DB (v1.5.0) implementations of the idempotency, transactional, lease and checkpoint ports; since v1.7.0 also the brokers (SQS/SNS, Pub/Sub, Service Bus), the secret stores and the cloud profiles `aws`, `gcp`, `azure`, in [`adapters/`](adapters/README.md), -a separate distribution at this tag (`pip install "e2a-adapters[aws,gcp,azure] @ git+https://github.com/subhamviky/e2a-framework.git@v1.7.0#subdirectory=adapters"`; +a separate distribution at this tag (`pip install "e2a-adapters[aws,gcp,azure] @ git+https://github.com/subhamviky/e2a-framework.git@v1.7.1#subdirectory=adapters"`; pick the extras you need). Every adapter passes the port contract kit, `e2a.port_contract`. @@ -33,6 +33,7 @@ registry: a service's CI builds it from the tag (`mvn -f java/pom.xml install -D | e2a-framework | a2c-framework | capability-oriented-dsa | |---|---|---| +| `v1.7.1` (durable outbox stores on DynamoDB, Firestore and Cosmos DB, with an outbox contract; Pub/Sub SDK imported on first use; the default temperature no longer warns) | `v1.4.0` | `v0.2.x`, `v0.3.x` | | `v1.7.0` (messaging with exactly-once effects, identity at ingress, the literal-secret rule; e2a-adapters `1.7.0`: brokers, secret stores, cloud profiles; e2a-java `1.7.0`) | `v1.4.0` | `v0.2.x`, `v0.3.x` | | `v1.6.1` (vendor-neutral handling for managed providers: model routes, structured-output checks, scopes, telemetry and cost, adaptive admission, delegated runtimes, reranker port, classifier guardrails, pipeline and MCP checks; the neutrality test) | `v1.3.0` | `v0.2.x`, `v0.3.x` | | `v1.6.0` (contracts for managed providers: canonical types, typed refusals, bindings and profiles, capabilities, credentials, `check_llm_service`) | `v1.3.0` | `v0.2.x`, `v0.3.x` | diff --git a/adapters/README.md b/adapters/README.md index 8275500..c43b56c 100644 --- a/adapters/README.md +++ b/adapters/README.md @@ -14,7 +14,7 @@ An application chooses its cloud by naming a profile; every adapter is built fro from e2a import bindings_from_profile bindings = bindings_from_profile("aws", verify=True) # or "gcp", "azure"; or E2A_PROVIDER_PROFILE config = bindings.config() # idempotency_store, checkpoint_store, transactional_store, lease, - # message_broker, credential_provider, identity_resolver + # message_broker, outbox_store (1.7.1), credential_provider, identity_resolver ``` | Profile | Stores | Messaging | Tool credentials | Identity claims | @@ -42,6 +42,7 @@ Clients are created on first use, so a profile binds and verifies without creden | Google Pub/Sub (since 1.7.0) | `e2a-adapters[gcp]` | `PubSubBroker`: topics and subscriptions, a `-dlq` topic and subscription | | Azure Service Bus (since 1.7.0) | `e2a-adapters[azure]` | `ServiceBusBroker`: queues, topic subscriptions, the native dead-letter subqueue | | Secret stores (since 1.7.0) | per cloud | `SecretsManagerCredentialProvider`, `SecretManagerCredentialProvider`, `KeyVaultCredentialProvider`: tool auth headers from `secret_headers` | +| Outbox stores (since 1.7.1) | per cloud | `DynamoDBOutboxStore` (`transact_item` for your TransactWriteItems), `FirestoreOutboxStore` (`add(..., transaction=)`), `CosmosOutboxStore` (`batch_operation` for your transactional batch, in the business document's partition) | | Identity presets (since 1.7.0) | – | `e2a_adapters.identity.PRESETS`: the claim maps of Cognito, Entra ID and Identity Platform | Firestore and Cosmos DB share one implementation of the ports, `e2a_adapters.cas`. It works over any document store that can create-if-absent and replace-if-unchanged (an update time, an ETag), and each database supplies only a small `CasBackend`. PostgreSQL would be a third backend. @@ -51,15 +52,15 @@ Firestore and Cosmos DB share one implementation of the ports, `e2a_adapters.cas The version follows E2A. Install e2a-framework from its tag first, as every project already does: ```bash -pip install "e2a-framework @ git+https://github.com/subhamviky/e2a-framework.git@v1.7.0" \ - "e2a-adapters[aws] @ git+https://github.com/subhamviky/e2a-framework.git@v1.7.0#subdirectory=adapters" +pip install "e2a-framework @ git+https://github.com/subhamviky/e2a-framework.git@v1.7.1" \ + "e2a-adapters[aws] @ git+https://github.com/subhamviky/e2a-framework.git@v1.7.1#subdirectory=adapters" ``` Poetry: ```toml -"e2a-framework" = { git = "https://github.com/subhamviky/e2a-framework.git", tag = "v1.7.0" } -"e2a-adapters" = { git = "https://github.com/subhamviky/e2a-framework.git", tag = "v1.7.0", subdirectory = "adapters", extras = ["aws"] } +"e2a-framework" = { git = "https://github.com/subhamviky/e2a-framework.git", tag = "v1.7.1" } +"e2a-adapters" = { git = "https://github.com/subhamviky/e2a-framework.git", tag = "v1.7.1", subdirectory = "adapters", extras = ["aws"] } ``` ## DynamoDB diff --git a/adapters/e2a_adapters/__init__.py b/adapters/e2a_adapters/__init__.py index af494af..577d45b 100644 --- a/adapters/e2a_adapters/__init__.py +++ b/adapters/e2a_adapters/__init__.py @@ -15,4 +15,4 @@ """ from . import config_keys # noqa: F401 (registers the cloud keys) -__version__ = '1.7.0' +__version__ = '1.7.1' diff --git a/adapters/e2a_adapters/aws/__init__.py b/adapters/e2a_adapters/aws/__init__.py index 8dfb2a8..da2f696 100644 --- a/adapters/e2a_adapters/aws/__init__.py +++ b/adapters/e2a_adapters/aws/__init__.py @@ -2,12 +2,14 @@ DynamoDB* claims, rows, leases, checkpoints SQSBroker messaging (SQS queues, SNS topics), with dead letters + DynamoDBOutboxStore the transactional outbox (v1.7.1) SecretsManagerCredentialProvider tool credentials from Secrets Manager """ from .dynamodb import (MAX_VALUE_BYTES, DynamoDBCheckpointStore, DynamoDBIdempotencyStore, DynamoDBLease, DynamoDBTransactionalStore, is_condition_failure) +from .outbox import DynamoDBOutboxStore from .secrets import SecretsManagerCredentialProvider from .sqs import SQSBroker __all__ = ['DynamoDBIdempotencyStore', 'DynamoDBTransactionalStore', 'DynamoDBLease', 'DynamoDBCheckpointStore', - 'MAX_VALUE_BYTES', 'is_condition_failure', 'SQSBroker', 'SecretsManagerCredentialProvider'] + 'MAX_VALUE_BYTES', 'is_condition_failure', 'SQSBroker', 'SecretsManagerCredentialProvider', 'DynamoDBOutboxStore'] diff --git a/adapters/e2a_adapters/aws/outbox.py b/adapters/e2a_adapters/aws/outbox.py new file mode 100644 index 0000000..da1173a --- /dev/null +++ b/adapters/e2a_adapters/aws/outbox.py @@ -0,0 +1,103 @@ +"""The transactional outbox on DynamoDB (v1.7.1, design note MP-05, N1). + +A record is written in the SAME transaction as the business change: put transact_item(record) in +the application's TransactWriteItems (or call add() when there is no other write). The +sweeper (e2a.BrokerOutboxSweeper) reads pending() and publishes, and mark_published() deletes +the record in one transaction with the fencing check, so a stale sweeper cannot mark. + +Layout, in any table with string keys pk / sk (e2a-adapters' state table by default): + + pk = E2A#OUTBOX# sk = a pending record + pk = E2A#OUTBOX-FENCE sk = the newest fencing token seen + +Record ids sort by creation time (microseconds, then a random suffix), so pending() is one +Query in key order. All pending records share a partition: one outbox handles what one +partition writes (about 1,000 records a second); use several outbox names to go beyond that. +""" +import json +import time +import uuid +from typing import Any, Callable, Dict, List, Optional + +from botocore.exceptions import ClientError +from e2a import resolve_config +from e2a.messaging import BaseOutboxStore + +from .. import config_keys # noqa: F401 +from .._lazy import Lazy + + +def _s(value: str) -> Dict[str, str]: + return {'S': str(value)} + + +class DynamoDBOutboxStore(BaseOutboxStore): + capabilities = frozenset({'outbox_store'}) + + def __init__(self, client=None, table_name: str = None, name: str = 'default', + clock: Callable[[], float] = time.time) -> None: + if client is None or not table_name: + raise ValueError('DynamoDBOutboxStore needs client= and table_name= (or use from_config)') + self.client, self.table_name, self.name, self.clock = client, table_name, name, clock + self.pk = f'E2A#OUTBOX#{name}' + + @classmethod + def from_config(cls, config: Dict[str, Any], **kwargs) -> 'DynamoDBOutboxStore': + import boto3 + region, endpoint = resolve_config('aws_region', config), resolve_config('aws_endpoint_url', config) + return cls(Lazy(lambda: boto3.client('dynamodb', region_name=region, endpoint_url=endpoint)), + resolve_config('dynamodb_state_table', config), resolve_config('outbox_name', config), **kwargs) + + def _new_id(self) -> str: + return f'{int(self.clock() * 1_000_000):017d}-{uuid.uuid4().hex[:12]}' + + def transact_item(self, record: Dict[str, Any]) -> Dict[str, Any]: + """The Put for the application's own TransactWriteItems: the record is committed if and + only if the business change is. record['id'] is set when absent.""" + record.setdefault('id', self._new_id()) + payload = {k: record.get(k) for k in ('destination', 'body', 'attributes', 'tenant_id', 'idempotency_key')} + return {'Put': {'TableName': self.table_name, + 'Item': {'pk': _s(self.pk), 'sk': _s(record['id']), 'created_at': {'N': repr(self.clock())}, + 'record': _s(json.dumps(payload, separators=(',', ':'), ensure_ascii=False))}, + 'ConditionExpression': 'attribute_not_exists(pk)'}} + + def add(self, record, config=None, **kwargs): + item = self.transact_item(dict(record)) + self.client.put_item(**item['Put']) + return item['Put']['Item']['sk']['S'] + + def pending(self, limit=100, config=None, **kwargs): + resp = self.client.query(TableName=self.table_name, KeyConditionExpression='pk = :pk', + ExpressionAttributeValues={':pk': _s(self.pk)}, ConsistentRead=True, + Limit=max(1, int(limit))) + out: List[Dict[str, Any]] = [] + for item in resp.get('Items', []): + out.append({'id': item['sk']['S'], **json.loads(item['record']['S'])}) + return out + + def mark_published(self, record_id, config=None, **kwargs): + delete = {'Delete': {'TableName': self.table_name, 'Key': {'pk': _s(self.pk), 'sk': _s(record_id)}, + 'ConditionExpression': 'attribute_exists(pk)'}} + token: Optional[int] = kwargs.get('fencing_token') + try: + if token is None: + self.client.delete_item(**delete['Delete']) + else: + fence = {'Update': {'TableName': self.table_name, + 'Key': {'pk': _s('E2A#OUTBOX-FENCE'), 'sk': _s(self.name)}, + 'UpdateExpression': 'SET fence = :t', + 'ConditionExpression': 'attribute_not_exists(fence) OR fence <= :t', + 'ExpressionAttributeValues': {':t': {'N': str(int(token))}}}} + self.client.transact_write_items(TransactItems=[fence, delete]) + except ClientError as e: + code = e.response.get('Error', {}).get('Code') + if code == 'ConditionalCheckFailedException': + return False + if code == 'TransactionCanceledException': + # a lost condition (stale fence, already published) or a concurrent transaction + # on the same items is a lost race; throttling or validation is an error + reasons = [r.get('Code') for r in e.response.get('CancellationReasons') or []] + if all(r in (None, 'None', 'ConditionalCheckFailed', 'TransactionConflict') for r in reasons): + return False + raise + return True diff --git a/adapters/e2a_adapters/azure/__init__.py b/adapters/e2a_adapters/azure/__init__.py index 08bd67f..22489a4 100644 --- a/adapters/e2a_adapters/azure/__init__.py +++ b/adapters/e2a_adapters/azure/__init__.py @@ -2,12 +2,14 @@ Cosmos* claims, rows, leases, checkpoints ServiceBusBroker messaging, with native dead letters + CosmosOutboxStore the transactional outbox (v1.7.1) KeyVaultCredentialProvider tool credentials from Key Vault """ from .cosmos import (CosmosBackend, CosmosCheckpointStore, CosmosIdempotencyStore, CosmosLease, CosmosTransactionalStore) from .keyvault import KeyVaultCredentialProvider +from .outbox import CosmosOutboxStore from .servicebus import ServiceBusBroker __all__ = ['CosmosBackend', 'CosmosIdempotencyStore', 'CosmosTransactionalStore', 'CosmosLease', - 'CosmosCheckpointStore', 'ServiceBusBroker', 'KeyVaultCredentialProvider'] + 'CosmosCheckpointStore', 'ServiceBusBroker', 'KeyVaultCredentialProvider', 'CosmosOutboxStore'] diff --git a/adapters/e2a_adapters/azure/outbox.py b/adapters/e2a_adapters/azure/outbox.py new file mode 100644 index 0000000..5bc5b2e --- /dev/null +++ b/adapters/e2a_adapters/azure/outbox.py @@ -0,0 +1,116 @@ +"""The transactional outbox on Cosmos DB (v1.7.1, design note MP-05, N1). + +Cosmos DB commits atomically only within one logical partition, so a record lives in the +partition of the business document it belongs to. Pass that partition key and put +batch_operation(record, partition_key) into the application's transactional batch +(container.execute_item_batch). The record is then committed if and only if the business change +is. add(record, partition_key=...) writes it alone when there is no other write. + +Layout, in e2a-adapters' container (partition key /pk): + {id: , pk: >, e2a_outbox: , ...} + {id: , pk: "OUTBOX-FENCE", fence: } + +pending() is one cross-partition query ordered by created_at. mark_published() checks and +advances the fence with an ETag condition, then deletes the record with its ETag: two +conditional writes, not one transaction, because the fence and the record are in different +partitions. A stale sweeper that passes the fence check just before a newer one advances it can +still mark a record the newer one also published. That is harmless: consumers deduplicate on +the record's idempotency key. +""" +import json +import time +import uuid +from typing import Any, Callable, Dict, List, Optional, Tuple + +from azure.core import MatchConditions +from azure.cosmos import exceptions as cx +from e2a import resolve_config +from e2a.messaging import BaseOutboxStore + +from .. import config_keys # noqa: F401 +from .cosmos import _container_from + +_FENCE_PK = 'OUTBOX-FENCE' + + +class CosmosOutboxStore(BaseOutboxStore): + capabilities = frozenset({'outbox_store'}) + + def __init__(self, container, name: str = 'default', partition_key_field: str = 'pk', + clock: Callable[[], float] = time.time) -> None: + self.container, self.name, self.pk, self.clock = container, name, partition_key_field, clock + + @classmethod + def from_config(cls, config: Dict[str, Any], **kwargs) -> 'CosmosOutboxStore': + return cls(_container_from(config), resolve_config('outbox_name', config), **kwargs) + + def _new_id(self) -> str: + return f'{int(self.clock() * 1_000_000):017d}-{uuid.uuid4().hex[:12]}' + + def _item(self, record: Dict[str, Any], partition_key: Optional[str]) -> Dict[str, Any]: + record.setdefault('id', self._new_id()) + return {'id': record['id'], self.pk: partition_key or f'OUTBOX#{self.name}', 'e2a_outbox': self.name, + 'created_at': self.clock(), + 'record': json.dumps({k: record.get(k) for k in ('destination', 'body', 'attributes', 'tenant_id', + 'idempotency_key')}, + separators=(',', ':'), ensure_ascii=False)} + + def batch_operation(self, record: Dict[str, Any], partition_key: str) -> Tuple[str, tuple]: + """The create operation for the application's transactional batch on `partition_key`.""" + return ('create', (self._item(record, partition_key),)) + + def add(self, record, config=None, **kwargs): + item = self._item(dict(record), kwargs.get('partition_key')) + self.container.create_item(item) + return item['id'] + + def pending(self, limit=100, config=None, **kwargs): + query = ('SELECT TOP @n c.id, c.record FROM c WHERE c.e2a_outbox = @name ' + 'ORDER BY c.created_at ASC') + items = self.container.query_items(query, parameters=[{'name': '@n', 'value': max(1, int(limit))}, + {'name': '@name', 'value': self.name}], + enable_cross_partition_query=True) + return [{'id': it['id'], **json.loads(it['record'])} for it in items] + + def _find(self, record_id: str) -> Optional[Dict[str, Any]]: + items = list(self.container.query_items( + 'SELECT * FROM c WHERE c.e2a_outbox = @name AND c.id = @id', + parameters=[{'name': '@name', 'value': self.name}, {'name': '@id', 'value': record_id}], + enable_cross_partition_query=True)) + return items[0] if items else None + + def _advance_fence(self, token: int) -> bool: + while True: + try: + fence = self.container.read_item(self.name, partition_key=_FENCE_PK) + except cx.CosmosResourceNotFoundError: + try: + self.container.create_item({'id': self.name, self.pk: _FENCE_PK, 'fence': int(token)}) + return True + except cx.CosmosResourceExistsError: + continue # another sweeper created it: read again + if int(fence['fence']) > int(token): + return False + if int(fence['fence']) == int(token): + return True + try: + self.container.replace_item(self.name, {**{k: v for k, v in fence.items() if not k.startswith('_')}, + 'fence': int(token)}, + etag=fence['_etag'], match_condition=MatchConditions.IfNotModified) + return True + except cx.CosmosAccessConditionFailedError: + continue # moved meanwhile: read again + + def mark_published(self, record_id, config=None, **kwargs): + token = kwargs.get('fencing_token') + if token is not None and not self._advance_fence(int(token)): + return False + item = self._find(record_id) + if item is None: + return False + try: + self.container.delete_item(item['id'], partition_key=item[self.pk], etag=item['_etag'], + match_condition=MatchConditions.IfNotModified) + return True + except (cx.CosmosResourceNotFoundError, cx.CosmosAccessConditionFailedError): + return False diff --git a/adapters/e2a_adapters/config_keys.py b/adapters/e2a_adapters/config_keys.py index edeb16d..d61f247 100644 --- a/adapters/e2a_adapters/config_keys.py +++ b/adapters/e2a_adapters/config_keys.py @@ -50,6 +50,8 @@ def _map(v: Any) -> Dict[str, Any]: ConfigKey('servicebus_namespace', 'SERVICEBUS_NAMESPACE', None, str, _GROUP, 'ServiceBusBroker — .servicebus.windows.net, with Entra ID'), ConfigKey('servicebus_connection_string', 'SERVICEBUS_CONNECTION_STRING', None, str, _GROUP, 'ServiceBusBroker — a connection string (the emulator); unset in the cloud', secret=True), ConfigKey('keyvault_url', 'KEYVAULT_URL', None, str, _GROUP, 'KeyVaultCredentialProvider — https://.vault.azure.net'), + # the outbox (v1.7.1), every cloud + ConfigKey('outbox_name', 'E2A_OUTBOX_NAME', 'default', str, _GROUP, 'the outbox stores — one service\'s outbox; several names spread the load'), # secrets as tool credentials (N2), every cloud ConfigKey('secret_headers', 'E2A_SECRET_HEADERS', [], _list, _GROUP, '[{target, secret, header, prefix, json_key}] — the header a tool call to carries, read from a secret store'), ConfigKey('secret_cache_seconds', 'E2A_SECRET_CACHE_SECONDS', 300, int, _GROUP, 'secret-backed credential providers — how long a secret value is reused'), diff --git a/adapters/e2a_adapters/gcp/__init__.py b/adapters/e2a_adapters/gcp/__init__.py index 191640b..f8bbf4e 100644 --- a/adapters/e2a_adapters/gcp/__init__.py +++ b/adapters/e2a_adapters/gcp/__init__.py @@ -2,12 +2,14 @@ Firestore* claims, rows, leases, checkpoints PubSubBroker messaging, with dead letters + FirestoreOutboxStore the transactional outbox (v1.7.1) SecretManagerCredentialProvider tool credentials from Secret Manager """ from .firestore import (FirestoreBackend, FirestoreCheckpointStore, FirestoreIdempotencyStore, FirestoreLease, FirestoreTransactionalStore) +from .outbox import FirestoreOutboxStore from .pubsub import PubSubBroker from .secrets import SecretManagerCredentialProvider __all__ = ['FirestoreBackend', 'FirestoreIdempotencyStore', 'FirestoreTransactionalStore', 'FirestoreLease', - 'FirestoreCheckpointStore', 'PubSubBroker', 'SecretManagerCredentialProvider'] + 'FirestoreCheckpointStore', 'PubSubBroker', 'SecretManagerCredentialProvider', 'FirestoreOutboxStore'] diff --git a/adapters/e2a_adapters/gcp/outbox.py b/adapters/e2a_adapters/gcp/outbox.py new file mode 100644 index 0000000..f068038 --- /dev/null +++ b/adapters/e2a_adapters/gcp/outbox.py @@ -0,0 +1,86 @@ +"""The transactional outbox on Firestore (v1.7.1, design note MP-05, N1). + +A record is written in the SAME transaction as the business change: add(record, +transaction=tx) inside the application's Firestore transaction (or a WriteBatch, batch=b). +mark_published() deletes the record and advances the fence in one transaction, so a stale +sweeper cannot mark. + +Layout: + -outbox-/ a pending record + -outbox-fence/ the newest fencing token seen + +pending() orders by `created_at` (a single-field index Firestore keeps by itself) and then by +document id, so no composite index is needed. +""" +import json +import time +import uuid +from typing import Any, Callable, Dict, List, Optional + +from e2a import resolve_config +from e2a.messaging import BaseOutboxStore +from google.cloud import firestore + +from .. import config_keys # noqa: F401 +from .._lazy import Lazy + + +class FirestoreOutboxStore(BaseOutboxStore): + capabilities = frozenset({'outbox_store'}) + + def __init__(self, client, collection: str = 'e2a', name: str = 'default', + clock: Callable[[], float] = time.time) -> None: + self.client, self.name, self.clock = client, name, clock + self.records, self.fences = f'{collection}-outbox-{name}', f'{collection}-outbox-fence' + + @classmethod + def from_config(cls, config: Dict[str, Any], **kwargs) -> 'FirestoreOutboxStore': + project, database = resolve_config('gcp_project', config), resolve_config('firestore_database', config) + return cls(Lazy(lambda: firestore.Client(project=project, database=database)), + resolve_config('firestore_collection', config), resolve_config('outbox_name', config), **kwargs) + + def _new_id(self) -> str: + return f'{int(self.clock() * 1_000_000):017d}-{uuid.uuid4().hex[:12]}' + + def add(self, record, config=None, **kwargs): + """kwargs: transaction= (a firestore.Transaction) or batch= (a WriteBatch) to commit the + record with the business change; without either, the record is written on its own.""" + rid = record.get('id') or self._new_id() + doc = {'created_at': self.clock(), + 'record': json.dumps({k: record.get(k) for k in ('destination', 'body', 'attributes', 'tenant_id', + 'idempotency_key')}, + separators=(',', ':'), ensure_ascii=False)} + ref = self.client.collection(self.records).document(rid) + writer = kwargs.get('transaction') or kwargs.get('batch') + if writer is not None: + writer.create(ref, doc) + else: + ref.create(doc) + return rid + + def pending(self, limit=100, config=None, **kwargs): + query = self.client.collection(self.records).order_by('created_at').order_by('__name__').limit(max(1, int(limit))) + out: List[Dict[str, Any]] = [] + for snap in query.stream(): + out.append({'id': snap.id, **json.loads(snap.get('record'))}) + return out + + def mark_published(self, record_id, config=None, **kwargs): + token: Optional[int] = kwargs.get('fencing_token') + ref = self.client.collection(self.records).document(record_id) + fence_ref = self.client.collection(self.fences).document(self.name) + + @firestore.transactional + def mark(tx) -> bool: + # every read of a transaction comes before its writes + fence = fence_ref.get(transaction=tx) if token is not None else None + snap = ref.get(transaction=tx) + if not snap.exists: + return False + if token is not None: + if fence.exists and int(fence.get('fence')) > int(token): + return False + tx.set(fence_ref, {'fence': int(token)}) + tx.delete(ref) + return True + return mark(self.client.transaction()) diff --git a/adapters/e2a_adapters/gcp/pubsub.py b/adapters/e2a_adapters/gcp/pubsub.py index 08fab85..cfa9238 100644 --- a/adapters/e2a_adapters/gcp/pubsub.py +++ b/adapters/e2a_adapters/gcp/pubsub.py @@ -24,10 +24,10 @@ class PubSubBroker(BaseMessageBroker): capabilities = frozenset({'messaging', 'dead_letter'}) def __init__(self, project: str, publisher=None, subscriber=None, dead_letter_suffix: str = '-dlq') -> None: - from google.cloud import pubsub_v1 + # v1.7.1: the SDK is imported on first use, so a profile binds without it installed self.project, self.suffix = project, dead_letter_suffix - self.publisher = publisher or Lazy(pubsub_v1.PublisherClient) - self.subscriber = subscriber or Lazy(pubsub_v1.SubscriberClient) + self.publisher = publisher or Lazy(lambda: __import__('google.cloud.pubsub_v1', fromlist=['x']).PublisherClient()) + self.subscriber = subscriber or Lazy(lambda: __import__('google.cloud.pubsub_v1', fromlist=['x']).SubscriberClient()) @classmethod def from_config(cls, config: Dict[str, Any]) -> 'PubSubBroker': diff --git a/adapters/e2a_adapters/profiles/aws.json b/adapters/e2a_adapters/profiles/aws.json index 511ee28..f80dedf 100644 --- a/adapters/e2a_adapters/profiles/aws.json +++ b/adapters/e2a_adapters/profiles/aws.json @@ -2,7 +2,7 @@ "name": "aws", "provider": "cloud-infrastructure", "cloud": "aws", - "description": "AWS infrastructure for E2A: DynamoDB claims, rows, leases and checkpoints; SQS/SNS messaging with dead letters; Secrets Manager for tool credentials; Cognito claims at ingress. A provider profile extends it.", + "description": "AWS infrastructure for E2A: DynamoDB claims, rows, leases and checkpoints; SQS/SNS messaging with dead letters and a transactional outbox; Secrets Manager for tool credentials; Cognito claims at ingress. A provider profile extends it.", "bindings": { "idempotency_store": "e2a_adapters.aws:DynamoDBIdempotencyStore", "checkpoint_store": "e2a_adapters.aws:DynamoDBCheckpointStore", @@ -10,7 +10,8 @@ "lease": "e2a_adapters.aws:DynamoDBLease", "message_broker": "e2a_adapters.aws:SQSBroker", "credential_provider": "e2a_adapters.aws:SecretsManagerCredentialProvider", - "identity_resolver": "e2a.identity:ClaimMapIdentityResolver" + "identity_resolver": "e2a.identity:ClaimMapIdentityResolver", + "outbox_store": "e2a_adapters.aws:DynamoDBOutboxStore" }, "config": { "aws_region": "${AWS_REGION:-us-east-1}", @@ -24,7 +25,8 @@ "subject": "sub", "roles": "cognito:groups", "issuer": "iss" - } + }, + "outbox_name": "${E2A_OUTBOX_NAME:-default}" }, "features": [ "idempotency_store", @@ -34,6 +36,7 @@ "messaging", "dead_letter", "secret_store", - "identity" + "identity", + "outbox_store" ] } diff --git a/adapters/e2a_adapters/profiles/azure.json b/adapters/e2a_adapters/profiles/azure.json index 5a341d3..6ab7512 100644 --- a/adapters/e2a_adapters/profiles/azure.json +++ b/adapters/e2a_adapters/profiles/azure.json @@ -2,7 +2,7 @@ "name": "azure", "provider": "cloud-infrastructure", "cloud": "azure", - "description": "Azure infrastructure for E2A: Cosmos DB claims, rows, leases and checkpoints; Service Bus messaging with native dead letters; Key Vault for tool credentials; Entra ID claims at ingress. A provider profile extends it.", + "description": "Azure infrastructure for E2A: Cosmos DB claims, rows, leases and checkpoints; Service Bus messaging with native dead letters and a transactional outbox; Key Vault for tool credentials; Entra ID claims at ingress. A provider profile extends it.", "bindings": { "idempotency_store": "e2a_adapters.azure:CosmosIdempotencyStore", "checkpoint_store": "e2a_adapters.azure:CosmosCheckpointStore", @@ -10,7 +10,8 @@ "lease": "e2a_adapters.azure:CosmosLease", "message_broker": "e2a_adapters.azure:ServiceBusBroker", "credential_provider": "e2a_adapters.azure:KeyVaultCredentialProvider", - "identity_resolver": "e2a.identity:ClaimMapIdentityResolver" + "identity_resolver": "e2a.identity:ClaimMapIdentityResolver", + "outbox_store": "e2a_adapters.azure:CosmosOutboxStore" }, "config": { "cosmos_endpoint": "${COSMOS_ENDPOINT}", @@ -25,7 +26,8 @@ "subject": "oid", "roles": "roles", "issuer": "iss" - } + }, + "outbox_name": "${E2A_OUTBOX_NAME:-default}" }, "features": [ "idempotency_store", @@ -35,6 +37,7 @@ "messaging", "dead_letter", "secret_store", - "identity" + "identity", + "outbox_store" ] } diff --git a/adapters/e2a_adapters/profiles/gcp.json b/adapters/e2a_adapters/profiles/gcp.json index 1823081..451854c 100644 --- a/adapters/e2a_adapters/profiles/gcp.json +++ b/adapters/e2a_adapters/profiles/gcp.json @@ -2,7 +2,7 @@ "name": "gcp", "provider": "cloud-infrastructure", "cloud": "gcp", - "description": "Google Cloud infrastructure for E2A: Firestore claims, rows, leases and checkpoints; Pub/Sub messaging with dead letters; Secret Manager for tool credentials; Identity Platform claims at ingress. A provider profile extends it.", + "description": "Google Cloud infrastructure for E2A: Firestore claims, rows, leases and checkpoints; Pub/Sub messaging with dead letters and a transactional outbox; Secret Manager for tool credentials; Identity Platform claims at ingress. A provider profile extends it.", "bindings": { "idempotency_store": "e2a_adapters.gcp:FirestoreIdempotencyStore", "checkpoint_store": "e2a_adapters.gcp:FirestoreCheckpointStore", @@ -10,7 +10,8 @@ "lease": "e2a_adapters.gcp:FirestoreLease", "message_broker": "e2a_adapters.gcp:PubSubBroker", "credential_provider": "e2a_adapters.gcp:SecretManagerCredentialProvider", - "identity_resolver": "e2a.identity:ClaimMapIdentityResolver" + "identity_resolver": "e2a.identity:ClaimMapIdentityResolver", + "outbox_store": "e2a_adapters.gcp:FirestoreOutboxStore" }, "config": { "gcp_project": "${GOOGLE_CLOUD_PROJECT}", @@ -22,7 +23,8 @@ "subject": "sub", "roles": "roles", "issuer": "iss" - } + }, + "outbox_name": "${E2A_OUTBOX_NAME:-default}" }, "features": [ "idempotency_store", @@ -32,6 +34,7 @@ "messaging", "dead_letter", "secret_store", - "identity" + "identity", + "outbox_store" ] } diff --git a/adapters/pyproject.toml b/adapters/pyproject.toml index 1189cad..1687efb 100644 --- a/adapters/pyproject.toml +++ b/adapters/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "e2a-adapters" -version = "1.7.0" +version = "1.7.1" description = "E2A ports on cloud infrastructure: stores, brokers, secrets and identity on AWS, Google Cloud and Azure" readme = "README.md" license = "Apache-2.0" diff --git a/adapters/tests/test_outbox.py b/adapters/tests/test_outbox.py new file mode 100644 index 0000000..ce49d73 --- /dev/null +++ b/adapters/tests/test_outbox.py @@ -0,0 +1,109 @@ +"""The outbox stores (v1.7.1) pass E2A's outbox contract (e2a.port_contract.check_outbox_store), and a +record commits with the business change or not at all, in each database's own transaction: +DynamoDB on moto (and DynamoDB Local for the race), Firestore and Cosmos DB on their emulators.""" +import os +import sys +import uuid + +import boto3 +import pytest +from moto import mock_aws + +from e2a import BrokerOutboxSweeper, InMemoryBroker +from e2a.port_contract import check_outbox_store + +SUITE = 11 +KEY = 'C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==' # emulator's public key + + +def _table(client, name): + client.create_table(TableName=name, BillingMode='PAY_PER_REQUEST', + KeySchema=[{'AttributeName': 'pk', 'KeyType': 'HASH'}, {'AttributeName': 'sk', 'KeyType': 'RANGE'}], + AttributeDefinitions=[{'AttributeName': 'pk', 'AttributeType': 'S'}, + {'AttributeName': 'sk', 'AttributeType': 'S'}]) + + +def test_dynamodb_outbox_meets_the_contract_and_commits_with_the_business_change(): + from e2a_adapters.aws import DynamoDBOutboxStore + with mock_aws(): + client = boto3.client('dynamodb', region_name='us-east-1') + _table(client, 'state') + # moto is not atomic under threads: the race runs on DynamoDB Local below + assert len(check_outbox_store(lambda: DynamoDBOutboxStore(client, 'state', uuid.uuid4().hex[:8]), race=1)) == SUITE - 1 + outbox = DynamoDBOutboxStore(client, 'state', 'orders') + order = {'Put': {'TableName': 'state', 'Item': {'pk': {'S': 'ORDER#1'}, 'sk': {'S': 'order'}}, + 'ConditionExpression': 'attribute_not_exists(pk)'}} + record = {'destination': 'orders', 'body': {'o': 1}, 'tenant_id': 't', 'idempotency_key': 'k1'} + client.transact_write_items(TransactItems=[order, outbox.transact_item(dict(record))]) + with pytest.raises(client.exceptions.TransactionCanceledException): # the order exists: rolled back + client.transact_write_items(TransactItems=[order, outbox.transact_item({**record, 'idempotency_key': 'k2'})]) + assert [r['idempotency_key'] for r in outbox.pending()] == ['k1'] + broker = InMemoryBroker() + assert BrokerOutboxSweeper(outbox, broker).sweep({})['published'] == 1 and outbox.pending() == [] + + +@pytest.mark.skipif(not os.getenv('DYNAMODB_LOCAL_ENDPOINT'), reason='DYNAMODB_LOCAL_ENDPOINT not set') +def test_dynamodb_outbox_race_on_dynamodb_local(monkeypatch): + from e2a_adapters.aws import DynamoDBOutboxStore + for k, v in (('AWS_ACCESS_KEY_ID', 'local'), ('AWS_SECRET_ACCESS_KEY', 'local')): + monkeypatch.setenv(k, v) + client = boto3.client('dynamodb', region_name='us-east-1', endpoint_url=os.environ['DYNAMODB_LOCAL_ENDPOINT']) + name = f'outbox-{uuid.uuid4().hex[:8]}' + _table(client, name) + try: + assert len(check_outbox_store(lambda: DynamoDBOutboxStore(client, name, uuid.uuid4().hex[:8]))) == SUITE + finally: + client.delete_table(TableName=name) + + +@pytest.mark.skipif(not os.getenv('FIRESTORE_EMULATOR_HOST'), reason='FIRESTORE_EMULATOR_HOST not set') +def test_firestore_outbox_meets_the_contract_and_commits_with_the_business_change(): + from google.cloud import firestore + + from e2a_adapters.gcp import FirestoreOutboxStore + client = firestore.Client(project='demo-e2a-adapters') + assert len(check_outbox_store(lambda: FirestoreOutboxStore(client, 'e2a', uuid.uuid4().hex[:8]))) == SUITE + outbox = FirestoreOutboxStore(client, 'e2a', f'tx-{uuid.uuid4().hex[:6]}') + + @firestore.transactional + def place(tx, fail): + tx.set(client.collection('orders').document(uuid.uuid4().hex), {'state': 'PLACED'}) + outbox.add({'destination': 'orders', 'body': {'o': 1}, 'tenant_id': 't', 'idempotency_key': str(fail)}, + transaction=tx) + if fail: + raise RuntimeError('a business rule failed') + with pytest.raises(RuntimeError): + place(client.transaction(), True) + place(client.transaction(), False) + assert [r['idempotency_key'] for r in outbox.pending()] == ['False'] + + +@pytest.mark.skipif(not os.getenv('COSMOS_EMULATOR_ENDPOINT'), reason='COSMOS_EMULATOR_ENDPOINT not set') +def test_cosmos_outbox_meets_the_contract_and_commits_with_the_business_change(): + from azure.cosmos import CosmosClient, PartitionKey + from azure.cosmos.exceptions import CosmosBatchOperationError + + from e2a_adapters.azure import CosmosOutboxStore + db = CosmosClient(os.environ['COSMOS_EMULATOR_ENDPOINT'], KEY).create_database_if_not_exists('e2a-adapters-test') + c = db.create_container(f'outbox-{uuid.uuid4().hex[:8]}', PartitionKey('/pk'), default_ttl=-1) + try: + assert len(check_outbox_store(lambda: CosmosOutboxStore(c, uuid.uuid4().hex[:8]))) == SUITE + outbox = CosmosOutboxStore(c, 'orders') + record = {'destination': 'orders', 'body': {'o': 1}, 'tenant_id': 'acme', 'idempotency_key': 'k1'} + order = ('create', ({'id': 'order-1', 'pk': 'tenant-acme'},)) + c.execute_item_batch([order, outbox.batch_operation(dict(record), 'tenant-acme')], partition_key='tenant-acme') + with pytest.raises(CosmosBatchOperationError): # the order exists: rolled back + c.execute_item_batch([order, outbox.batch_operation({**record, 'idempotency_key': 'k2'}, 'tenant-acme')], + partition_key='tenant-acme') + assert [r['idempotency_key'] for r in outbox.pending()] == ['k1'] + finally: + db.delete_container(c.id) + + +def test_the_pubsub_broker_binds_without_its_sdk(monkeypatch): + # v1.7.1: a profile binds and verifies offline even where google-cloud-pubsub is not installed + from e2a_adapters.gcp import PubSubBroker + monkeypatch.setitem(sys.modules, 'google.cloud.pubsub_v1', None) # importing it now fails + broker = PubSubBroker.from_config({'gcp_project': 'p'}) + with pytest.raises(ImportError): + broker.publisher.publish diff --git a/adapters/tests/test_profiles.py b/adapters/tests/test_profiles.py index c66c1f2..3fc2bfa 100644 --- a/adapters/tests/test_profiles.py +++ b/adapters/tests/test_profiles.py @@ -26,7 +26,7 @@ def test_each_cloud_profile_binds_and_verifies_offline(cloud): b = bindings_from_profile(cloud, verify=True) config = b.config() for seam in ('idempotency_store', 'checkpoint_store', 'transactional_store', 'lease', 'message_broker', - 'credential_provider', 'identity_resolver'): + 'outbox_store', 'credential_provider', 'identity_resolver'): assert config.get(seam) is not None, seam assert config['profile_settings']['identity_claim_map'] == PRESETS[CLOUDS[cloud]] assert b.warnings == [] and b.profile.literal_secrets == [] diff --git a/java/e2a-core/pom.xml b/java/e2a-core/pom.xml index 3693f17..c1b3c14 100644 --- a/java/e2a-core/pom.xml +++ b/java/e2a-core/pom.xml @@ -5,7 +5,7 @@ com.subhamviky.e2a e2a-java - 1.7.0 + 1.7.1 e2a-core e2a-core diff --git a/java/e2a-cqrs/pom.xml b/java/e2a-cqrs/pom.xml index 188f9d8..611907d 100644 --- a/java/e2a-cqrs/pom.xml +++ b/java/e2a-cqrs/pom.xml @@ -5,7 +5,7 @@ com.subhamviky.e2a e2a-java - 1.7.0 + 1.7.1 e2a-cqrs e2a-cqrs diff --git a/java/e2a-spring-jpa/pom.xml b/java/e2a-spring-jpa/pom.xml index 9a0fe44..4d7a8eb 100644 --- a/java/e2a-spring-jpa/pom.xml +++ b/java/e2a-spring-jpa/pom.xml @@ -5,7 +5,7 @@ com.subhamviky.e2a e2a-java - 1.7.0 + 1.7.1 e2a-spring-jpa e2a-spring-jpa diff --git a/java/e2a-test/pom.xml b/java/e2a-test/pom.xml index 375d95f..070f934 100644 --- a/java/e2a-test/pom.xml +++ b/java/e2a-test/pom.xml @@ -5,7 +5,7 @@ com.subhamviky.e2a e2a-java - 1.7.0 + 1.7.1 e2a-test e2a-test diff --git a/java/pom.xml b/java/pom.xml index 0c94a8c..d2048cd 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -5,7 +5,7 @@ com.subhamviky.e2a e2a-java - 1.7.0 + 1.7.1 pom e2a-java E2A runtime contracts for Java: the v1.2 agent lifecycle, declared workflows, governance stages, diff --git a/pyproject.toml b/pyproject.toml index e9d1615..bc3495b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "e2a-framework" -version = "1.7.0" +version = "1.7.1" description = "E2A - Enterprise-to-Agentic runtime contracts (reference implementation)" readme = "README.md" license = "Apache-2.0" diff --git a/reference/e2a/__init__.py b/reference/e2a/__init__.py index 19152ef..e03e50e 100644 --- a/reference/e2a/__init__.py +++ b/reference/e2a/__init__.py @@ -31,4 +31,4 @@ RedactionStage, SandboxRefusal, SandboxStage, TokenBudgetRefusal, TokenBudgetStage, ToolRefusal, compile_redactions, redact_text) -__version__ = "1.7.0" +__version__ = "1.7.1" diff --git a/reference/e2a/config.py b/reference/e2a/config.py index 6347b23..c41c2bd 100644 --- a/reference/e2a/config.py +++ b/reference/e2a/config.py @@ -350,6 +350,17 @@ def scoped_config(config: Optional[Dict[str, Any]], agent_name: Optional[str] = return out +def is_set(key: str, config: Optional[Dict[str, Any]] = None, kwargs: Optional[Dict[str, Any]] = None) -> bool: + """v1.7.1: whether a key has a value from a keyword, the config, the environment or the + profile, rather than only its registered default. E.g. warn about dropping a parameter only + when someone chose it.""" + spec = E2A_CONFIG[key] + profile = (config or {}).get('profile_settings') or {} + return ((kwargs or {}).get(key) is not None or (config or {}).get(key) is not None + or bool(spec.env and os.getenv(spec.env) is not None) + or (isinstance(profile, dict) and profile.get(key) is not None)) + + def unknown_config_keys(config: Optional[Dict[str, Any]]) -> List[str]: """Keys in a config dict that are neither registered nor io_ prefixed — a typo guard to call at start-up.""" diff --git a/reference/e2a/llm.py b/reference/e2a/llm.py index 1ff102c..a36652b 100644 --- a/reference/e2a/llm.py +++ b/reference/e2a/llm.py @@ -24,7 +24,7 @@ from .llm_types import ModelResult from .capabilities import capabilities_of from .schema_check import schema_errors -from .config import resolve_config +from .config import is_set, resolve_config from .resilience import ResiliencePolicy from ._propagation import _PropagationMixin @@ -340,7 +340,9 @@ async def __infer( f'{type(self).__name__} declares no structured output: bind a service that does, ' f'or list "structured_output" in the profile features so verify() refuses it at startup') if temperature is not None and features.get('sampling') == 'unsupported': - note('param_dropped', param='temperature', reason='sampling unsupported by the bound model') + # v1.7.1: warn only when someone set a temperature; the registry default is not a choice + if is_set('temperature', cfg, kwargs): + note('param_dropped', param='temperature', reason='sampling unsupported by the bound model') temperature = None effort = self._resolve_param('reasoning_effort', cfg, kwargs) if effort is not None: diff --git a/reference/e2a/port_contract.py b/reference/e2a/port_contract.py index 85c224c..0836c10 100644 --- a/reference/e2a/port_contract.py +++ b/reference/e2a/port_contract.py @@ -28,7 +28,7 @@ from .core import ClaimStatus, PhysicalOutcome __all__ = ['FakeClock', 'check_idempotency_store', 'check_transactional_store', 'check_lease', - 'check_checkpoint_store', 'check_llm_service', 'check_message_broker', 'check_all'] + 'check_checkpoint_store', 'check_llm_service', 'check_message_broker', 'check_outbox_store', 'check_all'] VALUE = {'amount': '100.25', 'count': 3, 'ratio': 0.5, 'ok': True, 'none': None, 'nested': {'items': ['a', 'b'], 'unicode': 'Zürich €'}} @@ -549,15 +549,55 @@ def unknown(c, message): return r.passed +# ------------------------------------------------------------ outbox (v1.7.1) +def check_outbox_store(make: Callable[[], object], race: int = 8) -> List[str]: + """The behaviour every BaseOutboxStore adapter keeps. ``make()`` returns a FRESH, EMPTY + outbox. ``race`` threads mark one record published at once; exactly one may win.""" + r = _Run('outbox store') + outbox = make() + rid = outbox.add({'destination': 'orders', 'body': VALUE, 'attributes': {'kind': 'order.created'}, + 'tenant_id': 't1', 'idempotency_key': 'k-1'}) + [rec] = outbox.pending(10) + r.check('a record is pending once added, under the id add() returned', bool(rid) and rec['id'] == rid, str(rec)) + r.check('the record round-trips exactly', + (rec['destination'], rec['body'], rec['attributes'], rec['tenant_id'], rec['idempotency_key']) + == ('orders', VALUE, {'kind': 'order.created'}, 't1', 'k-1'), str(rec)) + r.check('mark_published removes it from pending', outbox.mark_published(rid) is True and outbox.pending(10) == []) + r.check('marking it again is refused', outbox.mark_published(rid) is False) + r.check('an unknown id is refused', outbox.mark_published('no-such-record') is False) + + outbox = make() + ids = [outbox.add({'destination': 'd', 'body': {'n': n}, 'tenant_id': 't1', 'idempotency_key': f'k{n}'}) + for n in range(3)] + r.check('pending is oldest first', [x['id'] for x in outbox.pending(10)] == ids, str(outbox.pending(10))) + r.check('pending honours the limit', [x['id'] for x in outbox.pending(2)] == ids[:2]) + + outbox = make() + a, b, c = (outbox.add({'destination': 'd', 'body': {'n': n}, 'tenant_id': 't1', 'idempotency_key': f'f{n}'}) + for n in range(3)) + r.check('a sweeper with a fencing token may mark', outbox.mark_published(a, fencing_token=5) is True) + r.check('a stale sweeper (older token) may not', + outbox.mark_published(b, fencing_token=3) is False and b in [x['id'] for x in outbox.pending(10)]) + r.check('a newer token may', outbox.mark_published(c, fencing_token=6) is True) + + if race >= 2: + outbox = make() + rid = outbox.add({'destination': 'd', 'body': {}, 'tenant_id': 't1', 'idempotency_key': 'race'}) + wins = _race(race, lambda i: outbox.mark_published(rid)) + r.check(f'{race} sweepers marking one record: exactly one wins', wins.count(True) == 1, str(wins)) + return r.passed + + def check_all(idempotency=None, transactional=None, lease=None, checkpoint=None, llm_service=None, - message_broker=None) -> dict: + message_broker=None, outbox=None) -> dict: """Run the suites for the factories given; returns {port: [passed scenario names]}.""" out = {} for name, fn, make in (('idempotency', check_idempotency_store, idempotency), ('transactional', check_transactional_store, transactional), ('lease', check_lease, lease), ('checkpoint', check_checkpoint_store, checkpoint), ('llm_service', check_llm_service, llm_service), - ('message_broker', check_message_broker, message_broker)): + ('message_broker', check_message_broker, message_broker), + ('outbox', check_outbox_store, outbox)): if make is not None: out[name] = fn(make) return out diff --git a/tests/test_v161.py b/tests/test_v161.py index f00d211..87f36f9 100644 --- a/tests/test_v161.py +++ b/tests/test_v161.py @@ -152,6 +152,9 @@ def test_features_decide_what_is_sent_not_the_vendor(): assert [e.get('param') for e in log if e.get('event') == 'param_dropped'] == ['temperature', 'reasoning_effort'] out = infer(Adapter(lambda m: 'ok'), NO_RETRY, reasoning_effort='extreme') assert out['status'] == 'FAILED' and 'reasoning_effort' in out['error'] + quiet, log = Adapter(lambda m: 'ok', features={'sampling': 'unsupported'}), [] + infer(quiet, NO_RETRY, message_log=log) # v1.7.1: the default temperature is no choice + assert quiet.seen[0]['temperature'] is None and not [e for e in log if e.get('event') == 'param_dropped'] plain = Adapter(lambda m: 'ok') infer(plain, {**NO_RETRY, 'temperature': 0.7}, reasoning_effort='low') assert plain.seen[0]['temperature'] == 0.7 and plain.seen[0]['reasoning_effort'] == 'low' diff --git a/tests/test_v170.py b/tests/test_v170.py index 508d838..c148b4e 100644 --- a/tests/test_v170.py +++ b/tests/test_v170.py @@ -162,3 +162,9 @@ def test_the_infrastructure_seams_reach_the_config(): broker = InMemoryBroker() b = bindings_from_profile(Profile(name='p', provider='x'), overrides={'message_broker': broker}) assert b.config()['message_broker'] is broker + + +def test_the_reference_outbox_meets_the_outbox_contract(): + from e2a import InMemoryOutboxStore + from e2a.port_contract import check_outbox_store + assert len(check_outbox_store(InMemoryOutboxStore)) == 11