diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4aa741e..cdec287 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -46,22 +46,40 @@ jobs: python-version: "3.12" - name: Install the core, then the adapters with every backend and the test extras run: pip install . && pip install "./adapters[aws,gcp,azure,test]" - - name: Start the emulators (DynamoDB Local, Firestore, Cosmos DB) - # The contract kit's concurrent races need atomic backends: moto's in-process DynamoDB is not one + - name: Start the emulators (DynamoDB Local, Firestore, Cosmos DB, Pub/Sub, Service Bus) + # The contract kit's concurrent races need atomic backends: moto's in-process DynamoDB is not one. + # v1.7.0: the brokers' messaging suite runs on the Pub/Sub and Service Bus emulators (SQS on moto). + # The Service Bus emulator keeps its state in SQL Edge and declares its queues up front. + env: + SQL_PASSWORD: E2a-Emulator-Pass1! # a throwaway password for the CI-only database run: | docker run -d --name ddb -p 8000:8000 amazon/dynamodb-local:latest -jar DynamoDBLocal.jar -inMemory -port 8000 docker run -d --name firestore -p 8085:8085 gcr.io/google.com/cloudsdktool/google-cloud-cli:emulators \ gcloud emulators firestore start --host-port=0.0.0.0:8085 docker run -d --name cosmos -p 8081:8081 mcr.microsoft.com/cosmosdb/linux/azure-cosmos-emulator:vnext-preview - for port in 8000 8085 8081; do + docker run -d --name pubsub -p 8086:8086 gcr.io/google.com/cloudsdktool/google-cloud-cli:emulators \ + gcloud beta emulators pubsub start --host-port=0.0.0.0:8086 --project=demo-e2a-adapters + docker run -d --name sqledge --network host -e ACCEPT_EULA=Y -e MSSQL_SA_PASSWORD="$SQL_PASSWORD" \ + mcr.microsoft.com/azure-sql-edge:latest + sleep 20 + docker run -d --name servicebus --network host -e ACCEPT_EULA=Y -e SQL_SERVER=localhost \ + -e MSSQL_SA_PASSWORD="$SQL_PASSWORD" -e SQL_WAIT_INTERVAL=5 \ + -v "$PWD/adapters/tests/servicebus-emulator.json:/ServiceBus_Emulator/ConfigFiles/Config.json" \ + mcr.microsoft.com/azure-messaging/servicebus-emulator:latest + for port in 8000 8085 8081 8086; do for _ in $(seq 90); do curl -s -o /dev/null "http://127.0.0.1:$port/" && break; sleep 2; done done for _ in $(seq 60); do docker logs cosmos 2>&1 | grep -q "Gateway=OK" && break; sleep 2; done - - name: Adapter tests (the port contract kit on every backend) + for _ in $(seq 90); do docker logs servicebus 2>&1 | grep -q "Successfully Up" && break; sleep 2; done + docker logs servicebus 2>&1 | tail -2 + - name: Adapter tests (the port contract kit and the messaging suite on every backend) env: DYNAMODB_LOCAL_ENDPOINT: http://127.0.0.1:8000 FIRESTORE_EMULATOR_HOST: 127.0.0.1:8085 COSMOS_EMULATOR_ENDPOINT: http://127.0.0.1:8081 + PUBSUB_EMULATOR_HOST: 127.0.0.1:8086 + # the emulator's documented, fixed connection string (not a secret) + SERVICEBUS_CONNECTION_STRING: "Endpoint=sb://localhost;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=SAS_KEY_VALUE;UseDevelopmentEmulator=true;" # -rs lists skips: none is expected here, every emulator is running run: python -m pytest -q -rs adapters/tests - name: Governed Entry Point rules over the adapters diff --git a/CHANGELOG.md b/CHANGELOG.md index af09349..9ab2086 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,61 @@ All notable changes to the E2A framework documents and reference code. +## v1.7.0 — Messaging, identity, secrets and the cloud profiles (design note MP-05) + +Cloud infrastructure behind E2A's ports, selected by configuration. The contracts are vendor-neutral and +live in `e2a`; the cloud adapters live in e2a-adapters 1.7.0 (no AI vendor SDK there, by test). +Additive: no signature changes. e2a-adapters, e2a-java and the pins move to `1.7.0`. + +### Added (e2a) +- **`e2a.messaging` (N1): exactly-once effects under at-least-once delivery.** + - `BaseMessageBroker` is the port: publish, receive, ack, nack, dead_letter, receive_dead_letters, ack_dead_letter. `BrokerMessage` is one shape for every broker; tenant and idempotency key travel as attributes. + - **`IdempotentConsumer`**: `consume()` claims every message in a `BaseIdempotencyStore`. + - A duplicate or a lost acknowledgement is acked without a second effect. + - A message another consumer holds is deferred. + - A deterministic error is dead-lettered (`FAILED_FINAL`). + - A transient error is retried. + - An `UNKNOWN` outcome is dead-lettered, never retried blindly. + - Too many deliveries (`consumer_max_deliveries`) dead-letters the message. + - **`reconcile()`** (inherited from `BaseReconciliationService`) works the dead letters. It confirms physical truth through `_confirm` before any replay, and settles a dead letter only when its effect is confirmed or replayed. + - **The outbox:** a `BaseOutboxStore` port with `InMemoryOutboxStore`, and `BrokerOutboxSweeper`. The sweeper publishes only records a consumer can deduplicate, and marks them published under the lease's fencing token. + - `InMemoryBroker` is the reference broker. +- **`e2a.port_contract.check_message_broker`:** 25 scenarios every broker adapter passes, including: + - round trip, ack, nack and redelivery count, dead letters; + - a duplicate delivery, a lost acknowledgement, a poison message; + - an UNKNOWN outcome reconciled both ways; + - the outbox. +- **`e2a.identity` (N3):** `BaseIdentityResolver` / `ClaimMapIdentityResolver.resolve(claims, verified, request_tenant)` gives the tenant, subject and roles from claims the gateway verified, through `identity_claim_map`. + - It raises `IdentityRefused` (a `GovernanceRefusal`) for: unverified claims, an untrusted issuer (`identity_trusted_issuers`), no tenant, or a request naming another tenant. + - `identity_tenant_map` maps an external directory id to a tenant. +- **The literal-secret rule (N2):** a profile that sets a secret key to a literal value instead of `${VAR}` is recorded (`Profile.literal_secrets`, inherited through `extends`). `Bindings.verify()` warns, and `BasePipeline` fails the release. +- **Seams:** `message_broker`, `outbox_store`, `identity_resolver`, `transactional_store`, `lease`. The ones E2A reads are carried in `Bindings.config()`. +- **Capabilities:** `messaging`, `dead_letter`, `outbox_store`, `identity`, `secret_store`. +- **`BaseReconciliationService._settle_dlq_entry(entry, outcome)`:** a hook with a no-op default, called with each entry's decision. +- **Keys:** `consumer_batch_size`, `consumer_wait_seconds`, `consumer_max_deliveries`, `consumer_retry_delay_seconds`, `identity_claim_map`, `identity_trusted_issuers`, `identity_tenant_map`. + +### Added (e2a-adapters 1.7.0) +- **Brokers:** `SQSBroker` (SQS queues, SNS topics, `-dlq`), `PubSubBroker` (topics and subscriptions, `-dlq`) and `ServiceBusBroker` (queues and topic subscriptions, the native dead-letter subqueue). + - All three pass `check_message_broker`: SQS on moto, Pub/Sub and Service Bus on their emulators. +- **Secret stores (N2):** `SecretsManagerCredentialProvider`, `SecretManagerCredentialProvider` and `KeyVaultCredentialProvider`. Each gives tool calls their auth headers from `secret_headers` (longest target prefix, optional `json_key`), cached for `secret_cache_seconds` so rotation needs no restart. +- **`from_config(config)` on every store and broker,** from registered keys with environment variables. Clients are created on first use, so a profile binds and verifies offline. +- **The cloud profiles** `aws`, `gcp` and `azure` (shipped in the package): stores, broker, secret store and the identity claim map of their cloud. Provider profiles extend them. +- **`e2a_adapters.identity.PRESETS`:** Cognito, Entra ID and Identity Platform. +- **Capabilities** declared on every store. +- **Extras:** `[gcp]` adds Pub/Sub and Secret Manager; `[azure]` adds Service Bus, Key Vault and azure-identity. + +### Changed +- **`e2a.bindings` finds profiles shipped in e2a-adapters** (`e2a_adapters/profiles/`). `available_profiles()` now lists `aws`, `gcp` and `azure` when it is installed. +- **CI:** the adapters job also runs the Pub/Sub emulator, and the Service Bus emulator with SQL Edge. + +### Not in this release +- **Durable outbox stores** (DynamoDB, Firestore, Cosmos DB) and **the projects on the brokers:** + - reconciliation's SQS consumer on `IdempotentConsumer`; + - O2C's outbox on Pub/Sub; + - audit-ai on Service Bus. + + These are the next step. + ## v1.6.1 — Vendor-neutral handling for managed providers (design note MP-04) What a managed deployment needs from E2A, selected by configuration, capabilities and the diff --git a/README.md b/README.md index 512c0e2..1a52cb0 100644 --- a/README.md +++ b/README.md @@ -12,15 +12,16 @@ It focuses on the boundary where probabilistic reasoning meets deterministic ent ## Install - pip install "e2a-framework @ git+https://github.com/subhamviky/e2a-framework.git@v1.6.1" + pip install "e2a-framework @ git+https://github.com/subhamviky/e2a-framework.git@v1.7.0" -Poetry: `e2a-framework = { git = "https://github.com/subhamviky/e2a-framework.git", tag = "v1.6.1" }`. +Poetry: `e2a-framework = { git = "https://github.com/subhamviky/e2a-framework.git", tag = "v1.7.0" }`. The framework is the `e2a` package (`from e2a import BaseAgent, BaseWorkflow, Steps, Step`); the release gate is `from e2a.release import ProjectReleaseGate`; the `e2a_base` module re-exports the same names. Since v1.3 hooks use the v1.2 signature (`state, ctx, …`). **Production adapters (e2a-adapters, since v1.4.0):** DynamoDB (v1.4.0), Firestore and Cosmos DB (v1.5.0) -implementations of the idempotency, transactional, lease and checkpoint ports in [`adapters/`](adapters/README.md), -a separate distribution at this tag (`pip install "e2a-adapters[aws,gcp,azure] @ git+https://github.com/subhamviky/e2a-framework.git@v1.6.1#subdirectory=adapters"`; +implementations of the idempotency, transactional, lease and checkpoint ports; since v1.7.0 also the brokers +(SQS/SNS, Pub/Sub, Service Bus), the secret stores and the cloud profiles `aws`, `gcp`, `azure`, in [`adapters/`](adapters/README.md), +a separate distribution at this tag (`pip install "e2a-adapters[aws,gcp,azure] @ git+https://github.com/subhamviky/e2a-framework.git@v1.7.0#subdirectory=adapters"`; pick the extras you need). Every adapter passes the port contract kit, `e2a.port_contract`. @@ -32,6 +33,7 @@ registry: a service's CI builds it from the tag (`mvn -f java/pom.xml install -D | e2a-framework | a2c-framework | capability-oriented-dsa | |---|---|---| +| `v1.7.0` (messaging with exactly-once effects, identity at ingress, the literal-secret rule; e2a-adapters `1.7.0`: brokers, secret stores, cloud profiles; e2a-java `1.7.0`) | `v1.4.0` | `v0.2.x`, `v0.3.x` | | `v1.6.1` (vendor-neutral handling for managed providers: model routes, structured-output checks, scopes, telemetry and cost, adaptive admission, delegated runtimes, reranker port, classifier guardrails, pipeline and MCP checks; the neutrality test) | `v1.3.0` | `v0.2.x`, `v0.3.x` | | `v1.6.0` (contracts for managed providers: canonical types, typed refusals, bindings and profiles, capabilities, credentials, `check_llm_service`) | `v1.3.0` | `v0.2.x`, `v0.3.x` | | `v1.5.1` (model-call retries, provider metadata kept, rerank query; e2a-adapters and e2a-java `1.5.1`) | `v1.3.0` | `v0.2.x`, `v0.3.x` | diff --git a/adapters/README.md b/adapters/README.md index 8a8aace..8275500 100644 --- a/adapters/README.md +++ b/adapters/README.md @@ -1,13 +1,48 @@ # e2a-adapters -Production adapters for the E2A foundation ports (design note D-02). Each one passes E2A's -port contract kit (`e2a.port_contract`), the same suite as the in-memory reference adapters. +E2A's ports on the clouds' infrastructure (design notes D-02, D-03, MP-05). Each adapter passes +E2A's port contract kit (`e2a.port_contract`), the same suite as the in-memory reference adapters; +each broker passes its messaging suite. No AI vendor SDK is imported here: AI services live in +the provider layer (`e2a-managed-providers`). + +## Cloud profiles (since 1.7.0) + +An application chooses its cloud by naming a profile; every adapter is built from configuration +(`from_config`), so nothing is constructed by hand: + +```python +from e2a import bindings_from_profile +bindings = bindings_from_profile("aws", verify=True) # or "gcp", "azure"; or E2A_PROVIDER_PROFILE +config = bindings.config() # idempotency_store, checkpoint_store, transactional_store, lease, + # message_broker, credential_provider, identity_resolver +``` + +| Profile | Stores | Messaging | Tool credentials | Identity claims | +|---|---|---|---|---| +| `aws` | DynamoDB | SQS / SNS (`SQSBroker`) | Secrets Manager | Cognito | +| `gcp` | Firestore | Pub/Sub (`PubSubBroker`) | Secret Manager | Identity Platform | +| `azure` | Cosmos DB | Service Bus (`ServiceBusBroker`) | Key Vault | Entra ID | + +Every setting is a registered key with an environment variable (`AWS_REGION`, `DYNAMODB_IDEMPOTENCY_TABLE`, +`GOOGLE_CLOUD_PROJECT`, `COSMOS_ENDPOINT`, `SERVICEBUS_NAMESPACE`, `KEYVAULT_URL`, ...; `python -c +"from e2a import config_reference_markdown; import e2a_adapters; print(config_reference_markdown())"`). +Credentials come from the platform identity; the connection string and account key keys exist for +the emulators. A provider profile in `e2a-managed-providers` extends the profile of its cloud. + +Clients are created on first use, so a profile binds and verifies without credentials or network. + +## Adapters | Backend | Install | Adapters | |---|---|---| | AWS DynamoDB | `e2a-adapters[aws]` | `DynamoDBIdempotencyStore`, `DynamoDBTransactionalStore`, `DynamoDBLease`, `DynamoDBCheckpointStore` | | GCP Firestore (since 1.5.0) | `e2a-adapters[gcp]` | `FirestoreIdempotencyStore`, `FirestoreTransactionalStore`, `FirestoreLease`, `FirestoreCheckpointStore` | | Azure Cosmos DB, NoSQL API (since 1.5.0) | `e2a-adapters[azure]` | `CosmosIdempotencyStore`, `CosmosTransactionalStore`, `CosmosLease`, `CosmosCheckpointStore` | +| Amazon SQS / SNS (since 1.7.0) | `e2a-adapters[aws]` | `SQSBroker`: queues, SNS topics, a `-dlq` dead-letter queue | +| Google Pub/Sub (since 1.7.0) | `e2a-adapters[gcp]` | `PubSubBroker`: topics and subscriptions, a `-dlq` topic and subscription | +| Azure Service Bus (since 1.7.0) | `e2a-adapters[azure]` | `ServiceBusBroker`: queues, topic subscriptions, the native dead-letter subqueue | +| Secret stores (since 1.7.0) | per cloud | `SecretsManagerCredentialProvider`, `SecretManagerCredentialProvider`, `KeyVaultCredentialProvider`: tool auth headers from `secret_headers` | +| Identity presets (since 1.7.0) | – | `e2a_adapters.identity.PRESETS`: the claim maps of Cognito, Entra ID and Identity Platform | Firestore and Cosmos DB share one implementation of the ports, `e2a_adapters.cas`. It works over any document store that can create-if-absent and replace-if-unchanged (an update time, an ETag), and each database supplies only a small `CasBackend`. PostgreSQL would be a third backend. @@ -16,15 +51,15 @@ Firestore and Cosmos DB share one implementation of the ports, `e2a_adapters.cas The version follows E2A. Install e2a-framework from its tag first, as every project already does: ```bash -pip install "e2a-framework @ git+https://github.com/subhamviky/e2a-framework.git@v1.6.0" \ - "e2a-adapters[aws] @ git+https://github.com/subhamviky/e2a-framework.git@v1.6.0#subdirectory=adapters" +pip install "e2a-framework @ git+https://github.com/subhamviky/e2a-framework.git@v1.7.0" \ + "e2a-adapters[aws] @ git+https://github.com/subhamviky/e2a-framework.git@v1.7.0#subdirectory=adapters" ``` Poetry: ```toml -"e2a-framework" = { git = "https://github.com/subhamviky/e2a-framework.git", tag = "v1.6.0" } -"e2a-adapters" = { git = "https://github.com/subhamviky/e2a-framework.git", tag = "v1.6.0", subdirectory = "adapters", extras = ["aws"] } +"e2a-framework" = { git = "https://github.com/subhamviky/e2a-framework.git", tag = "v1.7.0" } +"e2a-adapters" = { git = "https://github.com/subhamviky/e2a-framework.git", tag = "v1.7.0", subdirectory = "adapters", extras = ["aws"] } ``` ## DynamoDB diff --git a/adapters/e2a_adapters/__init__.py b/adapters/e2a_adapters/__init__.py index f04f353..af494af 100644 --- a/adapters/e2a_adapters/__init__.py +++ b/adapters/e2a_adapters/__init__.py @@ -1,14 +1,18 @@ -"""e2a-adapters: production adapters for E2A's foundation ports (design note D-02). +"""e2a-adapters: E2A's ports on the clouds' infrastructure (design notes D-02, D-03, MP-05). -Each backend is an optional extra and a subpackage, so a project installs only what it uses: +Each cloud is an optional extra and a subpackage, so a project installs only what it uses: - e2a_adapters.aws DynamoDB (pip install "e2a-adapters[aws]") - e2a_adapters.gcp Firestore (pip install "e2a-adapters[gcp]") - e2a_adapters.azure Cosmos DB (pip install "e2a-adapters[azure]") - e2a_adapters.cas the ports over any document store with optimistic concurrency, which - Firestore and Cosmos DB use (and a new backend can too) + e2a_adapters.aws DynamoDB, SQS/SNS, Secrets Manager (pip install "e2a-adapters[aws]") + e2a_adapters.gcp Firestore, Pub/Sub, Secret Manager (pip install "e2a-adapters[gcp]") + e2a_adapters.azure Cosmos DB, Service Bus, Key Vault (pip install "e2a-adapters[azure]") + e2a_adapters.cas the ports over any document store with optimistic concurrency + e2a_adapters.identity claim maps of Cognito, Entra ID and Identity Platform -Every adapter passes E2A's port contract kit (e2a.port_contract), the same suite the in-memory -reference adapters pass. +v1.7.0: every adapter has from_config(config), and the cloud profiles `aws`, `gcp` and `azure` +bind them (bindings_from_profile("aws")); a provider profile extends one. Every adapter passes +E2A's port contract kit (e2a.port_contract), the brokers its messaging suite, on emulators. +No AI vendor SDK is imported here (tests/test_boundary.py). """ -__version__ = '1.6.0' +from . import config_keys # noqa: F401 (registers the cloud keys) + +__version__ = '1.7.0' diff --git a/adapters/e2a_adapters/_lazy.py b/adapters/e2a_adapters/_lazy.py new file mode 100644 index 0000000..1ddcb45 --- /dev/null +++ b/adapters/e2a_adapters/_lazy.py @@ -0,0 +1,22 @@ +"""A client built on first use, so from_config() never opens a connection. A profile can then be +bound and verified (bindings.verify()) without cloud credentials or network, and the client is +created where the adapter first talks to its service.""" +import threading +from typing import Any, Callable + + +class Lazy: + def __init__(self, factory: Callable[[], Any]) -> None: + object.__setattr__(self, '_factory', factory) + object.__setattr__(self, '_target', None) + object.__setattr__(self, '_lock', threading.Lock()) + + def _get(self) -> Any: + if self._target is None: + with self._lock: + if self._target is None: + object.__setattr__(self, '_target', self._factory()) + return self._target + + def __getattr__(self, name: str) -> Any: + return getattr(self._get(), name) diff --git a/adapters/e2a_adapters/aws/__init__.py b/adapters/e2a_adapters/aws/__init__.py index 6920bf0..8dfb2a8 100644 --- a/adapters/e2a_adapters/aws/__init__.py +++ b/adapters/e2a_adapters/aws/__init__.py @@ -1,6 +1,13 @@ -"""AWS adapters for E2A's foundation ports (design note D-02). Needs boto3: install e2a-adapters[aws].""" +"""AWS adapters for E2A's ports (design notes D-02, MP-05). Needs boto3: install e2a-adapters[aws]. + + DynamoDB* claims, rows, leases, checkpoints + SQSBroker messaging (SQS queues, SNS topics), with dead letters + SecretsManagerCredentialProvider tool credentials from Secrets Manager +""" from .dynamodb import (MAX_VALUE_BYTES, DynamoDBCheckpointStore, DynamoDBIdempotencyStore, DynamoDBLease, DynamoDBTransactionalStore, is_condition_failure) +from .secrets import SecretsManagerCredentialProvider +from .sqs import SQSBroker __all__ = ['DynamoDBIdempotencyStore', 'DynamoDBTransactionalStore', 'DynamoDBLease', 'DynamoDBCheckpointStore', - 'MAX_VALUE_BYTES', 'is_condition_failure'] + 'MAX_VALUE_BYTES', 'is_condition_failure', 'SQSBroker', 'SecretsManagerCredentialProvider'] diff --git a/adapters/e2a_adapters/aws/dynamodb.py b/adapters/e2a_adapters/aws/dynamodb.py index 96a5ad8..b797398 100644 --- a/adapters/e2a_adapters/aws/dynamodb.py +++ b/adapters/e2a_adapters/aws/dynamodb.py @@ -85,6 +85,21 @@ def __init__(self, table=None, *, client=None, table_name: str = None, prefix: s self.client, self.table_name, self.prefix = client, table_name, prefix self.pk, self.sk, self.ttl_attribute, self.clock = partition_key, sort_key, ttl_attribute, clock + _table_key = 'dynamodb_state_table' # the config key naming this adapter's table (v1.7.0) + + @classmethod + def from_config(cls, config: Dict[str, Any], **kwargs): + """v1.7.0: the adapter from configuration (a cloud profile): region, endpoint and table + name are registered keys. The client is created on first use, so binding and verifying + a profile needs no credentials.""" + import boto3 + from e2a import resolve_config + from .. import config_keys # noqa: F401 + from .._lazy import Lazy + region, endpoint = resolve_config('aws_region', config), resolve_config('aws_endpoint_url', config) + client = Lazy(lambda: boto3.client('dynamodb', region_name=region, endpoint_url=endpoint)) + return cls(client=client, table_name=resolve_config(cls._table_key, config), **kwargs) + def _key(self, kind: str, *parts: str) -> Dict[str, Dict[str, str]]: return {self.pk: _s(self.prefix + kind + '#' + '#'.join(str(p) for p in parts)), self.sk: _s(kind.lower())} @@ -107,6 +122,8 @@ def _update(self, key, update: str, condition: str, names: Dict, values: Dict, * class DynamoDBIdempotencyStore(_DynamoDB, BaseIdempotencyStore): """The claim state machine of BaseIdempotencyStore on one item per (tenant, key): ABSENT -> IN_PROGRESS(owner, lease_until) -> SUCCEEDED | FAILED_RETRYABLE | FAILED_FINAL | UNKNOWN.""" + capabilities = frozenset({'idempotency_store'}) + _table_key = 'dynamodb_idempotency_table' def _k(self, idempotency_key: str, kw: Dict): return self._key('IDEM', kw.get('tenant_id') or '-', idempotency_key) @@ -198,6 +215,7 @@ def state_of(self, key, tenant_id=None) -> Optional[str]: class DynamoDBTransactionalStore(_DynamoDB, BaseTransactionalStore): """Versioned JSON rows; put_if_version() is one conditional update. A fencing token is recorded per row and a write carrying an older one is FENCED.""" + capabilities = frozenset({'transactional_store'}) def _k(self, key: str, kw: Dict): return self._key('ROW', kw.get('table_or_collection_name') or 'default', key) @@ -252,6 +270,7 @@ def put_if_version(self, key, value, expected_version, config=None, **kw) -> Com class DynamoDBLease(_DynamoDB, BaseDistributedLock): """One item per lock key. Release clears the owner but keeps the item, so the fencing token (ADD fence 1 on every acquire) is strictly increasing for the key's whole life.""" + capabilities = frozenset({'lease'}) def _k(self, lock_key: str): return self._key('LEASE', lock_key) @@ -286,6 +305,8 @@ def release(self, handle, config=None, **kw) -> None: class DynamoDBCheckpointStore(_DynamoDB, BaseCheckpointStore): """A paused workflow's checkpoint as JSON, with its status as a separate attribute so that decide() is one conditional update (PENDING -> APPROVED | REJECTED).""" + capabilities = frozenset({'checkpoint_store'}) + _table_key = 'dynamodb_checkpoint_table' def __init__(self, *args, ttl_seconds: Optional[int] = None, **kwargs): super().__init__(*args, **kwargs) diff --git a/adapters/e2a_adapters/aws/secrets.py b/adapters/e2a_adapters/aws/secrets.py new file mode 100644 index 0000000..4bcb7b7 --- /dev/null +++ b/adapters/e2a_adapters/aws/secrets.py @@ -0,0 +1,16 @@ +"""AWS Secrets Manager as the source of tool credentials (v1.7.0, N2), through the platform's role.""" +from typing import Any, Dict + +from e2a import resolve_config + +from ..secrets import SecretHeaderProvider + + +class SecretsManagerCredentialProvider(SecretHeaderProvider): + def _build_client(self, config: Dict[str, Any]) -> Any: + import boto3 + return boto3.client('secretsmanager', region_name=resolve_config('aws_region', config), + endpoint_url=resolve_config('aws_endpoint_url', config)) + + def _fetch(self, secret: str) -> str: + return self.client.get_secret_value(SecretId=secret)['SecretString'] diff --git a/adapters/e2a_adapters/aws/sqs.py b/adapters/e2a_adapters/aws/sqs.py new file mode 100644 index 0000000..24cb31c --- /dev/null +++ b/adapters/e2a_adapters/aws/sqs.py @@ -0,0 +1,92 @@ +"""Amazon SQS (and SNS) as E2A's BaseMessageBroker (v1.7.0, design note MP-05, N1). + + destination a queue name, or an SNS topic ARN (fan-out to queues subscribed with raw + message delivery, so attributes arrive as SQS message attributes) + source a queue name + dead letters the queue (default "-dlq"). A redrive policy on + the source moves a message there by itself after maxReceiveCount; dead_letter() + moves it at once, with the reason, when E2A knows it can never succeed. + +Visibility is the lock: a received message is hidden for the queue's visibility timeout; +nack() makes it visible again after the delay. delivery_count is ApproximateReceiveCount. +""" +import json +from typing import Any, Dict, List, Optional + +from e2a import BrokerMessage, resolve_config +from e2a.messaging import BaseMessageBroker + +from .. import config_keys # noqa: F401 +from .._lazy import Lazy + + +class SQSBroker(BaseMessageBroker): + capabilities = frozenset({'messaging', 'dead_letter'}) + + def __init__(self, client=None, sns_client=None, dead_letter_suffix: str = '-dlq') -> None: + if client is None: + raise ValueError('SQSBroker needs a boto3 SQS client (or use SQSBroker.from_config)') + self.client, self.sns, self.suffix = client, sns_client, dead_letter_suffix + self._urls: Dict[str, str] = {} + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> 'SQSBroker': + import boto3 + region, endpoint = resolve_config('aws_region', config), resolve_config('aws_endpoint_url', config) + return cls(Lazy(lambda: boto3.client('sqs', region_name=region, endpoint_url=endpoint)), + Lazy(lambda: boto3.client('sns', region_name=region, endpoint_url=endpoint)), + resolve_config('sqs_dead_letter_suffix', config)) + + def _url(self, name: str) -> str: + if name not in self._urls: + self._urls[name] = self.client.get_queue_url(QueueName=name)['QueueUrl'] + return self._urls[name] + + @staticmethod + def _attrs(attributes: Optional[Dict[str, str]]) -> Dict[str, Dict[str, str]]: + return {str(k): {'DataType': 'String', 'StringValue': str(v)} for k, v in (attributes or {}).items()} + + def publish(self, destination, body, attributes=None, config=None): + text = json.dumps(body, ensure_ascii=False, separators=(',', ':')) + if destination.startswith('arn:aws:sns:'): + if self.sns is None: + raise ValueError('publishing to an SNS topic needs sns_client') + return self.sns.publish(TopicArn=destination, Message=text, + MessageAttributes=self._attrs(attributes))['MessageId'] + return self.client.send_message(QueueUrl=self._url(destination), MessageBody=text, + MessageAttributes=self._attrs(attributes))['MessageId'] + + def _receive(self, queue: str, max_messages: int, wait_seconds: float) -> List[BrokerMessage]: + resp = self.client.receive_message(QueueUrl=self._url(queue), MaxNumberOfMessages=max(1, min(int(max_messages), 10)), + WaitTimeSeconds=max(0, min(int(wait_seconds), 20)), + MessageAttributeNames=['All'], AttributeNames=['ApproximateReceiveCount']) + out = [] + for m in resp.get('Messages', []): + attrs = {k: v.get('StringValue') for k, v in (m.get('MessageAttributes') or {}).items()} + out.append(BrokerMessage(m['MessageId'], json.loads(m['Body']), attrs, + int((m.get('Attributes') or {}).get('ApproximateReceiveCount', 0)) or None, + queue, m['ReceiptHandle'])) + return out + + def receive(self, source, max_messages=10, wait_seconds=0.0, config=None): + return self._receive(source, max_messages, wait_seconds) + + def ack(self, source, message, config=None): + self.client.delete_message(QueueUrl=self._url(source), ReceiptHandle=message.receipt) + + def nack(self, source, message, delay_seconds=0.0, config=None): + self.client.change_message_visibility(QueueUrl=self._url(source), ReceiptHandle=message.receipt, + VisibilityTimeout=max(0, min(int(delay_seconds), 43200))) + + def dead_letter(self, source, message, reason, config=None): + attrs = {**message.attributes, 'dead_letter_reason': str(reason)[:1000], 'dead_letter_source': source} + self.client.send_message(QueueUrl=self._url(source + self.suffix), + MessageBody=json.dumps(message.body, ensure_ascii=False, separators=(',', ':')), + MessageAttributes=self._attrs(attrs)) + self.ack(source, message) # only after the dead letter is safely stored + + def receive_dead_letters(self, source, max_messages=10, config=None): + return self._receive(source + self.suffix, max_messages, 0) + + def ack_dead_letter(self, source, message, config=None): + self.client.delete_message(QueueUrl=self._url(source + self.suffix), ReceiptHandle=message.receipt) diff --git a/adapters/e2a_adapters/azure/__init__.py b/adapters/e2a_adapters/azure/__init__.py index 0009585..08bd67f 100644 --- a/adapters/e2a_adapters/azure/__init__.py +++ b/adapters/e2a_adapters/azure/__init__.py @@ -1,7 +1,13 @@ -"""Azure adapters for E2A's foundation ports (design note D-03). Needs azure-cosmos: -install e2a-adapters[azure].""" +"""Azure adapters for E2A's ports (design notes D-03, MP-05). Install e2a-adapters[azure]. + + Cosmos* claims, rows, leases, checkpoints + ServiceBusBroker messaging, with native dead letters + KeyVaultCredentialProvider tool credentials from Key Vault +""" from .cosmos import (CosmosBackend, CosmosCheckpointStore, CosmosIdempotencyStore, CosmosLease, CosmosTransactionalStore) +from .keyvault import KeyVaultCredentialProvider +from .servicebus import ServiceBusBroker __all__ = ['CosmosBackend', 'CosmosIdempotencyStore', 'CosmosTransactionalStore', 'CosmosLease', - 'CosmosCheckpointStore'] + 'CosmosCheckpointStore', 'ServiceBusBroker', 'KeyVaultCredentialProvider'] diff --git a/adapters/e2a_adapters/azure/cosmos.py b/adapters/e2a_adapters/azure/cosmos.py index 9349888..98be000 100644 --- a/adapters/e2a_adapters/azure/cosmos.py +++ b/adapters/e2a_adapters/azure/cosmos.py @@ -85,22 +85,49 @@ def delete(self, key): return False -class CosmosIdempotencyStore(CasIdempotencyStore): +def _container_from(config): + """v1.7.0: the Cosmos DB container from configuration, created on first use: an account key + when one is set (the emulator), else Entra ID through the platform identity.""" + from e2a import resolve_config + from .. import config_keys # noqa: F401 + from .._lazy import Lazy + endpoint, key = resolve_config('cosmos_endpoint', config), resolve_config('cosmos_key', config) + database, container = resolve_config('cosmos_database', config), resolve_config('cosmos_container', config) + + def build(): + from azure.cosmos import CosmosClient + if key: + credential = key + else: + from azure.identity import DefaultAzureCredential + credential = DefaultAzureCredential() + return CosmosClient(endpoint, credential=credential).get_database_client(database) \ + .get_container_client(container) + return Lazy(build) + + +class _FromConfig: + @classmethod + def from_config(cls, config, **kwargs): + return cls(_container_from(config), **kwargs) + + +class CosmosIdempotencyStore(_FromConfig, CasIdempotencyStore): def __init__(self, container, partition_key_field: str = 'pk', clock: Callable[[], float] = time.time): super().__init__(CosmosBackend(container, partition_key_field), clock) -class CosmosTransactionalStore(CasTransactionalStore): +class CosmosTransactionalStore(_FromConfig, CasTransactionalStore): def __init__(self, container, partition_key_field: str = 'pk', clock: Callable[[], float] = time.time): super().__init__(CosmosBackend(container, partition_key_field), clock) -class CosmosLease(CasLease): +class CosmosLease(_FromConfig, CasLease): def __init__(self, container, partition_key_field: str = 'pk', clock: Callable[[], float] = time.time): super().__init__(CosmosBackend(container, partition_key_field), clock) -class CosmosCheckpointStore(CasCheckpointStore): +class CosmosCheckpointStore(_FromConfig, CasCheckpointStore): def __init__(self, container, partition_key_field: str = 'pk', clock: Callable[[], float] = time.time, ttl_seconds: Optional[int] = None): super().__init__(CosmosBackend(container, partition_key_field), clock, ttl_seconds) diff --git a/adapters/e2a_adapters/azure/keyvault.py b/adapters/e2a_adapters/azure/keyvault.py new file mode 100644 index 0000000..c213c47 --- /dev/null +++ b/adapters/e2a_adapters/azure/keyvault.py @@ -0,0 +1,16 @@ +"""Azure Key Vault as the source of tool credentials (v1.7.0, N2), through the managed identity.""" +from typing import Any, Dict + +from e2a import resolve_config + +from ..secrets import SecretHeaderProvider + + +class KeyVaultCredentialProvider(SecretHeaderProvider): + def _build_client(self, config: Dict[str, Any]) -> Any: + from azure.identity import DefaultAzureCredential + from azure.keyvault.secrets import SecretClient + return SecretClient(vault_url=resolve_config('keyvault_url', config), credential=DefaultAzureCredential()) + + def _fetch(self, secret: str) -> str: + return self.client.get_secret(secret).value diff --git a/adapters/e2a_adapters/azure/servicebus.py b/adapters/e2a_adapters/azure/servicebus.py new file mode 100644 index 0000000..57a969f --- /dev/null +++ b/adapters/e2a_adapters/azure/servicebus.py @@ -0,0 +1,140 @@ +"""Azure Service Bus as E2A's BaseMessageBroker (v1.7.0, design note MP-05, N1). + + destination a queue name, or "topic/" + source a queue name, or "topic//subscriptions/" + dead letters the entity's own dead-letter subqueue (native): dead_letter() records the reason + there, and Service Bus moves a message there by itself after MaxDeliveryCount. + +Peek-lock is the lock: a received message is settled (complete, abandon, dead-letter) on the +receiver that received it, so receivers are kept per source. nack() abandons: Service Bus +redelivers at once (it has no per-message delay; a delay needs a scheduled re-send). +delivery_count is Service Bus's own count. + +Each sender and receiver is its own AMQP connection: a sender is opened per publish and +closed, and at most `max_receivers` receivers stay open (the least recently used is closed; +its unsettled messages are released and redelivered, which at-least-once delivery allows). + +Credentials: a connection string when one is configured (the emulator), else Entra ID through +the platform identity (servicebus_namespace). +""" +import json +import threading +import uuid +from collections import OrderedDict +from typing import Any, Dict, List, Tuple + +from e2a import BrokerMessage, resolve_config +from e2a.messaging import BaseMessageBroker + +from .. import config_keys # noqa: F401 + + +def _text(value: Any) -> str: + return value.decode('utf-8') if isinstance(value, (bytes, bytearray)) else str(value) + + +class ServiceBusBroker(BaseMessageBroker): + capabilities = frozenset({'messaging', 'dead_letter'}) + + def __init__(self, client=None, connection_string: str = None, namespace: str = None, + max_receivers: int = 4) -> None: + self._client, self._conn, self._namespace = client, connection_string, namespace + self._receivers: 'OrderedDict[Tuple[str, bool], Any]' = OrderedDict() + self.max_receivers = max(1, int(max_receivers)) + self._lock = threading.RLock() + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> 'ServiceBusBroker': + return cls(connection_string=resolve_config('servicebus_connection_string', config), + namespace=resolve_config('servicebus_namespace', config)) + + @property + def client(self): + if self._client is None: + if not (self._conn or self._namespace): + raise ValueError('ServiceBusBroker needs servicebus_connection_string or servicebus_namespace') + from azure.servicebus import ServiceBusClient + if self._conn: + self._client = ServiceBusClient.from_connection_string(self._conn) + else: + from azure.identity import DefaultAzureCredential + self._client = ServiceBusClient(self._namespace, credential=DefaultAzureCredential()) + return self._client + + def _sender(self, destination: str): + if destination.startswith('topic/'): + return self.client.get_topic_sender(topic_name=destination[len('topic/'):]) + return self.client.get_queue_sender(queue_name=destination) + + def _receiver(self, source: str, dead: bool = False): + from azure.servicebus import ServiceBusSubQueue + key = (source, dead) + with self._lock: + if key in self._receivers: + self._receivers.move_to_end(key) + else: + while len(self._receivers) >= self.max_receivers: + _, oldest = self._receivers.popitem(last=False) + oldest.close() + sub_queue = ServiceBusSubQueue.DEAD_LETTER if dead else None + if source.startswith('topic/'): + _, topic, _, subscription = source.split('/', 3) + self._receivers[key] = self.client.get_subscription_receiver( + topic_name=topic, subscription_name=subscription, sub_queue=sub_queue) + else: + self._receivers[key] = self.client.get_queue_receiver(queue_name=source, sub_queue=sub_queue) + return self._receivers[key] + + def publish(self, destination, body, attributes=None, config=None): + from azure.servicebus import ServiceBusMessage + mid = str(uuid.uuid4()) + message = ServiceBusMessage(json.dumps(body, ensure_ascii=False, separators=(',', ':')), message_id=mid, + content_type='application/json', + application_properties={str(k): str(v) for k, v in (attributes or {}).items()}) + with self._sender(destination) as sender: + sender.send_messages(message) + return mid + + def _convert(self, m, source: str) -> BrokerMessage: + body = b''.join(bytes(part) for part in m.body) if not isinstance(m.body, (bytes, str)) else m.body + attrs = {_text(k): _text(v) for k, v in (m.application_properties or {}).items()} + if m.dead_letter_reason and 'dead_letter_reason' not in attrs: + attrs['dead_letter_reason'] = m.dead_letter_reason + return BrokerMessage(str(m.message_id), json.loads(_text(body)), attrs, m.delivery_count, source, m) + + def _receive(self, source: str, dead: bool, max_messages: int, wait_seconds: float) -> List[BrokerMessage]: + with self._lock: + got = self._receiver(source, dead).receive_messages(max_message_count=max(1, int(max_messages)), + max_wait_time=max(0.5, float(wait_seconds))) + return [self._convert(m, source) for m in got] + + def receive(self, source, max_messages=10, wait_seconds=0.0, config=None): + return self._receive(source, False, max_messages, wait_seconds) + + def ack(self, source, message, config=None): + with self._lock: + self._receiver(source).complete_message(message.receipt) + + def nack(self, source, message, delay_seconds=0.0, config=None): + with self._lock: + self._receiver(source).abandon_message(message.receipt) + + def dead_letter(self, source, message, reason, config=None): + with self._lock: + self._receiver(source).dead_letter_message(message.receipt, reason=str(reason)[:1000], + error_description=str(reason)[:1000]) + + def receive_dead_letters(self, source, max_messages=10, config=None): + return self._receive(source, True, max_messages, 0.5) + + def ack_dead_letter(self, source, message, config=None): + with self._lock: + self._receiver(source, True).complete_message(message.receipt) + + def close(self) -> None: + with self._lock: + for handle in self._receivers.values(): + handle.close() + self._receivers.clear() + if self._client is not None: + self._client.close() diff --git a/adapters/e2a_adapters/cas.py b/adapters/e2a_adapters/cas.py index 67e9052..ff16978 100644 --- a/adapters/e2a_adapters/cas.py +++ b/adapters/e2a_adapters/cas.py @@ -104,6 +104,7 @@ def _exhausted(what: str): # ------------------------------------------------------------ idempotency class CasIdempotencyStore(BaseIdempotencyStore): """BaseIdempotencyStore's state machine on one document per (tenant, key).""" + capabilities = frozenset({'idempotency_store'}) def __init__(self, backend: CasBackend, clock: Callable[[], float] = time.time): self.backend, self.clock = backend, clock @@ -182,6 +183,7 @@ def state_of(self, key, tenant_id=None) -> Optional[str]: class CasTransactionalStore(BaseTransactionalStore): """Versioned JSON rows; put_if_version() is a replace-if-unchanged after checking the version and the fence that were read.""" + capabilities = frozenset({'transactional_store'}) def __init__(self, backend: CasBackend, clock: Callable[[], float] = time.time): self.backend, self.clock = backend, clock @@ -237,6 +239,7 @@ def put_if_version(self, key, value, expected_version, config=None, **kw) -> Com class CasLease(BaseDistributedLock): """One document per lock key. Release clears the owner and keeps the document, so the fencing token is strictly increasing for the key's whole life.""" + capabilities = frozenset({'lease'}) def __init__(self, backend: CasBackend, clock: Callable[[], float] = time.time): self.backend, self.clock = backend, clock @@ -287,6 +290,7 @@ def release(self, handle, config=None, **kw) -> None: class CasCheckpointStore(BaseCheckpointStore): """A paused workflow's checkpoint as JSON, with its status beside it, so that decide() is one replace-if-unchanged (PENDING -> APPROVED | REJECTED).""" + capabilities = frozenset({'checkpoint_store'}) def __init__(self, backend: CasBackend, clock: Callable[[], float] = time.time, ttl_seconds: Optional[int] = None): diff --git a/adapters/e2a_adapters/config_keys.py b/adapters/e2a_adapters/config_keys.py new file mode 100644 index 0000000..edeb16d --- /dev/null +++ b/adapters/e2a_adapters/config_keys.py @@ -0,0 +1,59 @@ +"""The configuration keys of the cloud adapters (v1.7.0, design note MP-05), registered in E2A's +central registry when e2a_adapters is imported. They are what the cloud profiles (aws, gcp, +azure) set and what every adapter's from_config(config) reads, so an application chooses its +cloud by naming a profile and overrides any value with an environment variable. + +Keys the provider layer registered before (aws_region, gcp_project, the table and database +names) are registered here with the same meaning, so both packages import together.""" +import json +from typing import Any, Dict, List + +from e2a import ConfigKey, register_config_key + +_GROUP = 'Cloud infrastructure (e2a-adapters)' + + +def _list(v: Any) -> List[Any]: + if isinstance(v, (list, tuple)): + return list(v) + text = str(v).strip() + if not text: + return [] + return json.loads(text) if text.startswith('[') else [x.strip() for x in text.split(',') if x.strip()] + + +def _map(v: Any) -> Dict[str, Any]: + if isinstance(v, dict): + return v + text = str(v).strip() + return json.loads(text) if text else {} + + +KEYS = [ + # AWS + ConfigKey('aws_region', 'AWS_REGION', 'us-east-1', str, _GROUP, 'every AWS client'), + ConfigKey('aws_endpoint_url', 'AWS_ENDPOINT_URL', None, str, _GROUP, 'every AWS client — a local endpoint (moto server, LocalStack); unset in the cloud'), + ConfigKey('dynamodb_idempotency_table', 'DYNAMODB_IDEMPOTENCY_TABLE', 'e2a-idempotency', str, _GROUP, 'DynamoDBIdempotencyStore.from_config'), + ConfigKey('dynamodb_checkpoint_table', 'DYNAMODB_CHECKPOINT_TABLE', 'e2a-checkpoints', str, _GROUP, 'DynamoDBCheckpointStore.from_config'), + ConfigKey('dynamodb_state_table', 'DYNAMODB_STATE_TABLE', 'e2a-state', str, _GROUP, 'DynamoDBTransactionalStore / DynamoDBLease.from_config'), + ConfigKey('sqs_dead_letter_suffix', 'SQS_DEAD_LETTER_SUFFIX', '-dlq', str, _GROUP, 'SQSBroker — the dead-letter queue of is '), + # GCP + ConfigKey('gcp_project', 'GOOGLE_CLOUD_PROJECT', None, str, _GROUP, 'every Google client'), + ConfigKey('firestore_database', 'FIRESTORE_DATABASE', '(default)', str, _GROUP, 'Firestore*.from_config'), + ConfigKey('firestore_collection', 'FIRESTORE_COLLECTION', 'e2a', str, _GROUP, 'Firestore*.from_config — the collection of E2A records'), + ConfigKey('pubsub_dead_letter_suffix', 'PUBSUB_DEAD_LETTER_SUFFIX', '-dlq', str, _GROUP, 'PubSubBroker — the dead-letter topic and subscription of are '), + # Azure + ConfigKey('cosmos_endpoint', 'COSMOS_ENDPOINT', None, str, _GROUP, 'Cosmos*.from_config'), + ConfigKey('cosmos_database', 'COSMOS_DATABASE', 'e2a', str, _GROUP, 'Cosmos*.from_config'), + ConfigKey('cosmos_container', 'COSMOS_CONTAINER', 'e2a', str, _GROUP, 'Cosmos*.from_config — the container of E2A records'), + ConfigKey('cosmos_key', 'COSMOS_KEY', None, str, _GROUP, 'Cosmos*.from_config — an account key (the emulator); unset in the cloud: Entra ID', secret=True), + ConfigKey('servicebus_namespace', 'SERVICEBUS_NAMESPACE', None, str, _GROUP, 'ServiceBusBroker — .servicebus.windows.net, with Entra ID'), + ConfigKey('servicebus_connection_string', 'SERVICEBUS_CONNECTION_STRING', None, str, _GROUP, 'ServiceBusBroker — a connection string (the emulator); unset in the cloud', secret=True), + ConfigKey('keyvault_url', 'KEYVAULT_URL', None, str, _GROUP, 'KeyVaultCredentialProvider — https://.vault.azure.net'), + # secrets as tool credentials (N2), every cloud + ConfigKey('secret_headers', 'E2A_SECRET_HEADERS', [], _list, _GROUP, '[{target, secret, header, prefix, json_key}] — the header a tool call to carries, read from a secret store'), + ConfigKey('secret_cache_seconds', 'E2A_SECRET_CACHE_SECONDS', 300, int, _GROUP, 'secret-backed credential providers — how long a secret value is reused'), +] + +for _spec in KEYS: + register_config_key(_spec) diff --git a/adapters/e2a_adapters/gcp/__init__.py b/adapters/e2a_adapters/gcp/__init__.py index 2a6bdce..191640b 100644 --- a/adapters/e2a_adapters/gcp/__init__.py +++ b/adapters/e2a_adapters/gcp/__init__.py @@ -1,7 +1,13 @@ -"""GCP adapters for E2A's foundation ports (design note D-03). Needs google-cloud-firestore: -install e2a-adapters[gcp].""" +"""GCP adapters for E2A's ports (design notes D-03, MP-05). Install e2a-adapters[gcp]. + + Firestore* claims, rows, leases, checkpoints + PubSubBroker messaging, with dead letters + SecretManagerCredentialProvider tool credentials from Secret Manager +""" from .firestore import (FirestoreBackend, FirestoreCheckpointStore, FirestoreIdempotencyStore, FirestoreLease, FirestoreTransactionalStore) +from .pubsub import PubSubBroker +from .secrets import SecretManagerCredentialProvider __all__ = ['FirestoreBackend', 'FirestoreIdempotencyStore', 'FirestoreTransactionalStore', 'FirestoreLease', - 'FirestoreCheckpointStore'] + 'FirestoreCheckpointStore', 'PubSubBroker', 'SecretManagerCredentialProvider'] diff --git a/adapters/e2a_adapters/gcp/firestore.py b/adapters/e2a_adapters/gcp/firestore.py index f363da5..e369a67 100644 --- a/adapters/e2a_adapters/gcp/firestore.py +++ b/adapters/e2a_adapters/gcp/firestore.py @@ -93,22 +93,40 @@ def _backend(client, collection, ttl_field): return FirestoreBackend(client, collection, ttl_field) -class FirestoreIdempotencyStore(CasIdempotencyStore): +def _client_from(config): + """v1.7.0: a Firestore client from configuration, created on first use. The emulator is + selected by FIRESTORE_EMULATOR_HOST, as the SDK does by itself.""" + from e2a import resolve_config + from .. import config_keys # noqa: F401 + from .._lazy import Lazy + project, database = resolve_config('gcp_project', config), resolve_config('firestore_database', config) + return Lazy(lambda: firestore.Client(project=project, database=database)), \ + resolve_config('firestore_collection', config) + + +class _FromConfig: + @classmethod + def from_config(cls, config, **kwargs): + client, collection = _client_from(config) + return cls(client, collection, **kwargs) + + +class FirestoreIdempotencyStore(_FromConfig, CasIdempotencyStore): def __init__(self, client, collection: str = 'e2a', ttl_field: str = 'ttl', clock: Callable[[], float] = time.time): super().__init__(_backend(client, collection, ttl_field), clock) -class FirestoreTransactionalStore(CasTransactionalStore): +class FirestoreTransactionalStore(_FromConfig, CasTransactionalStore): def __init__(self, client, collection: str = 'e2a', ttl_field: str = 'ttl', clock: Callable[[], float] = time.time): super().__init__(_backend(client, collection, ttl_field), clock) -class FirestoreLease(CasLease): +class FirestoreLease(_FromConfig, CasLease): def __init__(self, client, collection: str = 'e2a', ttl_field: str = 'ttl', clock: Callable[[], float] = time.time): super().__init__(_backend(client, collection, ttl_field), clock) -class FirestoreCheckpointStore(CasCheckpointStore): +class FirestoreCheckpointStore(_FromConfig, CasCheckpointStore): def __init__(self, client, collection: str = 'e2a', ttl_field: str = 'ttl', clock: Callable[[], float] = time.time, ttl_seconds: Optional[int] = None): super().__init__(_backend(client, collection, ttl_field), clock, ttl_seconds) diff --git a/adapters/e2a_adapters/gcp/pubsub.py b/adapters/e2a_adapters/gcp/pubsub.py new file mode 100644 index 0000000..08fab85 --- /dev/null +++ b/adapters/e2a_adapters/gcp/pubsub.py @@ -0,0 +1,86 @@ +"""Google Cloud Pub/Sub as E2A's BaseMessageBroker (v1.7.0, design note MP-05, N1). + + destination a topic id (in gcp_project) + source a subscription id + dead letters the topic and subscription (default "-dlq"). + A dead-letter policy on the subscription forwards a message there by itself after + its max delivery attempts; dead_letter() forwards it at once, with the reason. + +The acknowledgement deadline is the lock; nack() sets it to the delay (0: redeliver now). +delivery_count is Pub/Sub's delivery_attempt, which it sets only for a subscription with a +dead-letter policy (None otherwise: consumer_max_deliveries does not apply, the policy does). +""" +import json +from typing import Any, Dict, List + +from e2a import BrokerMessage, resolve_config +from e2a.messaging import BaseMessageBroker + +from .. import config_keys # noqa: F401 +from .._lazy import Lazy + + +class PubSubBroker(BaseMessageBroker): + capabilities = frozenset({'messaging', 'dead_letter'}) + + def __init__(self, project: str, publisher=None, subscriber=None, dead_letter_suffix: str = '-dlq') -> None: + from google.cloud import pubsub_v1 + self.project, self.suffix = project, dead_letter_suffix + self.publisher = publisher or Lazy(pubsub_v1.PublisherClient) + self.subscriber = subscriber or Lazy(pubsub_v1.SubscriberClient) + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> 'PubSubBroker': + return cls(resolve_config('gcp_project', config), dead_letter_suffix=resolve_config('pubsub_dead_letter_suffix', config)) + + def _project(self) -> str: + if not self.project: + raise ValueError('PubSubBroker needs a project (gcp_project / GOOGLE_CLOUD_PROJECT)') + return self.project + + def _topic(self, topic: str) -> str: + return topic if topic.startswith('projects/') else f'projects/{self._project()}/topics/{topic}' + + def _sub(self, sub: str) -> str: + return sub if sub.startswith('projects/') else f'projects/{self._project()}/subscriptions/{sub}' + + def publish(self, destination, body, attributes=None, config=None): + data = json.dumps(body, ensure_ascii=False, separators=(',', ':')).encode('utf-8') + return self.publisher.publish(self._topic(destination), data, + **{str(k): str(v) for k, v in (attributes or {}).items()}).result(timeout=30) + + def _pull(self, sub: str, max_messages: int, wait_seconds: float) -> List[BrokerMessage]: + from google.api_core import exceptions as gx + try: + resp = self.subscriber.pull(request={'subscription': self._sub(sub), 'max_messages': max(1, int(max_messages))}, + timeout=max(1.0, float(wait_seconds) + 1.0)) + except (gx.DeadlineExceeded, gx.RetryError): + return [] + out = [] + for r in resp.received_messages: + m = r.message + out.append(BrokerMessage(m.message_id, json.loads(m.data.decode('utf-8')), dict(m.attributes), + r.delivery_attempt or None, sub, r.ack_id)) + return out + + def receive(self, source, max_messages=10, wait_seconds=0.0, config=None): + return self._pull(source, max_messages, wait_seconds) + + def ack(self, source, message, config=None): + self.subscriber.acknowledge(request={'subscription': self._sub(source), 'ack_ids': [message.receipt]}) + + def nack(self, source, message, delay_seconds=0.0, config=None): + self.subscriber.modify_ack_deadline(request={'subscription': self._sub(source), 'ack_ids': [message.receipt], + 'ack_deadline_seconds': max(0, min(int(delay_seconds), 600))}) + + def dead_letter(self, source, message, reason, config=None): + attrs = {**message.attributes, 'dead_letter_reason': str(reason)[:1000], 'dead_letter_source': source} + self.publish(source + self.suffix, message.body, attrs) + self.ack(source, message) # only after the dead letter is safely published + + def receive_dead_letters(self, source, max_messages=10, config=None): + return self._pull(source + self.suffix, max_messages, 0) + + def ack_dead_letter(self, source, message, config=None): + self.subscriber.acknowledge(request={'subscription': self._sub(source + self.suffix), + 'ack_ids': [message.receipt]}) diff --git a/adapters/e2a_adapters/gcp/secrets.py b/adapters/e2a_adapters/gcp/secrets.py new file mode 100644 index 0000000..a300fb4 --- /dev/null +++ b/adapters/e2a_adapters/gcp/secrets.py @@ -0,0 +1,17 @@ +"""Google Secret Manager as the source of tool credentials (v1.7.0, N2), through the service account.""" +from typing import Any, Dict + +from e2a import resolve_config + +from ..secrets import SecretHeaderProvider + + +class SecretManagerCredentialProvider(SecretHeaderProvider): + def _build_client(self, config: Dict[str, Any]) -> Any: + from google.cloud import secretmanager + return secretmanager.SecretManagerServiceClient() + + def _fetch(self, secret: str) -> str: + name = secret if secret.startswith('projects/') else \ + f"projects/{resolve_config('gcp_project', self.config)}/secrets/{secret}/versions/latest" + return self.client.access_secret_version(request={'name': name}).payload.data.decode('utf-8') diff --git a/adapters/e2a_adapters/identity.py b/adapters/e2a_adapters/identity.py new file mode 100644 index 0000000..08a9333 --- /dev/null +++ b/adapters/e2a_adapters/identity.py @@ -0,0 +1,24 @@ +"""Claim maps of the cloud identity providers (v1.7.0, design note MP-05, N3), for E2A's +ClaimMapIdentityResolver. The cloud profiles set `identity_claim_map` to the preset of their +provider; these are the same maps, for code that configures a resolver directly. + +Verification (signature, expiry, audience) stays in the gateway in front of the service; E2A +resolves the verified claims. The tenant claim is the deployment's choice, so each preset names +the convention these projects use and can be overridden key by key.""" +from typing import Dict + +PRESETS: Dict[str, Dict[str, str]] = { + # Amazon Cognito user pools: a custom attribute for the tenant, groups as roles + 'cognito': {'tenant_id': 'custom:tenant_id', 'subject': 'sub', 'roles': 'cognito:groups', 'issuer': 'iss'}, + # Microsoft Entra ID: the directory (tid) is the tenant, mapped with identity_tenant_map; app roles + 'entra_id': {'tenant_id': 'tid', 'subject': 'oid', 'roles': 'roles', 'issuer': 'iss'}, + # Google Identity Platform with multi-tenancy: firebase.tenant; roles from a custom claim + 'identity_platform': {'tenant_id': 'firebase.tenant', 'subject': 'sub', 'roles': 'roles', 'issuer': 'iss'}, +} + + +def preset(name: str, **overrides: str) -> Dict[str, str]: + """A provider's claim map, with any claim renamed: preset('cognito', tenant_id='custom:org').""" + if name not in PRESETS: + raise ValueError(f'unknown identity preset {name!r}: one of {", ".join(sorted(PRESETS))}') + return {**PRESETS[name], **overrides} diff --git a/adapters/e2a_adapters/profiles/__init__.py b/adapters/e2a_adapters/profiles/__init__.py new file mode 100644 index 0000000..236701b --- /dev/null +++ b/adapters/e2a_adapters/profiles/__init__.py @@ -0,0 +1 @@ +"""The cloud profiles (aws, gcp, azure): JSON files bound by e2a.bindings.""" diff --git a/adapters/e2a_adapters/profiles/aws.json b/adapters/e2a_adapters/profiles/aws.json new file mode 100644 index 0000000..511ee28 --- /dev/null +++ b/adapters/e2a_adapters/profiles/aws.json @@ -0,0 +1,39 @@ +{ + "name": "aws", + "provider": "cloud-infrastructure", + "cloud": "aws", + "description": "AWS infrastructure for E2A: DynamoDB claims, rows, leases and checkpoints; SQS/SNS messaging with dead letters; Secrets Manager for tool credentials; Cognito claims at ingress. A provider profile extends it.", + "bindings": { + "idempotency_store": "e2a_adapters.aws:DynamoDBIdempotencyStore", + "checkpoint_store": "e2a_adapters.aws:DynamoDBCheckpointStore", + "transactional_store": "e2a_adapters.aws:DynamoDBTransactionalStore", + "lease": "e2a_adapters.aws:DynamoDBLease", + "message_broker": "e2a_adapters.aws:SQSBroker", + "credential_provider": "e2a_adapters.aws:SecretsManagerCredentialProvider", + "identity_resolver": "e2a.identity:ClaimMapIdentityResolver" + }, + "config": { + "aws_region": "${AWS_REGION:-us-east-1}", + "aws_endpoint_url": "${AWS_ENDPOINT_URL}", + "dynamodb_idempotency_table": "${DYNAMODB_IDEMPOTENCY_TABLE:-e2a-idempotency}", + "dynamodb_checkpoint_table": "${DYNAMODB_CHECKPOINT_TABLE:-e2a-checkpoints}", + "dynamodb_state_table": "${DYNAMODB_STATE_TABLE:-e2a-state}", + "sqs_dead_letter_suffix": "-dlq", + "identity_claim_map": { + "tenant_id": "custom:tenant_id", + "subject": "sub", + "roles": "cognito:groups", + "issuer": "iss" + } + }, + "features": [ + "idempotency_store", + "checkpoint_store", + "transactional_store", + "lease", + "messaging", + "dead_letter", + "secret_store", + "identity" + ] +} diff --git a/adapters/e2a_adapters/profiles/azure.json b/adapters/e2a_adapters/profiles/azure.json new file mode 100644 index 0000000..5a341d3 --- /dev/null +++ b/adapters/e2a_adapters/profiles/azure.json @@ -0,0 +1,40 @@ +{ + "name": "azure", + "provider": "cloud-infrastructure", + "cloud": "azure", + "description": "Azure infrastructure for E2A: Cosmos DB claims, rows, leases and checkpoints; Service Bus messaging with native dead letters; Key Vault for tool credentials; Entra ID claims at ingress. A provider profile extends it.", + "bindings": { + "idempotency_store": "e2a_adapters.azure:CosmosIdempotencyStore", + "checkpoint_store": "e2a_adapters.azure:CosmosCheckpointStore", + "transactional_store": "e2a_adapters.azure:CosmosTransactionalStore", + "lease": "e2a_adapters.azure:CosmosLease", + "message_broker": "e2a_adapters.azure:ServiceBusBroker", + "credential_provider": "e2a_adapters.azure:KeyVaultCredentialProvider", + "identity_resolver": "e2a.identity:ClaimMapIdentityResolver" + }, + "config": { + "cosmos_endpoint": "${COSMOS_ENDPOINT}", + "cosmos_database": "${COSMOS_DATABASE:-e2a}", + "cosmos_container": "${COSMOS_CONTAINER:-e2a}", + "cosmos_key": "${COSMOS_KEY}", + "servicebus_namespace": "${SERVICEBUS_NAMESPACE}", + "servicebus_connection_string": "${SERVICEBUS_CONNECTION_STRING}", + "keyvault_url": "${KEYVAULT_URL}", + "identity_claim_map": { + "tenant_id": "tid", + "subject": "oid", + "roles": "roles", + "issuer": "iss" + } + }, + "features": [ + "idempotency_store", + "checkpoint_store", + "transactional_store", + "lease", + "messaging", + "dead_letter", + "secret_store", + "identity" + ] +} diff --git a/adapters/e2a_adapters/profiles/gcp.json b/adapters/e2a_adapters/profiles/gcp.json new file mode 100644 index 0000000..1823081 --- /dev/null +++ b/adapters/e2a_adapters/profiles/gcp.json @@ -0,0 +1,37 @@ +{ + "name": "gcp", + "provider": "cloud-infrastructure", + "cloud": "gcp", + "description": "Google Cloud infrastructure for E2A: Firestore claims, rows, leases and checkpoints; Pub/Sub messaging with dead letters; Secret Manager for tool credentials; Identity Platform claims at ingress. A provider profile extends it.", + "bindings": { + "idempotency_store": "e2a_adapters.gcp:FirestoreIdempotencyStore", + "checkpoint_store": "e2a_adapters.gcp:FirestoreCheckpointStore", + "transactional_store": "e2a_adapters.gcp:FirestoreTransactionalStore", + "lease": "e2a_adapters.gcp:FirestoreLease", + "message_broker": "e2a_adapters.gcp:PubSubBroker", + "credential_provider": "e2a_adapters.gcp:SecretManagerCredentialProvider", + "identity_resolver": "e2a.identity:ClaimMapIdentityResolver" + }, + "config": { + "gcp_project": "${GOOGLE_CLOUD_PROJECT}", + "firestore_database": "${FIRESTORE_DATABASE:-(default)}", + "firestore_collection": "${FIRESTORE_COLLECTION:-e2a}", + "pubsub_dead_letter_suffix": "-dlq", + "identity_claim_map": { + "tenant_id": "firebase.tenant", + "subject": "sub", + "roles": "roles", + "issuer": "iss" + } + }, + "features": [ + "idempotency_store", + "checkpoint_store", + "transactional_store", + "lease", + "messaging", + "dead_letter", + "secret_store", + "identity" + ] +} diff --git a/adapters/e2a_adapters/secrets.py b/adapters/e2a_adapters/secrets.py new file mode 100644 index 0000000..fb544c4 --- /dev/null +++ b/adapters/e2a_adapters/secrets.py @@ -0,0 +1,71 @@ +"""Secrets as tool credentials (v1.7.0, design note MP-05, N2). + +E2A's BaseCredentialProvider (v1.6.0) gives a tool call its auth headers. The providers here read +them from the cloud's secret store, through the platform identity (a task or pod role, a +managed identity, a service account), so no secret is in configuration: + + secret_headers = [{"target": "https://api.partner.example", "secret": "partner-api-token", + "header": "Authorization", "prefix": "Bearer "}] + +The longest matching `target` prefix wins; `json_key` reads one field of a JSON secret. A value +is cached for `secret_cache_seconds`, so a rotated secret is picked up without a restart. +""" +import json +import threading +import time +from abc import abstractmethod +from typing import Any, Dict, Optional + +from e2a import BaseCredentialProvider, resolve_config + +from . import config_keys # noqa: F401 + + +class SecretHeaderProvider(BaseCredentialProvider): + capabilities = frozenset({'secret_store'}) + + def __init__(self, config: Optional[Dict[str, Any]] = None, client: Any = None, clock=time.monotonic) -> None: + self.config, self._client, self._clock = dict(config or {}), client, clock + self._cache: Dict[str, tuple] = {} + self._lock = threading.Lock() + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> 'SecretHeaderProvider': + return cls(config) + + @property + def client(self) -> Any: + if self._client is None: + self._client = self._build_client(self.config) + return self._client + + @abstractmethod + def _build_client(self, config: Dict[str, Any]) -> Any: + """The secret store's SDK client, through the platform identity.""" + + @abstractmethod + def _fetch(self, secret: str) -> str: + """The current value of one secret.""" + + def secret(self, name: str) -> str: + """Public: one secret's value, cached for secret_cache_seconds.""" + ttl = float(resolve_config('secret_cache_seconds', self.config)) + with self._lock: + hit = self._cache.get(name) + if hit and self._clock() - hit[1] < ttl: + return hit[0] + value = self._fetch(name) + with self._lock: + self._cache[name] = (value, self._clock()) + return value + + def headers(self, target, scope=None, config=None, **kwargs): + specs = resolve_config('secret_headers', {**self.config, **(config or {})}) or [] + match = max((s for s in specs if str(target).startswith(str(s.get('target', '')))), + key=lambda s: len(str(s.get('target', ''))), default=None) + if match is None: + return {} + value = self.secret(match['secret']) + if match.get('json_key'): + value = json.loads(value)[match['json_key']] + return {match.get('header', 'Authorization'): f"{match.get('prefix', 'Bearer ')}{value}"} diff --git a/adapters/pyproject.toml b/adapters/pyproject.toml index c9fc3cb..1189cad 100644 --- a/adapters/pyproject.toml +++ b/adapters/pyproject.toml @@ -4,24 +4,27 @@ build-backend = "setuptools.build_meta" [project] name = "e2a-adapters" -version = "1.6.1" -description = "Production adapters for the E2A foundation ports: DynamoDB, Firestore, Cosmos DB" +version = "1.7.0" +description = "E2A ports on cloud infrastructure: stores, brokers, secrets and identity on AWS, Google Cloud and Azure" readme = "README.md" license = "Apache-2.0" authors = [{ name = "Subham Gupta" }] requires-python = ">=3.11" # The E2A version this release is tested against. Install e2a-framework from its git tag first # (as every project already does); this requirement is then satisfied without an index lookup. -dependencies = ["e2a-framework>=1.5,<2"] +dependencies = ["e2a-framework>=1.7,<2"] [project.optional-dependencies] aws = ["boto3>=1.34"] -gcp = ["google-cloud-firestore>=2.16"] -azure = ["azure-cosmos>=4.7"] -test = ["pytest>=8.2", "moto[dynamodb]>=5.0"] +gcp = ["google-cloud-firestore>=2.16", "google-cloud-pubsub>=2.21", "google-cloud-secret-manager>=2.20"] +azure = ["azure-cosmos>=4.7", "azure-servicebus>=7.12", "azure-keyvault-secrets>=4.8", "azure-identity>=1.17"] +test = ["pytest>=8.2", "moto[dynamodb,sqs,sns,secretsmanager]>=5.0"] [project.urls] Repository = "https://github.com/subhamviky/e2a-framework" [tool.setuptools] -packages = ["e2a_adapters", "e2a_adapters.aws", "e2a_adapters.gcp", "e2a_adapters.azure"] +packages = ["e2a_adapters", "e2a_adapters.aws", "e2a_adapters.gcp", "e2a_adapters.azure", "e2a_adapters.profiles"] + +[tool.setuptools.package-data] +"e2a_adapters.profiles" = ["*.json"] diff --git a/adapters/tests/servicebus-emulator.json b/adapters/tests/servicebus-emulator.json new file mode 100644 index 0000000..02df8d0 --- /dev/null +++ b/adapters/tests/servicebus-emulator.json @@ -0,0 +1,435 @@ +{ + "UserConfig": { + "Namespaces": [ + { + "Name": "sbemulatorns", + "Queues": [ + { + "Name": "q0", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q1", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q2", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q3", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q4", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q5", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q6", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q7", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q8", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q9", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q10", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q11", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q12", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q13", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q14", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q15", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q16", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q17", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q18", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q19", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q20", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q21", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q22", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q23", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q24", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q25", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q26", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q27", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q28", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + }, + { + "Name": "q29", + "Properties": { + "DeadLetteringOnMessageExpiration": false, + "DefaultMessageTimeToLive": "PT1H", + "DuplicateDetectionHistoryTimeWindow": "PT20S", + "ForwardDeadLetteredMessagesTo": "", + "ForwardTo": "", + "LockDuration": "PT1M", + "MaxDeliveryCount": 10, + "RequiresDuplicateDetection": false, + "RequiresSession": false + } + } + ], + "Topics": [] + } + ], + "Logging": { + "Type": "File" + } + } +} \ No newline at end of file diff --git a/adapters/tests/test_messaging.py b/adapters/tests/test_messaging.py new file mode 100644 index 0000000..ffe14d3 --- /dev/null +++ b/adapters/tests/test_messaging.py @@ -0,0 +1,87 @@ +"""The brokers pass E2A's messaging suite (e2a.port_contract.check_message_broker, v1.7.0): SQS +on moto always; Pub/Sub and Service Bus on their emulators when CI runs them.""" +import itertools +import json +import os +import uuid + +import boto3 +import pytest +from moto import mock_aws + +from e2a.port_contract import check_message_broker + +SUITE = 25 + + +def test_sqs_meets_the_messaging_contract(): + from e2a_adapters.aws import SQSBroker + with mock_aws(): + sqs = boto3.client('sqs', region_name='us-east-1') + broker = SQSBroker(sqs) + + def make(): + q = f'q-{uuid.uuid4().hex[:8]}' + sqs.create_queue(QueueName=q) + sqs.create_queue(QueueName=q + '-dlq') + return broker, q, q + assert len(check_message_broker(make, timeout=5)) == SUITE + + +def test_sns_fans_out_to_queues_with_the_attributes(): + from e2a_adapters.aws import SQSBroker + with mock_aws(): + sqs, sns = boto3.client('sqs', region_name='us-east-1'), boto3.client('sns', region_name='us-east-1') + topic = sns.create_topic(Name='orders')['TopicArn'] + url = sqs.create_queue(QueueName='billing')['QueueUrl'] + arn = sqs.get_queue_attributes(QueueUrl=url, AttributeNames=['QueueArn'])['Attributes']['QueueArn'] + sns.subscribe(TopicArn=topic, Protocol='sqs', Endpoint=arn, Attributes={'RawMessageDelivery': 'true'}) + broker = SQSBroker(sqs, sns) + broker.publish(topic, {'order': 'A-1', 'amount': '10.50'}, {'tenant_id': 't1', 'idempotency_key': 'k1'}) + [m] = broker.receive('billing') + assert m.body == {'order': 'A-1', 'amount': '10.50'} and (m.tenant_id, m.idempotency_key) == ('t1', 'k1') + + +def test_sqs_from_config_reads_region_and_endpoint(): + from e2a_adapters.aws import SQSBroker + with mock_aws(): + boto3.client('sqs', region_name='eu-west-1').create_queue(QueueName='jobs') + broker = SQSBroker.from_config({'aws_region': 'eu-west-1'}) + broker.publish('jobs', {'n': 1}, {'tenant_id': 't', 'idempotency_key': 'k'}) + assert broker.receive('jobs')[0].body == {'n': 1} + + +@pytest.mark.skipif(not os.getenv('PUBSUB_EMULATOR_HOST'), reason='PUBSUB_EMULATOR_HOST not set (Pub/Sub emulator not running)') +def test_pubsub_meets_the_messaging_contract(): + from google.cloud import pubsub_v1 + + from e2a_adapters.gcp import PubSubBroker + project = 'demo-e2a-adapters' + pub, sub = pubsub_v1.PublisherClient(), pubsub_v1.SubscriberClient() + broker = PubSubBroker(project, pub, sub) + + def make(): + name = f'm{uuid.uuid4().hex[:8]}' + for n in (name, name + '-dlq'): + pub.create_topic(request={'name': f'projects/{project}/topics/{n}'}) + sub.create_subscription(request={'name': f'projects/{project}/subscriptions/{n}', + 'topic': f'projects/{project}/topics/{n}', 'ack_deadline_seconds': 30}) + return broker, name, name + # Pub/Sub counts deliveries only under a dead-letter policy, so the count check is not run + assert len(check_message_broker(make, timeout=15)) == SUITE - 1 + + +@pytest.mark.skipif(not os.getenv('SERVICEBUS_CONNECTION_STRING'), + reason='SERVICEBUS_CONNECTION_STRING not set (Service Bus emulator not running)') +def test_service_bus_meets_the_messaging_contract(): + from e2a_adapters.azure import ServiceBusBroker + broker = ServiceBusBroker(connection_string=os.environ['SERVICEBUS_CONNECTION_STRING']) + # the emulator declares its queues up front (adapters/tests/servicebus-emulator.json): q0 .. q29 + names = (f'q{i}' for i in itertools.count()) + try: + def make(): + q = next(names) + return broker, q, q + assert len(check_message_broker(make, timeout=15)) == SUITE + finally: + broker.close() diff --git a/adapters/tests/test_profiles.py b/adapters/tests/test_profiles.py new file mode 100644 index 0000000..c66c1f2 --- /dev/null +++ b/adapters/tests/test_profiles.py @@ -0,0 +1,97 @@ +"""The cloud profiles (v1.7.0): each binds and verifies without credentials or network, its +identity claim map is the preset of its cloud's identity provider, and its stores, built +from configuration, keep the port contracts (DynamoDB on moto; Firestore and Cosmos DB on +their emulators when CI runs them).""" +import json +import os +import uuid +from pathlib import Path + +import boto3 +import pytest +from moto import mock_aws + +import e2a_adapters +from e2a import bindings_from_profile +from e2a.bindings import available_profiles +from e2a.port_contract import check_idempotency_store, check_lease +from e2a_adapters.identity import PRESETS, preset + +CLOUDS = {'aws': 'cognito', 'gcp': 'identity_platform', 'azure': 'entra_id'} + + +@pytest.mark.parametrize('cloud', sorted(CLOUDS)) +def test_each_cloud_profile_binds_and_verifies_offline(cloud): + assert cloud in available_profiles() + b = bindings_from_profile(cloud, verify=True) + config = b.config() + for seam in ('idempotency_store', 'checkpoint_store', 'transactional_store', 'lease', 'message_broker', + 'credential_provider', 'identity_resolver'): + assert config.get(seam) is not None, seam + assert config['profile_settings']['identity_claim_map'] == PRESETS[CLOUDS[cloud]] + assert b.warnings == [] and b.profile.literal_secrets == [] + + +def test_the_profiles_are_shipped_in_the_package(): + names = sorted(p.stem for p in (Path(e2a_adapters.__file__).parent / 'profiles').glob('*.json')) + assert names == ['aws', 'azure', 'gcp'] + + +def test_an_environment_variable_overrides_a_profile_value(monkeypatch): + monkeypatch.setenv('DYNAMODB_IDEMPOTENCY_TABLE', 'orders-claims') + assert bindings_from_profile('aws').idempotency_store.table_name == 'orders-claims' + + +def test_presets_rename_a_claim(): + assert preset('cognito', tenant_id='custom:org')['tenant_id'] == 'custom:org' + with pytest.raises(ValueError): + preset('okta') + + +def test_the_aws_profile_resolves_a_cognito_identity(): + resolver = bindings_from_profile('aws').identity_resolver + ident = resolver.resolve({'custom:tenant_id': 'acme', 'sub': 'u1', 'cognito:groups': ['ops']}, verified=True) + assert (ident.tenant_id, ident.roles) == ('acme', ['ops']) + + +def _table(ddb, name): + ddb.create_table(TableName=name, BillingMode='PAY_PER_REQUEST', + KeySchema=[{'AttributeName': 'pk', 'KeyType': 'HASH'}, {'AttributeName': 'sk', 'KeyType': 'RANGE'}], + AttributeDefinitions=[{'AttributeName': 'pk', 'AttributeType': 'S'}, + {'AttributeName': 'sk', 'AttributeType': 'S'}]) + + +def test_dynamodb_stores_from_config_keep_the_contracts(): + from e2a_adapters.aws import DynamoDBIdempotencyStore, DynamoDBLease + with mock_aws(): + ddb = boto3.client('dynamodb', region_name='us-east-1') + _table(ddb, 'claims') + _table(ddb, 'state') + cfg = {'aws_region': 'us-east-1', 'dynamodb_idempotency_table': 'claims', 'dynamodb_state_table': 'state'} + # moto is not atomic under threads: the races run on DynamoDB Local (test_dynamodb.py) + assert check_idempotency_store(lambda c: DynamoDBIdempotencyStore.from_config(cfg, clock=c), race=1) + assert check_lease(lambda c: DynamoDBLease.from_config(cfg, clock=c), race=1) + + +@pytest.mark.skipif(not os.getenv('FIRESTORE_EMULATOR_HOST'), reason='FIRESTORE_EMULATOR_HOST not set') +def test_firestore_stores_from_config_keep_the_contracts(): + from e2a_adapters.gcp import FirestoreIdempotencyStore + cfg = {'gcp_project': 'demo-e2a-adapters', 'firestore_collection': f'cfg-{uuid.uuid4().hex[:8]}'} + assert check_idempotency_store(lambda c: FirestoreIdempotencyStore.from_config(cfg, clock=c)) + + +@pytest.mark.skipif(not os.getenv('COSMOS_EMULATOR_ENDPOINT'), reason='COSMOS_EMULATOR_ENDPOINT not set') +def test_cosmos_stores_from_config_keep_the_contracts(): + from azure.cosmos import CosmosClient, PartitionKey + + from e2a_adapters.azure import CosmosIdempotencyStore + key = 'C2y6yDjf5/R+ob0N8A7Cgv30VRDJIWEHLM+4QDU5DE2nQ9nDuVTqobD4b8mGGyPMbIZnqyMsEcaGQy67XIw/Jw==' # the emulator's public key + db = CosmosClient(os.environ['COSMOS_EMULATOR_ENDPOINT'], key).create_database_if_not_exists('e2a-adapters-test') + name = f'cfg-{uuid.uuid4().hex[:8]}' + db.create_container(name, PartitionKey('/pk'), default_ttl=-1) + try: + cfg = {'cosmos_endpoint': os.environ['COSMOS_EMULATOR_ENDPOINT'], 'cosmos_key': key, + 'cosmos_database': 'e2a-adapters-test', 'cosmos_container': name} + assert check_idempotency_store(lambda c: CosmosIdempotencyStore.from_config(cfg, clock=c)) + finally: + db.delete_container(name) diff --git a/adapters/tests/test_secrets.py b/adapters/tests/test_secrets.py new file mode 100644 index 0000000..ca6f497 --- /dev/null +++ b/adapters/tests/test_secrets.py @@ -0,0 +1,81 @@ +"""Secret-backed tool credentials (v1.7.0, N2): Secrets Manager on moto; Secret Manager and Key Vault +through their SDKs' client shape (no official emulator exists for either).""" +import json +from types import SimpleNamespace + +import boto3 +from moto import mock_aws + +from e2a_adapters.aws import SecretsManagerCredentialProvider + +SPECS = [{'target': 'https://api.partner.example', 'secret': 'partner-token'}, + {'target': 'https://api.partner.example/v2', 'secret': 'partner-v2', 'header': 'X-Api-Key', 'prefix': '', + 'json_key': 'key'}] + + +class Clock: + def __init__(self): + self.now = 0.0 + + def __call__(self): + return self.now + + +def test_secrets_manager_supplies_tool_headers_and_caches_them(): + with mock_aws(): + sm = boto3.client('secretsmanager', region_name='us-east-1') + sm.create_secret(Name='partner-token', SecretString='tok-1') + sm.create_secret(Name='partner-v2', SecretString=json.dumps({'key': 'k-2'})) + clock = Clock() + provider = SecretsManagerCredentialProvider({'secret_headers': SPECS, 'secret_cache_seconds': 60}, clock=clock) + assert provider.headers('https://api.partner.example/orders') == {'Authorization': 'Bearer tok-1'} + assert provider.headers('https://api.partner.example/v2/orders') == {'X-Api-Key': 'k-2'} # longest target + assert provider.headers('https://elsewhere.example') == {} + sm.put_secret_value(SecretId='partner-token', SecretString='tok-2') # rotated + assert provider.headers('https://api.partner.example/x') == {'Authorization': 'Bearer tok-1'} + clock.now = 61 + assert provider.headers('https://api.partner.example/x') == {'Authorization': 'Bearer tok-2'} + + +def test_a_tool_call_carries_the_secret_header(): + import asyncio + + from e2a import RestToolService + seen = {} + + class Tool(RestToolService): + async def _send_request(self, endpoint, body, token=None, timeout=None, config=None, **kwargs): + seen['headers'] = kwargs.get('auth_headers') + return {'status': 200, 'physical_success': True} + + with mock_aws(): + boto3.client('secretsmanager', region_name='us-east-1').create_secret(Name='partner-token', SecretString='tok') + provider = SecretsManagerCredentialProvider({'secret_headers': SPECS}) + asyncio.run(Tool().execute({'q': 1}, {'tool_base_url': 'https://api.partner.example/orders', + 'credential_provider': provider}, tenant_id='t1')) + assert seen['headers'] == {'Authorization': 'Bearer tok'} + + +def test_secret_manager_reads_the_latest_version(): + from e2a_adapters.gcp import SecretManagerCredentialProvider + asked = [] + + class Client: + def access_secret_version(self, request): + asked.append(request['name']) + return SimpleNamespace(payload=SimpleNamespace(data=b'gtok')) + + provider = SecretManagerCredentialProvider({'secret_headers': SPECS[:1], 'gcp_project': 'p1'}, client=Client()) + assert provider.headers('https://api.partner.example/a') == {'Authorization': 'Bearer gtok'} + assert asked == ['projects/p1/secrets/partner-token/versions/latest'] + + +def test_key_vault_reads_the_secret_value(): + from e2a_adapters.azure import KeyVaultCredentialProvider + + class Client: + def get_secret(self, name): + return SimpleNamespace(value=f'kv-{name}') + + provider = KeyVaultCredentialProvider({'secret_headers': SPECS[:1]}, client=Client()) + assert provider.headers('https://api.partner.example/a') == {'Authorization': 'Bearer kv-partner-token'} diff --git a/java/e2a-core/pom.xml b/java/e2a-core/pom.xml index 68d9db7..3693f17 100644 --- a/java/e2a-core/pom.xml +++ b/java/e2a-core/pom.xml @@ -5,7 +5,7 @@ com.subhamviky.e2a e2a-java - 1.6.1 + 1.7.0 e2a-core e2a-core diff --git a/java/e2a-cqrs/pom.xml b/java/e2a-cqrs/pom.xml index 4fb8b91..188f9d8 100644 --- a/java/e2a-cqrs/pom.xml +++ b/java/e2a-cqrs/pom.xml @@ -5,7 +5,7 @@ com.subhamviky.e2a e2a-java - 1.6.1 + 1.7.0 e2a-cqrs e2a-cqrs diff --git a/java/e2a-spring-jpa/pom.xml b/java/e2a-spring-jpa/pom.xml index ce79b02..9a0fe44 100644 --- a/java/e2a-spring-jpa/pom.xml +++ b/java/e2a-spring-jpa/pom.xml @@ -5,7 +5,7 @@ com.subhamviky.e2a e2a-java - 1.6.1 + 1.7.0 e2a-spring-jpa e2a-spring-jpa diff --git a/java/e2a-test/pom.xml b/java/e2a-test/pom.xml index 652b5a5..375d95f 100644 --- a/java/e2a-test/pom.xml +++ b/java/e2a-test/pom.xml @@ -5,7 +5,7 @@ com.subhamviky.e2a e2a-java - 1.6.1 + 1.7.0 e2a-test e2a-test diff --git a/java/pom.xml b/java/pom.xml index e6ae25d..0c94a8c 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -5,7 +5,7 @@ com.subhamviky.e2a e2a-java - 1.6.1 + 1.7.0 pom e2a-java E2A runtime contracts for Java: the v1.2 agent lifecycle, declared workflows, governance stages, diff --git a/pyproject.toml b/pyproject.toml index 88492c0..e9d1615 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "e2a-framework" -version = "1.6.1" +version = "1.7.0" description = "E2A - Enterprise-to-Agentic runtime contracts (reference implementation)" readme = "README.md" license = "Apache-2.0" diff --git a/reference/e2a/__init__.py b/reference/e2a/__init__.py index de52f5c..19152ef 100644 --- a/reference/e2a/__init__.py +++ b/reference/e2a/__init__.py @@ -1,6 +1,6 @@ """E2A: Enterprise-to-Agentic runtime contracts. -Modules: core, config, resilience, observability, governance, validation, llm, agents, workflows, rag, tools, ports, pipeline, infra; context, stages, conformance (v1.2); release (v1.2.2: ProjectReleaseGate, import it from e2a.release); port_contract (v1.4.0: the behavioural suite every port adapter passes, import it from e2a.port_contract); llm_types, capabilities, bindings (v1.6.0: canonical model types, capabilities, profiles and bindings); delegated, schema_check (v1.6.1: the delegated-runtime base, the structured-output check). +Modules: core, config, resilience, observability, governance, validation, llm, agents, workflows, rag, tools, ports, pipeline, infra; context, stages, conformance (v1.2); release (v1.2.2: ProjectReleaseGate, import it from e2a.release); port_contract (v1.4.0: the behavioural suite every port adapter passes, import it from e2a.port_contract); llm_types, capabilities, bindings (v1.6.0: canonical model types, capabilities, profiles and bindings); delegated, schema_check (v1.6.1: the delegated-runtime base, the structured-output check); messaging, identity (v1.7.0: exactly-once effects from at-least-once brokers, identity at ingress). """ from .core import ErrorClass, classify_error, ModelRefusal, DehydrationInterrupt, NFRViolationError, ConcurrencyConflict, AdmissionTimeout, DeadlineExceeded, CircuitOpenError, BulkheadFullError, PhysicalOutcome, outcome_of, Deadline, ClaimStatus, ClaimResult, CommitResult, LeaseHandle, E2A_METRICS from .config import ConfigKey, E2A_CONFIG, register_config_key, resolve_config, unknown_config_keys, scoped_config, config_reference_markdown, env_example @@ -20,6 +20,9 @@ from .bindings import Bindings, Profile, bindings_from_profile, register_provider_package from .delegated import DelegatedRuntime, DelegatedTurnLimit, MaterialToolRefused from .schema_check import schema_errors +from .messaging import (BaseMessageBroker, BaseOutboxStore, BrokerMessage, BrokerOutboxSweeper, IdempotentConsumer, + InMemoryBroker, InMemoryOutboxStore) +from .identity import BaseIdentityResolver, ClaimMapIdentityResolver, Identity, IdentityRefused from .infra import BaseInfraProvisioner, GCPInfraProvisioner from .context import FailedKeys, MessageLog, ModifyAccess, ReadAccess, RequestContext, ToolDenied from .stages import (ApprovalRequired, ApprovalStage, CapacityRefusal, CapacityStage, EgressStage, @@ -28,4 +31,4 @@ RedactionStage, SandboxRefusal, SandboxStage, TokenBudgetRefusal, TokenBudgetStage, ToolRefusal, compile_redactions, redact_text) -__version__ = "1.6.1" +__version__ = "1.7.0" diff --git a/reference/e2a/bindings.py b/reference/e2a/bindings.py index 41b5e59..f0754c9 100644 --- a/reference/e2a/bindings.py +++ b/reference/e2a/bindings.py @@ -59,11 +59,17 @@ 'idempotency_store', # BaseIdempotencyStore -> config['idempotency_store'] 'checkpoint_store', # BaseCheckpointStore -> config['checkpoint_store'] 'credential_provider', # BaseCredentialProvider -> config['credential_provider'] + 'message_broker', # BaseMessageBroker (v1.7.0) -> config['message_broker'] + 'outbox_store', # BaseOutboxStore (v1.7.0) -> config['outbox_store'] + 'identity_resolver', # BaseIdentityResolver (v1.7.0) -> config['identity_resolver'] + 'transactional_store', # BaseTransactionalStore (v1.7.0)-> config['transactional_store'] + 'lease', # BaseDistributedLock (v1.7.0) -> config['lease'] 'infra_provisioner', # BaseInfraProvisioner (CI/CD time) 'release_pipeline', # BasePipeline (CI/CD time) ) _CONFIG_SEAMS = ('llm_service', 'observability_engine', 'idempotency_store', 'checkpoint_store', - 'credential_provider') + 'credential_provider', 'message_broker', 'outbox_store', 'identity_resolver', + 'transactional_store', 'lease') _PROFILE_FIELDS = {'name', 'provider', 'cloud', 'description', 'features', 'recommends', 'bindings', 'config', 'extends'} _BUILTIN_PACKAGES = {'e2a', 'e2a_simulated_llm', 'e2a_inmemory_adapters', 'e2a_adapters'} @@ -107,7 +113,7 @@ def _import(target: str) -> Callable[..., Any]: def _profile_dirs() -> List[Path]: dirs = [Path(__file__).parent / 'profiles'] for name in _provider_packages(): - if name in _BUILTIN_PACKAGES: + if name in _BUILTIN_PACKAGES and name != 'e2a_adapters': # e2a-adapters ships the cloud profiles continue try: module = importlib.import_module(name) @@ -189,17 +195,22 @@ class Profile: bindings: Dict[str, str] = field(default_factory=dict) config: Dict[str, Any] = field(default_factory=dict) source: Optional[str] = None + # v1.7.0 (N2): secret keys the profile file sets to a literal value instead of ${VAR} + literal_secrets: List[str] = field(default_factory=list) @classmethod def from_dict(cls, data: Dict[str, Any], source: Optional[str] = None, _seen: tuple = ()) -> 'Profile': unknown = set(data) - _PROFILE_FIELDS if unknown: raise ValueError(f"profile {source or data.get('name')}: unknown fields {sorted(unknown)}") + raw_config = dict(data.get('config') or {}) # before placeholders are resolved (N2) + inherited_literals: List[str] = [] if data.get('extends'): parent_name = data['extends'] if parent_name in _seen: raise ValueError(f'profile {data.get("name")}: circular extends ({" -> ".join(_seen + (parent_name,))})') parent = cls.load(parent_name, _seen=_seen + (data.get('name', '?'),)) + inherited_literals = [k for k in parent.literal_secrets if k not in (data.get('config') or {})] base = {'name': parent.name, 'provider': parent.provider, 'cloud': parent.cloud, 'description': parent.description, 'features': parent.features, 'recommends': parent.recommends, 'bindings': parent.bindings, 'config': parent.config} @@ -207,10 +218,15 @@ def from_dict(cls, data: Dict[str, Any], source: Optional[str] = None, _seen: tu unknown_seams = set(data.get('bindings') or {}) - set(SEAMS) if unknown_seams: raise ValueError(f"profile {data.get('name')}: unknown seams {sorted(unknown_seams)}") + literals = sorted(set(inherited_literals) | { + k for k, v in raw_config.items() + if k in E2A_CONFIG and E2A_CONFIG[k].secret and v not in (None, '') and not ( + isinstance(v, str) and _PLACEHOLDER.fullmatch(v.strip()))}) return cls(name=data['name'], provider=data['provider'], cloud=data.get('cloud', ''), description=data.get('description', ''), features=list(data.get('features') or []), recommends=list(data.get('recommends') or []), bindings=dict(data.get('bindings') or {}), - config=_typed(_interpolate(dict(data.get('config') or {}))), source=source) + config=_typed(_interpolate(dict(data.get('config') or {}))), source=source, + literal_secrets=literals) @classmethod def load(cls, name_or_path: str, _seen: tuple = ()) -> 'Profile': @@ -313,6 +329,8 @@ def verify(self, required: Optional[Iterable[str]] = None) -> Dict[str, List[str # v1.6.1: what the profile recommends but nothing provides is a warning, and the # embedding model and the vector index must agree on vector size self.warnings = [f'recommended capability {c!r} is not provided' for c in missing(self.profile.recommends, everything)] + self.warnings += [f'secret {k!r} is a literal value in the profile: use ${{VAR}} or a secret store' + for k in self.profile.literal_secrets] for w in self.warnings: _log.warning('profile %s: %s', self.profile.name, w) if 'embedding_model' in self.bound() and 'vector_index' in self.bound(): diff --git a/reference/e2a/capabilities.py b/reference/e2a/capabilities.py index c153369..25aca75 100644 --- a/reference/e2a/capabilities.py +++ b/reference/e2a/capabilities.py @@ -26,6 +26,8 @@ # substrate 'idempotency_store', 'checkpoint_store', 'transactional_store', 'lease', 'telemetry', 'infra_as_code', 'release_pipeline', + # v1.7.0: cloud infrastructure + 'messaging', 'dead_letter', 'outbox_store', 'identity', 'secret_store', } diff --git a/reference/e2a/config.py b/reference/e2a/config.py index 085f025..6347b23 100644 --- a/reference/e2a/config.py +++ b/reference/e2a/config.py @@ -115,6 +115,14 @@ def _k(key, env, default, cast, group, used_by, scoped=None, secret=False): _k('delegated_max_turns', 'DELEGATED_MAX_TURNS', 8, int, 'LLM service', 'v1.6.1: DelegatedRuntime — most tool round-trips in one managed session'), _k('artifact_matrix', 'E2A_ARTIFACT_MATRIX', [], _to_json_list, 'Pipeline', 'v1.6.1: BasePipeline — the images to build (empty: the default matrix)'), _k('secret_scan_paths', 'E2A_SECRET_SCAN_PATHS', ['src'], _to_json_list, 'Pipeline', 'v1.6.1: BasePipeline built-in secret scan roots ([] turns it off)'), + # -- v1.7.0 (design note MP-05): messaging, identity -- + _k('consumer_batch_size', 'E2A_CONSUMER_BATCH_SIZE', 10, int, 'Messaging', 'v1.7.0: IdempotentConsumer / BrokerOutboxSweeper — messages or records per batch'), + _k('consumer_wait_seconds', 'E2A_CONSUMER_WAIT_SECONDS', 0.0, float, 'Messaging', 'v1.7.0: IdempotentConsumer.consume() — long-poll wait for a batch'), + _k('consumer_max_deliveries', 'E2A_CONSUMER_MAX_DELIVERIES', 5, int, 'Messaging', 'v1.7.0: a message delivered more often is dead-lettered (0 = no limit; brokers without a count are not limited)'), + _k('consumer_retry_delay_seconds', 'E2A_CONSUMER_RETRY_DELAY_SECONDS', 5.0, float, 'Messaging', 'v1.7.0: a retried or deferred message becomes visible again after this'), + _k('identity_claim_map', 'E2A_IDENTITY_CLAIM_MAP', {}, _to_json_map, 'Identity', 'v1.7.0: {tenant_id, subject, roles, issuer: claim name or dotted path} — the identity provider\'s claim names'), + _k('identity_trusted_issuers', 'E2A_IDENTITY_TRUSTED_ISSUERS', [], _to_json_list, 'Identity', 'v1.7.0: issuers accepted by BaseIdentityResolver (empty: any issuer the gateway verified)'), + _k('identity_tenant_map', 'E2A_IDENTITY_TENANT_MAP', {}, _to_json_map, 'Identity', 'v1.7.0: external tenant id (an Entra directory id) -> E2A tenant_id'), _k('scopes', 'E2A_SCOPES', {}, _to_json_map, 'Bindings', 'v1.6.1: {"tenant:": {key: value}, "agent:": {...}} — overlays applied per call (tenant, then agent)'), _k('log_sink', 'LOG_SINK', 'stdout', str, 'Common', 'BaseObservability._ship_logs() / _flush_logs()'), _k('dlq_queue_url', 'DLQ_QUEUE_URL', None, str, 'Common', '_send_to_dlq()'), diff --git a/reference/e2a/identity.py b/reference/e2a/identity.py new file mode 100644 index 0000000..aba463b --- /dev/null +++ b/reference/e2a/identity.py @@ -0,0 +1,103 @@ +"""Identity at ingress: the tenant comes from verified claims, not from the request body +(v1.7.0, design note MP-05, N3). + +Token verification (signature, expiry, audience) is the gateway's job: API Gateway / ALB with +Cognito, API Management or App Gateway with Entra ID, API Gateway / IAP with Identity Platform. +E2A receives the claims the gateway verified and resolves them, the same way for every +identity provider, through a claim map from configuration: + + identity_claim_map = {"tenant_id": "custom:tenant_id", "subject": "sub", "roles": "cognito:groups"} + +The provider presets (Cognito, Entra ID, Identity Platform) are cloud infrastructure and live +in e2a-adapters' cloud profiles. E2A refuses, with IdentityRefused (a GovernanceRefusal): + * claims the gateway did not mark verified; + * an issuer outside `identity_trusted_issuers` (when the list is set); + * claims without a tenant; + * a tenant in the request that differs from the tenant in the claims. +""" +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from .config import resolve_config +from .stages import GovernanceRefusal + +__all__ = ['Identity', 'IdentityRefused', 'BaseIdentityResolver', 'ClaimMapIdentityResolver'] + + +class IdentityRefused(GovernanceRefusal, PermissionError): + code = 'IDENTITY' + + +@dataclass(frozen=True) +class Identity: + tenant_id: str + subject: Optional[str] = None + roles: List[str] = field(default_factory=list) + issuer: Optional[str] = None + + +def _claim(claims: Dict[str, Any], path: Optional[str]) -> Any: + """A claim by name; a dotted path reaches into nested claims ("app_metadata.tenant").""" + if not path: + return None + if path in claims: + return claims[path] + value: Any = claims + for part in path.split('.'): + if not isinstance(value, dict) or part not in value: + return None + value = value[part] + return value + + +class BaseIdentityResolver: + """Port (v1.7.0). Public: resolve(). A subclass may override _tenant_from(claims, config) + when a provider encodes the tenant in a way a claim name cannot express.""" + capabilities = frozenset({'identity'}) + + def __init__(self, config: Optional[Dict[str, Any]] = None) -> None: + self.config = dict(config or {}) + + @classmethod + def from_config(cls, config: Dict[str, Any]) -> 'BaseIdentityResolver': + return cls(config) + + def resolve(self, claims: Dict[str, Any], verified: bool, request_tenant: Optional[str] = None, + config: Optional[Dict[str, Any]] = None) -> Identity: + """The identity of a request whose claims the gateway verified (`verified=True`). + `request_tenant`, when the request names one, must equal the claims' tenant.""" + cfg = {**self.config, **(config or {})} + if not verified: + raise IdentityRefused('claims were not verified by the gateway', 'identity') + claims = dict(claims or {}) + mapping = {'tenant_id': 'tenant_id', 'subject': 'sub', 'roles': 'roles', 'issuer': 'iss', + **(resolve_config('identity_claim_map', cfg) or {})} + issuer = _claim(claims, mapping.get('issuer')) + trusted = resolve_config('identity_trusted_issuers', cfg) or [] + if trusted and issuer not in trusted: + raise IdentityRefused(f'issuer {issuer!r} is not trusted', 'identity') + tenant = self._tenant_from(claims, mapping, cfg) + if not tenant: + raise IdentityRefused(f"no tenant in the claims (claim {mapping.get('tenant_id')!r})", 'identity') + if request_tenant is not None and str(request_tenant) != str(tenant): + raise IdentityRefused('the request names another tenant than its credentials', 'identity') + roles = _claim(claims, mapping.get('roles')) + if isinstance(roles, str): + roles = [r for r in roles.replace(',', ' ').split() if r] + subject = _claim(claims, mapping.get('subject')) + return Identity(tenant_id=str(tenant), subject=None if subject is None else str(subject), + roles=[str(r) for r in (roles or [])], issuer=issuer) + + def _tenant_from(self, claims: Dict[str, Any], mapping: Dict[str, str], config: Dict[str, Any]) -> Optional[str]: + """Hook: the tenant from the claims. Default: the mapped claim, through + `identity_tenant_map` when the claim holds an external id (an Entra ID directory id).""" + value = _claim(claims, mapping.get('tenant_id')) + if value is None: + return None + aliases = resolve_config('identity_tenant_map', config) or {} + return aliases.get(str(value), str(value)) if aliases else str(value) + + +class ClaimMapIdentityResolver(BaseIdentityResolver): + """The resolver as configured: everything comes from identity_claim_map, + identity_trusted_issuers and identity_tenant_map.""" diff --git a/reference/e2a/messaging.py b/reference/e2a/messaging.py new file mode 100644 index 0000000..b0186a0 --- /dev/null +++ b/reference/e2a/messaging.py @@ -0,0 +1,393 @@ +"""Messaging: exactly-once effects under at-least-once delivery (v1.7.0, design note MP-05, N1). + +Every managed broker (SQS/SNS, Pub/Sub, Service Bus) delivers at least once: a message can +arrive twice, arrive after its effect already happened (a lost acknowledgement), or never be +processable at all (a poison message). E2A turns that into exactly-once *effects* with the +claim it already uses for requests: + + broker ──receive──▶ IdempotentConsumer.consume() + claim(idempotency_key, tenant) BaseIdempotencyStore + CLAIMED -> _handle(message) -> COMMITTED: complete, ack + -> NOT_COMMITTED / transient error: fail(retryable), nack + -> deterministic error: fail(final), dead-letter + -> UNKNOWN: fail(UNKNOWN), dead-letter + REPLAY -> ack (a duplicate: the effect already happened) + IN_PROGRESS -> nack with a delay (another consumer holds it) + CONFLICT / FAILED_FINAL / RECOVERY_REQUIRED -> dead-letter + dead letters ──▶ IdempotentConsumer.reconcile() (BaseReconciliationService) + confirm physical truth first (_confirm), replay only NOT_COMMITTED + +Ports here are vendor-neutral. The adapters (e2a-adapters: SQS/SNS, Pub/Sub, Service Bus) and +the outbox stores implement them; `e2a.port_contract.check_message_broker` is the suite every +adapter passes, on emulators. +""" +import hashlib +import json +import threading +import time +import uuid +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional + +from .config import resolve_config +from .core import _DETERMINISTIC_ERRORS, ClaimStatus, PhysicalOutcome, outcome_of +from .ports import BaseIdempotencyStore, BaseOutboxSweeper, BaseReconciliationService + +__all__ = ['BrokerMessage', 'BaseMessageBroker', 'InMemoryBroker', 'IdempotentConsumer', 'BaseOutboxStore', + 'InMemoryOutboxStore', 'BrokerOutboxSweeper', 'body_fingerprint'] + + +@dataclass +class BrokerMessage: + """One delivered message, the same shape whatever the broker. + + `attributes` are strings (every broker carries string attributes); `tenant_id` and + `idempotency_key` travel there. `delivery_count` is the broker's own count when it has + one (None when it does not, as Pub/Sub without a dead-letter policy). `receipt` is the + adapter's handle for ack / nack / dead-letter, opaque to everything else.""" + id: str + body: Dict[str, Any] + attributes: Dict[str, str] = field(default_factory=dict) + delivery_count: Optional[int] = None + destination: str = '' + receipt: Any = None + + @property + def tenant_id(self) -> Optional[str]: + return self.attributes.get('tenant_id') + + @property + def idempotency_key(self) -> Optional[str]: + return self.attributes.get('idempotency_key') + + +def body_fingerprint(body: Dict[str, Any]) -> str: + """The request fingerprint of a message body: the same key with a different body is a CONFLICT.""" + return hashlib.sha256(json.dumps(body, sort_keys=True, default=str).encode()).hexdigest() + + +class BaseMessageBroker(ABC): + """Port (v1.7.0): one broker, publish and consume. `destination` is where a message is + published (a queue or topic); `source` is where it is received from (a queue or a + subscription). Every source has a dead-letter source of its own, which the adapter + resolves (native where the broker has one, a configured queue or topic where not).""" + capabilities = frozenset({'messaging'}) + + @abstractmethod + def publish(self, destination: str, body: Dict[str, Any], attributes: Optional[Dict[str, str]] = None, + config: Dict[str, Any] = None) -> str: + """Publish one JSON body; return the broker's message id.""" + + @abstractmethod + def receive(self, source: str, max_messages: int = 10, wait_seconds: float = 0.0, + config: Dict[str, Any] = None) -> List[BrokerMessage]: + """Up to max_messages, each hidden from other consumers until acked, nacked or its + lock / visibility expires.""" + + @abstractmethod + def ack(self, source: str, message: BrokerMessage, config: Dict[str, Any] = None) -> None: + """Done: the broker never delivers this message again.""" + + @abstractmethod + def nack(self, source: str, message: BrokerMessage, delay_seconds: float = 0.0, + config: Dict[str, Any] = None) -> None: + """Not done: deliver it again (after delay_seconds where the broker supports a delay).""" + + @abstractmethod + def dead_letter(self, source: str, message: BrokerMessage, reason: str, config: Dict[str, Any] = None) -> None: + """Move the message to the source's dead letters, with the reason, and remove it from the source.""" + + @abstractmethod + def receive_dead_letters(self, source: str, max_messages: int = 10, + config: Dict[str, Any] = None) -> List[BrokerMessage]: + """Dead letters of a source; `attributes['dead_letter_reason']` carries the reason.""" + + @abstractmethod + def ack_dead_letter(self, source: str, message: BrokerMessage, config: Dict[str, Any] = None) -> None: + """A dead letter is settled (its effect confirmed, or replayed): remove it.""" + + +# ------------------------------------------------------------------ the reference broker +class InMemoryBroker(BaseMessageBroker): + """The reference broker: one process, at-least-once like a real one (a nacked or + unacked message is delivered again, with its delivery count). Destinations and sources + are the same names (queues). Used by tests and by the contract suite's own self-check.""" + + def __init__(self, visibility_seconds: float = 30.0, clock=time.monotonic) -> None: + self._lock = threading.Lock() + self._queues: Dict[str, List[dict]] = {} + self._dead: Dict[str, List[dict]] = {} + self.visibility_seconds, self._clock = visibility_seconds, clock + + def publish(self, destination, body, attributes=None, config=None): + mid = str(uuid.uuid4()) + with self._lock: + self._queues.setdefault(destination, []).append( + {'id': mid, 'body': json.loads(json.dumps(body)), 'attributes': dict(attributes or {}), + 'count': 0, 'visible_at': 0.0, 'destination': destination}) + return mid + + def receive(self, source, max_messages=10, wait_seconds=0.0, config=None): + now, out = self._clock(), [] + with self._lock: + for item in self._queues.get(source, []): + if len(out) >= max_messages: + break + if item['visible_at'] <= now: + item['count'] += 1 + item['visible_at'] = now + self.visibility_seconds + item['receipt'] = str(uuid.uuid4()) + out.append(BrokerMessage(item['id'], json.loads(json.dumps(item['body'])), dict(item['attributes']), + item['count'], item['destination'], item['receipt'])) + return out + + def _take(self, queue, message): + for item in queue: + if item['id'] == message.id and item.get('receipt') == message.receipt: + queue.remove(item) + return item + return None + + def ack(self, source, message, config=None): + with self._lock: + self._take(self._queues.get(source, []), message) + + def nack(self, source, message, delay_seconds=0.0, config=None): + with self._lock: + for item in self._queues.get(source, []): + if item['id'] == message.id and item.get('receipt') == message.receipt: + item['visible_at'] = self._clock() + float(delay_seconds) + + def dead_letter(self, source, message, reason, config=None): + with self._lock: + item = self._take(self._queues.get(source, []), message) + if item is not None: + item['attributes'] = {**item['attributes'], 'dead_letter_reason': str(reason)} + item['visible_at'] = 0.0 + self._dead.setdefault(source, []).append(item) + + def receive_dead_letters(self, source, max_messages=10, config=None): + now, out = self._clock(), [] + with self._lock: + for item in self._dead.get(source, []): + if len(out) >= max_messages: + break + if item['visible_at'] <= now: + item['visible_at'] = now + self.visibility_seconds + item['receipt'] = str(uuid.uuid4()) + out.append(BrokerMessage(item['id'], json.loads(json.dumps(item['body'])), dict(item['attributes']), + item['count'], item['destination'], item['receipt'])) + return out + + def ack_dead_letter(self, source, message, config=None): + with self._lock: + self._take(self._dead.get(source, []), message) + + +# ------------------------------------------------------------------ the consumer +class IdempotentConsumer(BaseReconciliationService): + """Exactly-once effects from an at-least-once broker. Public: consume() (the live path) and + the inherited reconcile() (the dead letters, confirming physical truth before any replay). + + A subclass implements _handle(message, config, tenant_id, idempotency_key) and returns + the physical_success contract ({'physical_success': bool, 'outcome': ...}). It may + override _idempotency_key / _tenant_of (default: the message attributes) and _confirm + (default UNKNOWN, so nothing is replayed until the system of record can be asked).""" + + def __init__(self, broker: BaseMessageBroker, idempotency_store: BaseIdempotencyStore, + source: Optional[str] = None, lock=None) -> None: + super().__init__(lock=lock, idempotency_store=idempotency_store) + self.broker, self.source = broker, source + self.settlements: List[tuple] = [] # (dead letter id, decision) from reconcile(), newest last + + # ------------------------------------------------------------ public: the live path + def consume(self, source: Optional[str] = None, config: Dict[str, Any] = None, + correlation_id: Optional[str] = None, **kwargs) -> Dict[str, Any]: + """Receive one batch from `source` and settle every message. Never raises for one + message: each is acked, nacked or dead-lettered, and counted in the report.""" + config = config or {} + source = source or self.source + if not source: + raise ValueError('consume(): no source (pass one, or construct the consumer with source=)') + correlation_id = correlation_id or str(uuid.uuid4()) + batch = int(resolve_config('consumer_batch_size', config, kwargs)) + wait = float(resolve_config('consumer_wait_seconds', config, kwargs)) + report = {'correlation_id': correlation_id, 'received': 0, 'processed': 0, 'duplicates': 0, + 'deferred': 0, 'retried': 0, 'dead_lettered': 0, 'dead_letter_reasons': []} + for message in self.broker.receive(source, max_messages=batch, wait_seconds=wait, config=config): + report['received'] += 1 + self.__settle(source, message, config, correlation_id, report, kwargs) + return report + + def __settle(self, source, message, config, owner, report, kwargs): + def dead(reason): + self.broker.dead_letter(source, message, reason, config) + report['dead_lettered'] += 1 + report['dead_letter_reasons'].append(reason) + + tenant, key = self._tenant_of(message), self._idempotency_key(message) + if not tenant or not key: + return dead('missing tenant_id or idempotency_key') + limit = int(resolve_config('consumer_max_deliveries', config, kwargs)) + if message.delivery_count is not None and limit and message.delivery_count > limit: + return dead(f'delivered {message.delivery_count} times (consumer_max_deliveries={limit})') + claim = self.idempotency_store.claim( + key, config, tenant_id=tenant, owner=owner, fingerprint=body_fingerprint(message.body), + lease_seconds=int(resolve_config('idempotency_lease_seconds', config, kwargs))) + if claim.status == ClaimStatus.REPLAY: + self.broker.ack(source, message, config) # a duplicate, or a lost acknowledgement + report['duplicates'] += 1 + return None + if claim.status == ClaimStatus.IN_PROGRESS: + self.broker.nack(source, message, float(resolve_config('consumer_retry_delay_seconds', config, kwargs)), + config) + report['deferred'] += 1 + return None + if claim.status != ClaimStatus.CLAIMED: # CONFLICT, FAILED_FINAL, RECOVERY_REQUIRED + return dead(f'claim {claim.status.value}') + try: + result = self._handle(message, config, tenant_id=tenant, idempotency_key=key) + except _DETERMINISTIC_ERRORS as e: + self.idempotency_store.fail(key, retryable=False, config=config, tenant_id=tenant, owner=owner) + return dead(f'{type(e).__name__}: {e}') + except Exception: + self.idempotency_store.fail(key, retryable=True, config=config, tenant_id=tenant, owner=owner) + self.broker.nack(source, message, float(resolve_config('consumer_retry_delay_seconds', config, kwargs)), + config) + report['retried'] += 1 + return None + outcome = outcome_of(result) + if outcome == PhysicalOutcome.COMMITTED: + self.idempotency_store.complete(key, dict(result or {}), config, tenant_id=tenant, owner=owner) + self.broker.ack(source, message, config) + report['processed'] += 1 + elif outcome == PhysicalOutcome.UNKNOWN: + # the effect may have happened: never retried blindly; reconcile() confirms it first + self.idempotency_store.fail(key, retryable=False, config=config, tenant_id=tenant, owner=owner, + outcome=PhysicalOutcome.UNKNOWN) + dead('outcome UNKNOWN') + else: + self.idempotency_store.fail(key, retryable=True, config=config, tenant_id=tenant, owner=owner) + self.broker.nack(source, message, float(resolve_config('consumer_retry_delay_seconds', config, kwargs)), + config) + report['retried'] += 1 + return None + + # ------------------------------------------------------------ hooks + @abstractmethod + def _handle(self, message: BrokerMessage, config: Dict[str, Any], tenant_id: str, + idempotency_key: str) -> Dict[str, Any]: + """The effect. Return {'physical_success': bool, 'outcome': PhysicalOutcome (optional)}. + Raise a ValueError / TypeError / PermissionError for a message that can never succeed.""" + + def _idempotency_key(self, message: BrokerMessage) -> Optional[str]: + return message.idempotency_key + + def _tenant_of(self, message: BrokerMessage) -> Optional[str]: + return message.tenant_id + + def _confirm(self, message: BrokerMessage, config: Dict[str, Any]) -> PhysicalOutcome: + """Did this message's effect commit? Ask the system of record. Default UNKNOWN: an + UNKNOWN dead letter stays parked until a subclass can answer.""" + return PhysicalOutcome.UNKNOWN + + # ------------------------------------------------------------ reconcile(): BaseReconciliationService hooks + def _fetch_dlq_entries(self, config=None, **kwargs): + source = kwargs.get('dlq_source') or self.source + batch = int(resolve_config('consumer_batch_size', config, kwargs)) + return [{'id': m.id, 'idempotency_key': self._idempotency_key(m), 'tenant_id': self._tenant_of(m), + 'fingerprint': body_fingerprint(m.body), 'message': m, 'source': source} + for m in self.broker.receive_dead_letters(source, max_messages=batch, config=config)] + + def _confirm_physical_outcome(self, entry, config=None, **kwargs): + return self._confirm(entry['message'], config or {}) + + def _replay_dlq_entry(self, entry, config=None, **kwargs): + message = entry['message'] + try: + result = self._handle(message, config or {}, tenant_id=entry['tenant_id'], + idempotency_key=entry['idempotency_key']) + except Exception: + return False + if outcome_of(result) == PhysicalOutcome.COMMITTED: + entry['result'] = dict(result or {}) + return True + return False + + def _settle_dlq_entry(self, entry, outcome, config=None, **kwargs): + # settled dead letters leave the DLQ; UNRESOLVED and FAILED ones stay for the next run or a person + self.settlements.append((entry['id'], outcome)) + if outcome in ('COMMITTED', 'REPLAYED', 'SKIPPED_REPLAY'): + self.broker.ack_dead_letter(entry['source'], entry['message'], config) + + +# ------------------------------------------------------------------ the outbox +class BaseOutboxStore(ABC): + """Port (v1.7.0): the transactional outbox. A record is written in the same transaction + as the business change (add()), and published later by a sweeper, so a message is + published if and only if the change committed. Record: {id, destination, body, + attributes, tenant_id, idempotency_key}.""" + + @abstractmethod + def add(self, record: Dict[str, Any], config: Dict[str, Any] = None, **kwargs) -> str: + """Store a pending record; return its id. kwargs may carry the business transaction.""" + + @abstractmethod + def pending(self, limit: int = 100, config: Dict[str, Any] = None, **kwargs) -> List[Dict[str, Any]]: + """The oldest pending records first.""" + + @abstractmethod + def mark_published(self, record_id: str, config: Dict[str, Any] = None, **kwargs) -> bool: + """Pending -> published. kwargs: fencing_token (a stale sweeper's write is refused).""" + + +class InMemoryOutboxStore(BaseOutboxStore): + def __init__(self) -> None: + self._lock, self._records, self._fence = threading.Lock(), {}, -1 + + def add(self, record, config=None, **kwargs): + rid = record.get('id') or str(uuid.uuid4()) + with self._lock: + self._records[rid] = {**record, 'id': rid, 'status': 'PENDING', 'created_at': time.time()} + return rid + + def pending(self, limit=100, config=None, **kwargs): + with self._lock: + rows = sorted((r for r in self._records.values() if r['status'] == 'PENDING'), key=lambda r: r['created_at']) + return [dict(r) for r in rows[:limit]] + + def mark_published(self, record_id, config=None, **kwargs): + token = kwargs.get('fencing_token') + with self._lock: + if token is not None: + if token < self._fence: + return False + self._fence = token + record = self._records.get(record_id) + if record is None or record['status'] != 'PENDING': + return False + record['status'] = 'PUBLISHED' + return True + + +class BrokerOutboxSweeper(BaseOutboxSweeper): + """BaseOutboxSweeper over an outbox store and a broker. A record is published with its + tenant_id and idempotency_key as attributes, so the consumer's claim makes a second + publication (a sweeper that died after publish, before mark_published) harmless.""" + + def __init__(self, outbox: BaseOutboxStore, broker: BaseMessageBroker, lock=None) -> None: + super().__init__(lock=lock) + self.outbox, self.broker = outbox, broker + + def _fetch_pending_records(self, config=None, **kwargs): + return self.outbox.pending(int(kwargs.get('batch_size') or resolve_config('consumer_batch_size', config, kwargs)), + config) + + def _publish_pending_record(self, record, config=None, **kwargs): + if not record.get('tenant_id') or not record.get('idempotency_key'): + return False # never publish what a consumer could not deduplicate + attributes = {**(record.get('attributes') or {}), 'tenant_id': str(record['tenant_id']), + 'idempotency_key': str(record['idempotency_key'])} + self.broker.publish(record['destination'], record['body'], attributes, config) + return self.outbox.mark_published(record['id'], config, fencing_token=kwargs.get('fencing_token')) diff --git a/reference/e2a/pipeline.py b/reference/e2a/pipeline.py index 44506ff..12d5377 100644 --- a/reference/e2a/pipeline.py +++ b/reference/e2a/pipeline.py @@ -80,6 +80,10 @@ def run_pipeline(self, config: Dict[str, Any] = None, raise ValueError('Secret scan failed: ' + '; '.join(leaks[:5])) bindings = kwargs.get('bindings') or config.get('bindings') if bindings is not None: + literal = list(getattr(bindings.profile, 'literal_secrets', [])) + if literal: # v1.7.0 (N2): secrets come from ${VAR} or a secret store + raise ValueError(f"Secret scan failed: profile '{bindings.profile.name}' sets secret key(s) " + f"{literal} to a literal value") report['bindings'] = bindings.verify() report['binding_warnings'] = list(getattr(bindings, 'warnings', [])) report['stages']['binding_verification'] = 'PASSED' diff --git a/reference/e2a/port_contract.py b/reference/e2a/port_contract.py index 63c77b1..85c224c 100644 --- a/reference/e2a/port_contract.py +++ b/reference/e2a/port_contract.py @@ -28,7 +28,7 @@ from .core import ClaimStatus, PhysicalOutcome __all__ = ['FakeClock', 'check_idempotency_store', 'check_transactional_store', 'check_lease', - 'check_checkpoint_store', 'check_llm_service', 'check_all'] + 'check_checkpoint_store', 'check_llm_service', 'check_message_broker', 'check_all'] VALUE = {'amount': '100.25', 'count': 3, 'ratio': 0.5, 'ok': True, 'none': None, 'nested': {'items': ['a', 'b'], 'unicode': 'Zürich €'}} @@ -357,13 +357,207 @@ async def stream(svc): return r.passed -def check_all(idempotency=None, transactional=None, lease=None, checkpoint=None, llm_service=None) -> dict: +# ------------------------------------------------------------ messaging (v1.7.0) +def _poll(fn, timeout: float, until=lambda got: bool(got), step: float = 0.2): + """Call fn() until until(result) or the timeout; a broker delivers asynchronously.""" + import time as _t + end, got = _t.monotonic() + timeout, fn() + while not until(got) and _t.monotonic() < end: + _t.sleep(step) + got = fn() + return got + + +def check_message_broker(make: Callable[[], tuple], timeout: float = 15.0, store_factory: Callable = None) -> List[str]: + """The behaviour every BaseMessageBroker adapter keeps, and the exactly-once guarantees + IdempotentConsumer builds on it. ``make()`` returns ``(broker, destination, source)`` for a + FRESH, EMPTY destination whose messages arrive at ``source`` (a queue: the same name; a + topic: its subscription). ``store_factory()`` returns an empty BaseIdempotencyStore + (default: the in-memory reference store). Waits are bounded by ``timeout`` seconds.""" + from .messaging import BrokerOutboxSweeper, IdempotentConsumer, InMemoryOutboxStore + if store_factory is None: + from e2a_inmemory_adapters import InMemoryIdempotencyStore as store_factory + r = _Run('message broker') + quiet = min(timeout, 2.0) + + def take(broker, source, n=1, dead=False): + def once(): + got = (broker.receive_dead_letters(source, max_messages=n) if dead + else broker.receive(source, max_messages=n, wait_seconds=1.0)) + return got + return _poll(once, timeout) + + # 1. a message arrives intact, and an acknowledged one never comes back + broker, dest, source = make() + attrs = {'tenant_id': 't1', 'idempotency_key': _key('k'), 'kind': 'order.created'} + mid = broker.publish(dest, VALUE, attrs) + got = take(broker, source) + r.check('a published message is received', len(got) == 1, str(got)) + m = got[0] + r.check('the body round-trips exactly', m.body == VALUE, str(m.body)) + r.check('the attributes round-trip', all(m.attributes.get(k) == v for k, v in attrs.items()), str(m.attributes)) + r.check('the message has an id, and the publish returned one', bool(m.id) and bool(mid)) + broker.ack(source, m) + r.check('an acknowledged message is not delivered again', + not _poll(lambda: broker.receive(source, max_messages=1, wait_seconds=0.5), quiet)) + + # 2. a nacked message is delivered again, with a higher count where the broker counts + broker, dest, source = make() + broker.publish(dest, {'n': 1}, {'tenant_id': 't1', 'idempotency_key': _key('k')}) + first = take(broker, source)[0] + broker.nack(source, first, 0) + again = take(broker, source) + r.check('a nacked message is delivered again', len(again) == 1 and again[0].body == {'n': 1}, str(again)) + if first.delivery_count is not None: + r.check('the delivery count grows', (again[0].delivery_count or 0) > first.delivery_count, + f'{first.delivery_count} -> {again[0].delivery_count}') + broker.ack(source, again[0]) + + # 3. dead letters: out of the source, into its dead letters with the reason, then settled + broker, dest, source = make() + broker.publish(dest, {'bad': True}, {'tenant_id': 't1', 'idempotency_key': _key('k')}) + m = take(broker, source)[0] + broker.dead_letter(source, m, 'poison: schema') + r.check('a dead-lettered message leaves the source', + not _poll(lambda: broker.receive(source, max_messages=1, wait_seconds=0.5), quiet)) + dl = take(broker, source, dead=True) + r.check('it is among the dead letters, with the reason', + len(dl) == 1 and dl[0].body == {'bad': True} and 'poison' in dl[0].attributes.get('dead_letter_reason', ''), + str(dl)) + broker.ack_dead_letter(source, dl[0]) + r.check('a settled dead letter is gone', + not _poll(lambda: broker.receive_dead_letters(source, max_messages=1), quiet)) + + # 4-7. the consumer's guarantees, on this broker + class Effects(IdempotentConsumer): + def __init__(self, broker, store, source, behaviour, truth=PhysicalOutcome.UNKNOWN): + super().__init__(broker, store, source) + self.behaviour, self.truth, self.effects = behaviour, truth, [] + + def _handle(self, message, config, tenant_id, idempotency_key): + return self.behaviour(self, message) + + def _confirm(self, message, config): + return self.truth + + def commit(c, message): + c.effects.append(message.body) + return {'physical_success': True} + + def drain(consumer): + reports = [] + for _ in range(4): + got = _poll(lambda: consumer.consume(config={'consumer_retry_delay_seconds': 0}), quiet, + until=lambda rep: rep['received'] > 0) + reports.append(got) + if not got['received']: + break + return reports + + broker, dest, source = make() + key = _key('dup') + for _ in range(2): # the same message, delivered twice + broker.publish(dest, {'order': 'A'}, {'tenant_id': 't1', 'idempotency_key': key}) + consumer = Effects(broker, store_factory(), source, commit) + reports = drain(consumer) + r.check('a duplicate delivery has its effect once', consumer.effects == [{'order': 'A'}], str(consumer.effects)) + r.check('the duplicate is acknowledged as a duplicate', sum(x['duplicates'] for x in reports) == 1, str(reports)) + + class LostAck: # the ack never reaches the broker + def __init__(self, inner): + self.inner, self.lost = inner, 0 + + def __getattr__(self, name): + return getattr(self.inner, name) + + def ack(self, source, message, config=None): + if not self.lost: + self.lost += 1 + return self.inner.nack(source, message, 0, config) + return self.inner.ack(source, message, config) + + inner, dest, source = make() + inner.publish(dest, {'order': 'B'}, {'tenant_id': 't1', 'idempotency_key': _key('ack')}) + consumer = Effects(LostAck(inner), store_factory(), source, commit) + reports = drain(consumer) + r.check('a lost acknowledgement does not repeat the effect', consumer.effects == [{'order': 'B'}], + f'{consumer.effects} {reports}') + + def poison(c, message): + raise ValueError('amount is not a number') + + class Settles: # counts the dead letters settled + def __init__(self, inner): + self.inner, self.settled = inner, 0 + + def __getattr__(self, name): + return getattr(self.inner, name) + + def ack_dead_letter(self, source, message, config=None): + self.settled += 1 + return self.inner.ack_dead_letter(source, message, config) + + inner, dest, source = make() + broker = Settles(inner) + broker.publish(dest, {'order': 'C'}, {'tenant_id': 't1', 'idempotency_key': _key('poison')}) + consumer = Effects(broker, store_factory(), source, poison) + reports = drain(consumer) + r.check('a poison message is dead-lettered with its reason, not retried', + sum(x['dead_lettered'] for x in reports) == 1 and sum(x['retried'] for x in reports) == 0 + and 'ValueError' in ' '.join(sum((x['dead_letter_reasons'] for x in reports), [])), str(reports)) + rec = _poll(lambda: consumer.reconcile({}), timeout, until=lambda rep: bool(consumer.settlements)) + r.check('reconcile sees the poison message and never replays it', + rec['replayed'] == 0 and consumer.settlements[-1][1] == 'SKIPPED_FAILED_FINAL', f'{rec} {consumer.settlements}') + r.check('the poison message is not settled: it stays among the dead letters for a person', + broker.settled == 0, str(rec)) + + def unknown(c, message): + if c.behaviour_state.get('first', True): + c.behaviour_state['first'] = False + return {'physical_success': False, 'outcome': PhysicalOutcome.UNKNOWN} + c.effects.append(message.body) + return {'physical_success': True} + + for truth, expect_effects, label in ((PhysicalOutcome.NOT_COMMITTED, 1, 'replayed once'), + (PhysicalOutcome.COMMITTED, 0, 'settled without a replay')): + inner, dest, source = make() + broker = Settles(inner) + broker.publish(dest, {'order': 'D'}, {'tenant_id': 't1', 'idempotency_key': _key('unknown')}) + consumer = Effects(broker, store_factory(), source, unknown, truth) + consumer.behaviour_state = {} + drain(consumer) + r.check('an UNKNOWN outcome is dead-lettered, never retried blindly', consumer.effects == [], + str(consumer.effects)) + rec = _poll(lambda: consumer.reconcile({}), timeout, until=lambda rep: bool(consumer.settlements)) + r.check(f'reconcile confirms first: {truth.value} is {label}', len(consumer.effects) == expect_effects, + f'{consumer.effects} {rec} {consumer.settlements}') + r.check(f'the dead letter is settled ({truth.value})', broker.settled == 1, + f'{consumer.settlements}') + + # 8. the outbox: published once, with the attributes a consumer deduplicates on + broker, dest, source = make() + outbox = InMemoryOutboxStore() + outbox.add({'destination': dest, 'body': {'order': 'E'}, 'tenant_id': 't1', 'idempotency_key': _key('outbox')}) + outbox.add({'destination': dest, 'body': {'order': 'F'}, 'tenant_id': None, 'idempotency_key': None}) + sweep = BrokerOutboxSweeper(outbox, broker).sweep({}) + r.check('the outbox publishes a record a consumer can deduplicate, and refuses one it could not', + sweep['published'] == 1 and len(sweep['failed_keys']) == 1, str(sweep)) + r.check('a second sweep publishes nothing new', BrokerOutboxSweeper(outbox, broker).sweep({})['published'] == 0) + consumer = Effects(broker, store_factory(), source, commit) + drain(consumer) + r.check('the published record is consumed once', consumer.effects == [{'order': 'E'}], str(consumer.effects)) + return r.passed + + +def check_all(idempotency=None, transactional=None, lease=None, checkpoint=None, llm_service=None, + message_broker=None) -> dict: """Run the suites for the factories given; returns {port: [passed scenario names]}.""" out = {} for name, fn, make in (('idempotency', check_idempotency_store, idempotency), ('transactional', check_transactional_store, transactional), ('lease', check_lease, lease), ('checkpoint', check_checkpoint_store, checkpoint), - ('llm_service', check_llm_service, llm_service)): + ('llm_service', check_llm_service, llm_service), + ('message_broker', check_message_broker, message_broker)): if make is not None: out[name] = fn(make) return out diff --git a/reference/e2a/ports.py b/reference/e2a/ports.py index ac04a94..6f31d89 100644 --- a/reference/e2a/ports.py +++ b/reference/e2a/ports.py @@ -549,33 +549,46 @@ def reconcile(self, config: Dict[str, Any] = None, correlation_id: str = None, cr = store.claim(key, config, tenant_id=tenant, owner=correlation_id, fingerprint=entry.get('fingerprint'), lease_seconds=lease_s, recover=True) if cr.status != ClaimStatus.CLAIMED: - continue # already reconciled, being worked, or finally failed + # already reconciled, being worked, or finally failed + self._settle_dlq_entry(entry, f'SKIPPED_{cr.status.value}', config, **kwargs) + continue owned = True outcome = self._confirm_physical_outcome(entry, config, fencing_token=fencing, **kwargs) if outcome == PhysicalOutcome.COMMITTED: store.complete(key, entry.get('result') or {'outcome': 'COMMITTED'}, config, tenant_id=tenant, owner=correlation_id) + self._settle_dlq_entry(entry, 'COMMITTED', config, **kwargs) continue # the effect already happened — nothing to replay if outcome == PhysicalOutcome.UNKNOWN: store.fail(key, retryable=False, config=config, tenant_id=tenant, owner=correlation_id, outcome=PhysicalOutcome.UNKNOWN) unresolved.append(entry.get('id')) + self._settle_dlq_entry(entry, 'UNRESOLVED', config, **kwargs) continue if self._replay_dlq_entry(entry, config, fencing_token=fencing, **kwargs): replayed += 1 if owned: store.complete(key, entry.get('result') or {'outcome': 'COMMITTED'}, config, tenant_id=tenant, owner=correlation_id) + self._settle_dlq_entry(entry, 'REPLAYED', config, **kwargs) else: exhausted.append(entry.get('id')) if owned: store.fail(key, retryable=True, config=config, tenant_id=tenant, owner=correlation_id) + self._settle_dlq_entry(entry, 'EXHAUSTED', config, **kwargs) return {'status': 'RECONCILED', 'correlation_id': correlation_id, 'replayed': replayed, 'failed_keys': exhausted, 'unresolved': unresolved} finally: if self.lock and handle: self.lock.release(handle, config=config) + def _settle_dlq_entry(self, entry: Dict[str, Any], outcome: str, config: Dict[str, Any] = None, + **kwargs) -> None: + """Hook (v1.7.0): one entry's decision, so the DLQ can be settled: COMMITTED (confirmed, + not replayed), REPLAYED, UNRESOLVED (outcome UNKNOWN), EXHAUSTED (replay failed), or + SKIPPED_ (REPLAY: reconciled before; IN_PROGRESS; FAILED_FINAL; ...). + Default: nothing (a DLQ that is not a broker settles itself in _replay_dlq_entry).""" + def _confirm_physical_outcome(self, entry: Dict[str, Any], config: Dict[str, Any] = None, **kwargs) -> 'PhysicalOutcome': """Hook: ask the system of record whether the entry's material diff --git a/tests/test_v170.py b/tests/test_v170.py new file mode 100644 index 0000000..508d838 --- /dev/null +++ b/tests/test_v170.py @@ -0,0 +1,164 @@ +"""v1.7.0 (design note MP-05): messaging with exactly-once effects (N1), identity at ingress +(N3), the literal-secret rule (N2), and the seams the cloud profiles bind.""" +import json +import uuid + +import pytest + +import e2a +from e2a import (BrokerMessage, ClaimMapIdentityResolver, IdempotentConsumer, IdentityRefused, InMemoryBroker, + PhysicalOutcome, bindings_from_profile) +from e2a.bindings import SEAMS, Profile +from e2a.port_contract import check_message_broker +from e2a_inmemory_adapters import InMemoryIdempotencyStore + + +def fresh(): + broker, queue = InMemoryBroker(), f'q-{uuid.uuid4().hex[:6]}' + return broker, queue, queue + + +# ------------------------------------------------------------------ N1 messaging +def test_the_reference_broker_meets_the_messaging_contract(): + passed = check_message_broker(fresh, timeout=1.5) + assert len(passed) == 25 and 'a lost acknowledgement does not repeat the effect' in passed + + +class Ledger(IdempotentConsumer): + def __init__(self, broker, store, source): + super().__init__(broker, store, source) + self.posted = [] + + def _handle(self, message, config, tenant_id, idempotency_key): + if message.body.get('amount') is None: + raise ValueError('no amount') + if message.body.get('flaky') and not self.posted: + self.posted.append('flaky-attempt') + raise ConnectionError('ledger down') + self.posted.append((tenant_id, message.body['amount'])) + return {'physical_success': True, 'outcome': PhysicalOutcome.COMMITTED} + + +def test_the_consumer_reports_every_settlement(): + broker, q, _ = fresh() + broker.publish(q, {'amount': 5}, {'tenant_id': 't1', 'idempotency_key': 'a'}) + broker.publish(q, {'amount': 5}, {'tenant_id': 't1', 'idempotency_key': 'a'}) # duplicate + broker.publish(q, {'x': 1}, {'tenant_id': 't1', 'idempotency_key': 'b'}) # poison + broker.publish(q, {'amount': 7}, {'idempotency_key': 'c'}) # no tenant + consumer = Ledger(broker, InMemoryIdempotencyStore(), q) + report = consumer.consume(config={'consumer_retry_delay_seconds': 0}) + assert report['received'] == 4 and report['processed'] == 1 and report['duplicates'] == 1 + assert report['dead_lettered'] == 2 and consumer.posted == [('t1', 5)] + assert any('missing tenant_id' in r for r in report['dead_letter_reasons']) + + +def test_a_transient_failure_is_retried_and_the_same_key_with_another_body_is_a_conflict(): + broker, q, _ = fresh() + broker.publish(q, {'amount': 9, 'flaky': True}, {'tenant_id': 't1', 'idempotency_key': 'k'}) + consumer = Ledger(broker, InMemoryIdempotencyStore(), q) + first = consumer.consume(config={'consumer_retry_delay_seconds': 0}) + second = consumer.consume(config={'consumer_retry_delay_seconds': 0}) + assert first['retried'] == 1 and second['processed'] == 1 and consumer.posted[-1] == ('t1', 9) + broker.publish(q, {'amount': 10}, {'tenant_id': 't1', 'idempotency_key': 'k'}) + third = consumer.consume() + assert third['dead_lettered'] == 1 and 'CONFLICT' in third['dead_letter_reasons'][0] + + +def test_a_message_delivered_too_often_is_dead_lettered(): + broker, q, _ = fresh() + broker.publish(q, {'amount': 1}, {'tenant_id': 't1', 'idempotency_key': 'z'}) + message = broker.receive(q)[0] + broker.nack(q, message, 0) + for _ in range(2): + broker.nack(q, broker.receive(q)[0], 0) + report = Ledger(broker, InMemoryIdempotencyStore(), q).consume(config={'consumer_max_deliveries': 3}) + assert report['dead_lettered'] == 1 and 'delivered 4 times' in report['dead_letter_reasons'][0] + + +def test_the_consumer_needs_a_source(): + with pytest.raises(ValueError, match='no source'): + Ledger(InMemoryBroker(), InMemoryIdempotencyStore(), None).consume() + + +def test_a_broker_message_reads_its_routing_attributes(): + m = BrokerMessage('1', {}, {'tenant_id': 't9', 'idempotency_key': 'k9'}) + assert (m.tenant_id, m.idempotency_key) == ('t9', 'k9') + + +# ------------------------------------------------------------------ N3 identity +COGNITO = {'tenant_id': 'custom:tenant_id', 'subject': 'sub', 'roles': 'cognito:groups', 'issuer': 'iss'} + + +def test_the_tenant_comes_from_verified_claims_through_the_claim_map(): + resolver = ClaimMapIdentityResolver({'identity_claim_map': COGNITO, + 'identity_trusted_issuers': ['https://idp.example']}) + claims = {'custom:tenant_id': 'acme', 'sub': 'u-1', 'cognito:groups': ['approver'], 'iss': 'https://idp.example'} + ident = resolver.resolve(claims, verified=True, request_tenant='acme') + assert (ident.tenant_id, ident.subject, ident.roles) == ('acme', 'u-1', ['approver']) + with pytest.raises(IdentityRefused, match='not verified'): + resolver.resolve(claims, verified=False) + with pytest.raises(IdentityRefused, match='another tenant'): + resolver.resolve(claims, verified=True, request_tenant='globex') + with pytest.raises(IdentityRefused, match='not trusted'): + resolver.resolve({**claims, 'iss': 'https://evil.example'}, verified=True) + with pytest.raises(IdentityRefused, match='no tenant'): + resolver.resolve({'sub': 'u-1', 'iss': 'https://idp.example'}, verified=True) + assert isinstance(IdentityRefused('x'), PermissionError) + + +def test_nested_claims_space_separated_roles_and_a_tenant_map(): + resolver = ClaimMapIdentityResolver({'identity_claim_map': {'tenant_id': 'tid', 'roles': 'scp'}, + 'identity_tenant_map': {'9f1c-directory': 'acme'}}) + ident = resolver.resolve({'tid': '9f1c-directory', 'scp': 'orders.read orders.write'}, verified=True) + assert ident.tenant_id == 'acme' and ident.roles == ['orders.read', 'orders.write'] + nested = ClaimMapIdentityResolver({'identity_claim_map': {'tenant_id': 'app_metadata.tenant'}}) + assert nested.resolve({'app_metadata': {'tenant': 'globex'}}, verified=True).tenant_id == 'globex' + + +# ------------------------------------------------------------------ N2 literal secrets; seams +def test_a_literal_secret_in_a_profile_is_found_and_fails_the_gate(tmp_path): + (tmp_path / 'base.json').write_text(json.dumps({'name': 'base', 'provider': 'x', 'config': { + 'auth_token': 'sk-literal-value', 'timeout': 9}})) + (tmp_path / 'child.json').write_text(json.dumps({'name': 'child', 'provider': 'x', 'extends': str(tmp_path / 'base.json'), + 'config': {'temperature': 0.1}})) + (tmp_path / 'clean.json').write_text(json.dumps({'name': 'clean', 'provider': 'x', 'extends': str(tmp_path / 'base.json'), + 'config': {'auth_token': '${TOOL_AUTH_TOKEN}'}})) + assert Profile.load(str(tmp_path / 'base.json')).literal_secrets == ['auth_token'] + assert Profile.load(str(tmp_path / 'child.json')).literal_secrets == ['auth_token'] # inherited + assert Profile.load(str(tmp_path / 'clean.json')).literal_secrets == [] # overridden by ${VAR} + + bindings = bindings_from_profile(Profile.load(str(tmp_path / 'child.json'))) + bindings.verify() + assert any('auth_token' in w for w in bindings.warnings) + + class Gate(e2a.BasePipeline): + conformance_paths = () + + def _run_static_security_scans(self, config, **kwargs): + return True + + def _execute_test_suite(self, config, **kwargs): + return True + + def _run_rag_eval(self, config, **kwargs): + return 1.0 + + def _compile_artifact(self, artifact_name, config, **kwargs): + return True + + def _run_dynamic_security_checks(self, config, **kwargs): + return True + + def _execute_deployment_rollout(self, strategy, config, **kwargs): + return {} + + report = Gate().run_pipeline({'secret_scan_paths': []}, bindings=bindings) + assert report['status'] == 'FAILED' and 'literal value' in report['error'] + + +def test_the_infrastructure_seams_reach_the_config(): + for seam in ('message_broker', 'outbox_store', 'identity_resolver', 'transactional_store', 'lease'): + assert seam in SEAMS + broker = InMemoryBroker() + b = bindings_from_profile(Profile(name='p', provider='x'), overrides={'message_broker': broker}) + assert b.config()['message_broker'] is broker