From 3bf85b2e8907adeaf5808fb7f1d28404fac1127a Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 13 Jun 2026 14:20:37 -0500 Subject: [PATCH 1/4] refactor: reduce duplicated source code --- src/lib.rs | 1 - src/outbox/commit.rs | 6 +- src/outbox_worker/mod.rs | 31 +- src/outbox_worker/outbox_source.rs | 5 +- src/outbox_worker/store.rs | 362 +++++------------ src/postgres_repo/mod.rs | 81 +--- src/read_model/session.rs | 51 +-- src/sqlite_repo/mod.rs | 81 +--- src/sqlx_repo/read_model.rs | 67 +++- tests/bomberman/main.rs | 5 +- tests/distributed_read_model/main.rs | 322 ++++----------- tests/microsvc/convention.rs | 9 +- .../outbox.rs | 292 ++++++-------- tests/sourced_snapshot/main.rs | 4 +- tests/sql_lock_manager/main.rs | 107 +++-- tests/todos/main.rs | 366 ++++++------------ tests/transport_conformance/mod.rs | 117 +++--- 17 files changed, 651 insertions(+), 1256 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 09c8cab..d5a1bb3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -87,7 +87,6 @@ pub use outbox_worker::{ OutboxClaimRef, OutboxPublishFailureAction, OutboxPublisher, - OutboxStore, OutboxWorker, ProcessOneResult, }; diff --git a/src/outbox/commit.rs b/src/outbox/commit.rs index 2dbdf3b..94eaec0 100644 --- a/src/outbox/commit.rs +++ b/src/outbox/commit.rs @@ -208,7 +208,7 @@ impl AggregateRepository { #[cfg(test)] mod tests { use super::*; - use crate::{sourced, AggregateBuilder, Entity, HashMapRepository, OutboxStore}; + use crate::{sourced, AggregateBuilder, AsyncOutboxStore, Entity, HashMapRepository}; use std::sync::Mutex; #[derive(Default)] @@ -267,7 +267,7 @@ mod tests { assert!(receipt.has_outbox_messages()); assert_eq!(receipt.outbox_message_ids(), ["msg-1".to_string()]); - let pending = repo.repo().outbox_store().pending().unwrap(); + let pending = repo.repo().outbox_store().pending_async().await.unwrap(); assert_eq!(pending.len(), 1); assert_eq!(pending[0].id(), "msg-1"); } @@ -404,7 +404,7 @@ mod tests { ); // 2) outbox row present (pending — no bus attached here) - let pending = repo.repo().outbox_store().pending().unwrap(); + let pending = repo.repo().outbox_store().pending_async().await.unwrap(); assert_eq!(pending.len(), 1); assert_eq!(pending[0].id(), "evt-c1"); diff --git a/src/outbox_worker/mod.rs b/src/outbox_worker/mod.rs index 98cc9d1..7ea3078 100644 --- a/src/outbox_worker/mod.rs +++ b/src/outbox_worker/mod.rs @@ -2,15 +2,8 @@ //! //! This module provides the worker infrastructure for processing outbox messages. //! -//! There are two drain paths: -//! - **Production / extension point**: the async `OutboxDispatcher` + -//! `AsyncMessagePublisher` (`BusPublisher` over a `Bus`), wired by -//! `service.with_bus(bus)`. -//! - **Dev/test**: the synchronous `OutboxWorker` + `OutboxPublisher` + -//! `LogPublisher` trio — no async runtime, no real transport. -//! //! Items: -//! - `OutboxStore` - Store operations for claiming and completing messages +//! - `AsyncOutboxStore` - Store operations for claiming and completing messages //! - `OutboxDispatcher` / `BusPublisher` - the async production drain path //! - `OutboxWorker` - synchronous dev/test message processor //! - `OutboxPublisher` - synchronous dev/test publish trait @@ -26,24 +19,16 @@ //! ## Example //! //! ```ignore -//! use distributed::{ClaimOutboxMessages, OutboxClaimRef, OutboxStore, OutboxWorker, LogPublisher}; +//! use distributed::{AsyncOutboxStore, ClaimOutboxMessages, OutboxClaimRef}; //! use std::time::Duration; //! -//! // Claim pending messages //! let worker_id = "worker-1"; -//! let messages = outbox.claim(ClaimOutboxMessages::new(worker_id, 10, Duration::from_secs(60)))?; -//! -//! // Process with a worker -//! let mut worker = OutboxWorker::new(LogPublisher::default()).with_worker_id(worker_id); -//! for mut msg in messages { +//! let messages = outbox +//! .claim_async(ClaimOutboxMessages::new(worker_id, 10, Duration::from_secs(60))) +//! .await?; +//! for msg in messages { //! let claim = OutboxClaimRef::from_message(&msg)?; -//! let result = worker.process_message(&mut msg)?; -//! if result.completed { -//! outbox.complete(&claim)?; -//! } else if result.released || result.failed { -//! let error = msg.last_error.as_deref().unwrap_or("publish failed"); -//! outbox.record_failure(&claim, error, 3)?; -//! } +//! outbox.complete_async(&claim).await?; //! } //! ``` @@ -64,7 +49,7 @@ pub use publisher::{LogPublisher, LogPublisherError, OutboxPublisher}; #[cfg(any(feature = "postgres", feature = "sqlite"))] pub(crate) use store::ensure_active_claim; pub use store::{ - AsyncOutboxStore, ClaimOutboxMessages, OutboxClaimRef, OutboxPublishFailureAction, OutboxStore, + AsyncOutboxStore, ClaimOutboxMessages, OutboxClaimRef, OutboxPublishFailureAction, }; // Worker diff --git a/src/outbox_worker/outbox_source.rs b/src/outbox_worker/outbox_source.rs index e7745ef..ac4ff43 100644 --- a/src/outbox_worker/outbox_source.rs +++ b/src/outbox_worker/outbox_source.rs @@ -176,7 +176,7 @@ mod tests { use crate::bus::{run_source, RunOptions}; use crate::microsvc::Service; use crate::{ - CommitBatch, HashMapRepository, OutboxMessage, OutboxMessageStatus, OutboxStore, + AsyncOutboxStore, CommitBatch, HashMapRepository, OutboxMessage, OutboxMessageStatus, TransactionalCommit, }; use serde_json::json; @@ -218,8 +218,7 @@ mod tests { ] .into_iter() .find(|status| { - store - .messages_by_status(status.clone()) + block_on(store.messages_by_status_async(status.clone())) .unwrap() .iter() .any(|m| m.id() == id) diff --git a/src/outbox_worker/store.rs b/src/outbox_worker/store.rs index 8494e7d..a4511dc 100644 --- a/src/outbox_worker/store.rs +++ b/src/outbox_worker/store.rs @@ -93,41 +93,6 @@ impl OutboxClaimRef { } } -/// Store capability for claiming and updating durable outbox messages. -pub trait OutboxStore: Send + Sync { - /// Return all outbox messages with the given status. - fn messages_by_status( - &self, - status: OutboxMessageStatus, - ) -> Result, RepositoryError>; - - /// Return all pending outbox messages. - fn pending(&self) -> Result, RepositoryError> { - self.messages_by_status(OutboxMessageStatus::Pending) - } - - /// Claim pending outbox messages for processing. - fn claim(&self, request: ClaimOutboxMessages) -> Result, RepositoryError>; - - /// Mark an outbox message as completed if it is still claimed by this worker. - fn complete(&self, claim: &OutboxClaimRef) -> Result<(), RepositoryError>; - - /// Release an outbox message if it is still claimed by this worker. - fn release(&self, claim: &OutboxClaimRef, error: &str) -> Result<(), RepositoryError>; - - /// Mark an outbox message as permanently failed if it is still claimed by this worker. - fn fail(&self, claim: &OutboxClaimRef, error: &str) -> Result<(), RepositoryError>; - - /// Record a publish failure for a claimed message, releasing it for retry - /// or permanently failing it when the attempt ceiling is reached. - fn record_failure( - &self, - claim: &OutboxClaimRef, - error: &str, - max_attempts: u32, - ) -> Result; -} - /// Async store capability for claiming and updating durable outbox messages. pub trait AsyncOutboxStore: Send + Sync { fn messages_by_status_async( @@ -171,7 +136,17 @@ pub trait AsyncOutboxStore: Send + Sync { claim: &'a OutboxClaimRef, error: &'a str, max_attempts: u32, - ) -> impl Future> + Send + 'a; + ) -> impl Future> + Send + 'a { + async move { + if claim.attempt >= max_attempts { + self.fail_async(claim, error).await?; + Ok(OutboxPublishFailureAction::Failed) + } else { + self.release_async(claim, error).await?; + Ok(OutboxPublishFailureAction::Released) + } + } + } } fn outbox_state(message: &OutboxMessage) -> String { @@ -262,108 +237,6 @@ impl HashMapOutboxStore { } } -impl OutboxStore for HashMapOutboxStore { - fn messages_by_status( - &self, - status: OutboxMessageStatus, - ) -> Result, RepositoryError> { - let storage = self - .storage - .read() - .map_err(|_| RepositoryError::LockPoisoned("outbox read"))?; - - let mut messages = storage - .values() - .filter(|message| message.status == status) - .cloned() - .collect::>(); - sort_by_claim_order(&mut messages); - Ok(messages) - } - - fn claim(&self, request: ClaimOutboxMessages) -> Result, RepositoryError> { - let mut storage = self - .storage - .write() - .map_err(|_| RepositoryError::LockPoisoned("outbox write"))?; - - if request.batch_size == 0 { - return Ok(Vec::new()); - } - - let now = SystemTime::now(); - let ids = claim_order_ids(storage.values()); - let mut claimed = Vec::new(); - for id in ids { - if !request.selects(&id) { - continue; - } - - let Some(message) = storage.get_mut(&id) else { - continue; - }; - - if message.is_claimable_at(now) { - if let Some(destination) = request.destination.as_deref() { - if message.destination.as_deref() != Some(destination) { - continue; - } - } - message.claim_at(&request.worker_id, request.lease, now)?; - claimed.push(message.clone()); - } - - if claimed.len() >= request.batch_size { - break; - } - } - - Ok(claimed) - } - - fn complete(&self, claim: &OutboxClaimRef) -> Result<(), RepositoryError> { - self.update_outbox_message(&claim.message_id, |message| { - ensure_active_claim(message, Some(claim), SystemTime::now())?; - message.complete()?; - Ok(()) - }) - } - - fn release(&self, claim: &OutboxClaimRef, error: &str) -> Result<(), RepositoryError> { - self.update_outbox_message(&claim.message_id, |message| { - ensure_active_claim(message, Some(claim), SystemTime::now())?; - message.release(error.to_string())?; - Ok(()) - }) - } - - fn fail(&self, claim: &OutboxClaimRef, error: &str) -> Result<(), RepositoryError> { - self.update_outbox_message(&claim.message_id, |message| { - ensure_active_claim(message, Some(claim), SystemTime::now())?; - message.fail(error.to_string())?; - Ok(()) - }) - } - - fn record_failure( - &self, - claim: &OutboxClaimRef, - error: &str, - max_attempts: u32, - ) -> Result { - self.update_outbox_message(&claim.message_id, |message| { - ensure_active_claim(message, Some(claim), SystemTime::now())?; - if message.attempts >= max_attempts { - message.fail(error.to_string())?; - Ok(OutboxPublishFailureAction::Failed) - } else { - message.release(error.to_string())?; - Ok(OutboxPublishFailureAction::Released) - } - }) - } -} - impl AsyncOutboxStore for HashMapOutboxStore { fn messages_by_status_async( &self, @@ -373,7 +246,8 @@ impl AsyncOutboxStore for HashMapOutboxStore { let storage = self .storage .read() - .map_err(|_| RepositoryError::LockPoisoned("async outbox read"))?; + .map_err(|_| RepositoryError::LockPoisoned("outbox read"))?; + let mut messages = storage .values() .filter(|message| message.status == status) @@ -389,17 +263,17 @@ impl AsyncOutboxStore for HashMapOutboxStore { request: ClaimOutboxMessages, ) -> impl Future, RepositoryError>> + Send + 'a { async move { + let mut storage = self + .storage + .write() + .map_err(|_| RepositoryError::LockPoisoned("outbox write"))?; + if request.batch_size == 0 { return Ok(Vec::new()); } let now = SystemTime::now(); - let mut storage = self - .storage - .write() - .map_err(|_| RepositoryError::LockPoisoned("async outbox write"))?; let ids = claim_order_ids(storage.values()); - let mut claimed = Vec::new(); for id in ids { if !request.selects(&id) { @@ -410,19 +284,16 @@ impl AsyncOutboxStore for HashMapOutboxStore { continue; }; - if !message.is_claimable_at(now) { - continue; - } - - if let Some(destination) = request.destination.as_deref() { - if message.destination.as_deref() != Some(destination) { - continue; + if message.is_claimable_at(now) { + if let Some(destination) = request.destination.as_deref() { + if message.destination.as_deref() != Some(destination) { + continue; + } } + message.claim_at(&request.worker_id, request.lease, now)?; + claimed.push(message.clone()); } - message.claim_at(&request.worker_id, request.lease, now)?; - claimed.push(message.clone()); - if claimed.len() >= request.batch_size { break; } @@ -437,19 +308,11 @@ impl AsyncOutboxStore for HashMapOutboxStore { claim: &'a OutboxClaimRef, ) -> impl Future> + Send + 'a { async move { - let mut storage = self - .storage - .write() - .map_err(|_| RepositoryError::LockPoisoned("async outbox write"))?; - let message = - storage - .get_mut(&claim.message_id) - .ok_or_else(|| RepositoryError::NotFound { - id: claim.message_id.clone(), - })?; - ensure_active_claim(message, Some(claim), SystemTime::now())?; - message.complete()?; - Ok(()) + self.update_outbox_message(&claim.message_id, |message| { + ensure_active_claim(message, Some(claim), SystemTime::now())?; + message.complete()?; + Ok(()) + }) } } @@ -459,19 +322,11 @@ impl AsyncOutboxStore for HashMapOutboxStore { error: &'a str, ) -> impl Future> + Send + 'a { async move { - let mut storage = self - .storage - .write() - .map_err(|_| RepositoryError::LockPoisoned("async outbox write"))?; - let message = - storage - .get_mut(&claim.message_id) - .ok_or_else(|| RepositoryError::NotFound { - id: claim.message_id.clone(), - })?; - ensure_active_claim(message, Some(claim), SystemTime::now())?; - message.release(error.to_string())?; - Ok(()) + self.update_outbox_message(&claim.message_id, |message| { + ensure_active_claim(message, Some(claim), SystemTime::now())?; + message.release(error.to_string())?; + Ok(()) + }) } } @@ -481,47 +336,11 @@ impl AsyncOutboxStore for HashMapOutboxStore { error: &'a str, ) -> impl Future> + Send + 'a { async move { - let mut storage = self - .storage - .write() - .map_err(|_| RepositoryError::LockPoisoned("async outbox write"))?; - let message = - storage - .get_mut(&claim.message_id) - .ok_or_else(|| RepositoryError::NotFound { - id: claim.message_id.clone(), - })?; - ensure_active_claim(message, Some(claim), SystemTime::now())?; - message.fail(error.to_string())?; - Ok(()) - } - } - - fn record_failure_async<'a>( - &'a self, - claim: &'a OutboxClaimRef, - error: &'a str, - max_attempts: u32, - ) -> impl Future> + Send + 'a { - async move { - let mut storage = self - .storage - .write() - .map_err(|_| RepositoryError::LockPoisoned("async outbox write"))?; - let message = - storage - .get_mut(&claim.message_id) - .ok_or_else(|| RepositoryError::NotFound { - id: claim.message_id.clone(), - })?; - ensure_active_claim(message, Some(claim), SystemTime::now())?; - if message.attempts >= max_attempts { + self.update_outbox_message(&claim.message_id, |message| { + ensure_active_claim(message, Some(claim), SystemTime::now())?; message.fail(error.to_string())?; - Ok(OutboxPublishFailureAction::Failed) - } else { - message.release(error.to_string())?; - Ok(OutboxPublishFailureAction::Released) - } + Ok(()) + }) } } } @@ -561,11 +380,12 @@ mod tests { let store = repo.outbox_store(); let claimed = store - .claim(ClaimOutboxMessages::new( + .claim_async(ClaimOutboxMessages::new( "worker-2", 1, Duration::from_secs(60), )) + .await .unwrap(); assert_eq!(claimed.len(), 1); @@ -589,11 +409,12 @@ mod tests { let store = repo.outbox_store(); let claimed = store - .claim(ClaimOutboxMessages::new( + .claim_async(ClaimOutboxMessages::new( "worker-2", 1, Duration::from_secs(60), )) + .await .unwrap(); assert!(claimed.is_empty()); @@ -614,11 +435,12 @@ mod tests { let claimed = repo .outbox_store() - .claim(ClaimOutboxMessages::new( + .claim_async(ClaimOutboxMessages::new( "worker-1", 1, Duration::from_secs(60), )) + .await .unwrap(); assert_eq!(claimed[0].id(), "msg-z"); @@ -645,11 +467,12 @@ mod tests { let claimed = repo .outbox_store() - .claim(ClaimOutboxMessages::for_ids( + .claim_async(ClaimOutboxMessages::for_ids( "worker-1", vec!["msg-b".to_string(), "msg-c".to_string()], Duration::from_secs(60), )) + .await .unwrap(); let mut claimed_ids = claimed @@ -675,11 +498,12 @@ mod tests { // not an error. let claimed = repo .outbox_store() - .claim(ClaimOutboxMessages::for_ids( + .claim_async(ClaimOutboxMessages::for_ids( "worker-1", vec!["msg-a".to_string(), "missing".to_string()], Duration::from_secs(60), )) + .await .unwrap(); assert!(claimed.is_empty()); @@ -720,25 +544,25 @@ mod tests { let worker_a = thread::spawn(move || { barrier_a.wait(); - store_a - .claim(ClaimOutboxMessages::new( - "worker-a", - 1, - Duration::from_secs(60), - )) - .unwrap() - .len() + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(store_a.claim_async(ClaimOutboxMessages::new( + "worker-a", + 1, + Duration::from_secs(60), + ))) + .unwrap() + .len() }); let worker_b = thread::spawn(move || { barrier_b.wait(); - store_b - .claim(ClaimOutboxMessages::new( - "worker-b", - 1, - Duration::from_secs(60), - )) - .unwrap() - .len() + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(store_b.claim_async(ClaimOutboxMessages::new( + "worker-b", + 1, + Duration::from_secs(60), + ))) + .unwrap() + .len() }); barrier.wait(); @@ -758,14 +582,18 @@ mod tests { let store = repo.outbox_store(); let claimed = store - .claim(ClaimOutboxMessages::new( + .claim_async(ClaimOutboxMessages::new( "worker-1", 1, Duration::from_secs(60), )) + .await .unwrap(); let claim = OutboxClaimRef::from_message(&claimed[0]).unwrap(); - let action = store.record_failure(&claim, "first failure", 2).unwrap(); + let action = store + .record_failure_async(&claim, "first failure", 2) + .await + .unwrap(); assert_eq!(action, OutboxPublishFailureAction::Released); let stored = load_message(&repo, &id); @@ -774,14 +602,18 @@ mod tests { assert_eq!(stored.last_error.as_deref(), Some("first failure")); let claimed = store - .claim(ClaimOutboxMessages::new( + .claim_async(ClaimOutboxMessages::new( "worker-1", 1, Duration::from_secs(60), )) + .await .unwrap(); let claim = OutboxClaimRef::from_message(&claimed[0]).unwrap(); - let action = store.record_failure(&claim, "second failure", 2).unwrap(); + let action = store + .record_failure_async(&claim, "second failure", 2) + .await + .unwrap(); assert_eq!(action, OutboxPublishFailureAction::Failed); let stored = load_message(&repo, &id); @@ -790,8 +622,8 @@ mod tests { assert_eq!(stored.last_error.as_deref(), Some("second failure")); } - #[test] - fn missing_message_updates_return_not_found() { + #[tokio::test] + async fn missing_message_updates_return_not_found() { let store = HashMapOutboxStore { storage: Default::default(), }; @@ -803,9 +635,13 @@ mod tests { }; let is_missing = |err: RepositoryError| matches!(&err, RepositoryError::NotFound { id } if id == "missing"); - assert!(is_missing(store.complete(&claim).unwrap_err())); - assert!(is_missing(store.release(&claim, "error").unwrap_err())); - assert!(is_missing(store.fail(&claim, "error").unwrap_err())); + assert!(is_missing(store.complete_async(&claim).await.unwrap_err())); + assert!(is_missing( + store.release_async(&claim, "error").await.unwrap_err() + )); + assert!(is_missing( + store.fail_async(&claim, "error").await.unwrap_err() + )); } #[tokio::test] @@ -816,15 +652,16 @@ mod tests { let store = repo.outbox_store(); let claimed = store - .claim(ClaimOutboxMessages::new( + .claim_async(ClaimOutboxMessages::new( "worker-1", 1, Duration::from_secs(60), )) + .await .unwrap(); let mut claim = OutboxClaimRef::from_message(&claimed[0]).unwrap(); claim.worker_id = "worker-2".into(); - let err = store.complete(&claim).unwrap_err(); + let err = store.complete_async(&claim).await.unwrap_err(); assert!(matches!(err, RepositoryError::InvalidState { .. })); let mut expired = OutboxMessage::create("msg-2", "Event", b"{}".to_vec()).unwrap(); @@ -834,7 +671,7 @@ mod tests { let expired_id = store_message(&repo, expired).await; let expired = load_message(&repo, &expired_id); let claim = OutboxClaimRef::from_message(&expired).unwrap(); - let err = store.complete(&claim).unwrap_err(); + let err = store.complete_async(&claim).await.unwrap_err(); assert!(matches!(err, RepositoryError::InvalidState { .. })); } @@ -846,27 +683,29 @@ mod tests { let store = repo.outbox_store(); let claimed = store - .claim(ClaimOutboxMessages::new( + .claim_async(ClaimOutboxMessages::new( "worker-1", 1, Duration::from_secs(60), )) + .await .unwrap(); let stale_claim = OutboxClaimRef::from_message(&claimed[0]).unwrap(); - store.release(&stale_claim, "retry").unwrap(); + store.release_async(&stale_claim, "retry").await.unwrap(); let claimed = store - .claim(ClaimOutboxMessages::new( + .claim_async(ClaimOutboxMessages::new( "worker-1", 1, Duration::from_secs(60), )) + .await .unwrap(); let current_claim = OutboxClaimRef::from_message(&claimed[0]).unwrap(); - let err = store.complete(&stale_claim).unwrap_err(); + let err = store.complete_async(&stale_claim).await.unwrap_err(); assert!(matches!(err, RepositoryError::InvalidState { .. })); - store.complete(¤t_claim).unwrap(); + store.complete_async(¤t_claim).await.unwrap(); } #[tokio::test] @@ -877,16 +716,17 @@ mod tests { let store = repo.outbox_store(); let claimed = store - .claim(ClaimOutboxMessages::new( + .claim_async(ClaimOutboxMessages::new( "worker-1", 1, Duration::from_secs(60), )) + .await .unwrap(); let claim = OutboxClaimRef::from_message(&claimed[0]).unwrap(); - store.complete(&claim).unwrap(); + store.complete_async(&claim).await.unwrap(); - let err = store.complete(&claim).unwrap_err(); + let err = store.complete_async(&claim).await.unwrap_err(); assert!(matches!(err, RepositoryError::InvalidState { .. })); } } diff --git a/src/postgres_repo/mod.rs b/src/postgres_repo/mod.rs index 512e0e9..5ec666a 100644 --- a/src/postgres_repo/mod.rs +++ b/src/postgres_repo/mod.rs @@ -21,12 +21,11 @@ use crate::entity::EventRecord; use crate::outbox::{OutboxMessage, OutboxMessageStatus}; use crate::outbox_worker::{ ensure_active_claim, AsyncOutboxStore, ClaimOutboxMessages, OutboxClaimRef, - OutboxPublishFailureAction, }; use crate::read_model::{ - validate_key, ColumnDef, ColumnType, ReadModelAdapterCapabilities, ReadModelCommitOutcome, - ReadModelError, ReadModelIncludeRows, ReadModelLoadGraph, ReadModelLoadRequest, - ReadModelQueryCapabilities, ReadModelWritePlan, RowValue, + ColumnDef, ColumnType, ReadModelAdapterCapabilities, ReadModelCommitOutcome, ReadModelError, + ReadModelLoadGraph, ReadModelLoadRequest, ReadModelQueryCapabilities, ReadModelWritePlan, + RowValue, }; use crate::repository::{ reject_duplicate_outbox_messages, reject_duplicate_streams, @@ -37,9 +36,8 @@ use crate::repository::{ }; use crate::snapshot::SnapshotRecord; use crate::sqlx_repo::read_model::{ - apply_read_model_write_plan_in_tx, begin_read_model_tx, commit_read_model_tx, - empty_string_as_none, load_relational_row_by_key, load_relationship_rows, quote_identifier, - remember_read_model_schemas, resolve_registered_read_model_schemas, + apply_read_model_write_plan_in_tx, commit_read_model_write_plan, empty_string_as_none, + load_read_model_graph, quote_identifier, remember_read_model_schemas, sql_read_model_capabilities, validate_sql_write_plan, }; use crate::sqlx_repo::{ @@ -497,13 +495,7 @@ impl ReadModelWritePlanStore for PostgresRepository { &self, plan: ReadModelWritePlan, ) -> impl Future> + Send + '_ { - async move { - validate_sql_write_plan(&plan)?; - let mut tx = begin_read_model_tx(&self.pool).await?; - let outcome = apply_read_model_write_plan_in_tx(&mut tx, plan).await?; - commit_read_model_tx(tx).await?; - Ok(outcome) - } + async move { commit_read_model_write_plan(&self.pool, plan).await } } } @@ -517,36 +509,13 @@ impl RelationalReadModelQueryStore for PostgresRepository { request: ReadModelLoadRequest, ) -> impl Future> + Send + '_ { async move { - request.validate_for_query_capabilities(&self.read_model_query_capabilities())?; - - let (root_schema, include_specs) = - resolve_registered_read_model_schemas(&self.read_model_schemas, &request)?; - validate_key(&root_schema, &request.key)?; - - let Some(root) = - load_relational_row_by_key(&self.pool, &root_schema, &request.key).await? - else { - return Ok(ReadModelLoadGraph::default()); - }; - - let mut includes = BTreeMap::new(); - for spec in include_specs { - let loaded_rows = - load_relationship_rows(&self.pool, &root_schema, &root.data, &spec).await?; - includes.insert( - spec.name, - ReadModelIncludeRows { - relationship: spec.relationship, - target_schema: spec.target_schema, - rows: loaded_rows, - }, - ); - } - - Ok(ReadModelLoadGraph { - root: Some(root), - includes, - }) + load_read_model_graph( + &self.pool, + &self.read_model_schemas, + request, + self.read_model_query_capabilities(), + ) + .await } } } @@ -810,30 +779,6 @@ impl AsyncOutboxStore for PostgresOutboxStore { .await } } - - fn record_failure_async<'a>( - &'a self, - claim: &'a OutboxClaimRef, - error: &'a str, - max_attempts: u32, - ) -> impl Future> + Send + 'a { - async move { - let message = outbox_message_by_id_pool(&self.pool, &claim.message_id) - .await? - .ok_or_else(|| RepositoryError::NotFound { - id: claim.message_id.clone(), - })?; - ensure_active_claim(&message, Some(claim), SystemTime::now())?; - - if message.attempts >= max_attempts { - self.fail_async(claim, error).await?; - Ok(OutboxPublishFailureAction::Failed) - } else { - self.release_async(claim, error).await?; - Ok(OutboxPublishFailureAction::Released) - } - } - } } impl SnapshotStore for PostgresRepository { diff --git a/src/read_model/session.rs b/src/read_model/session.rs index b42a19d..f09e566 100644 --- a/src/read_model/session.rs +++ b/src/read_model/session.rs @@ -215,30 +215,18 @@ pub enum ReadModelMutation { impl ReadModelMutation { pub fn table_name(&self) -> &str { - match self { - ReadModelMutation::UpsertRow(mutation) => mutation.schema.table_name.as_str(), - ReadModelMutation::PatchRow(mutation) => mutation.schema.table_name.as_str(), - ReadModelMutation::DeleteRow(mutation) => mutation.schema.table_name.as_str(), - } + self.schema().table_name.as_str() } pub fn lock_key(&self) -> String { + format!("{}:{}", self.table_name(), key_fingerprint(self.key())) + } + + fn key(&self) -> &RowKey { match self { - ReadModelMutation::UpsertRow(mutation) => format!( - "{}:{}", - mutation.schema.table_name, - key_fingerprint(&mutation.key) - ), - ReadModelMutation::PatchRow(mutation) => format!( - "{}:{}", - mutation.schema.table_name, - key_fingerprint(&mutation.key) - ), - ReadModelMutation::DeleteRow(mutation) => format!( - "{}:{}", - mutation.schema.table_name, - key_fingerprint(&mutation.key) - ), + ReadModelMutation::UpsertRow(mutation) => &mutation.key, + ReadModelMutation::PatchRow(mutation) => &mutation.key, + ReadModelMutation::DeleteRow(mutation) => &mutation.key, } } @@ -290,23 +278,12 @@ impl ReadModelMutation { } fn sort_key(&self) -> String { - match self { - ReadModelMutation::UpsertRow(mutation) => format!( - "1|{}|{}", - mutation.schema.table_name, - key_fingerprint(&mutation.key) - ), - ReadModelMutation::PatchRow(mutation) => format!( - "2|{}|{}", - mutation.schema.table_name, - key_fingerprint(&mutation.key) - ), - ReadModelMutation::DeleteRow(mutation) => format!( - "3|{}|{}", - mutation.schema.table_name, - key_fingerprint(&mutation.key) - ), - } + format!( + "{}|{}|{}", + self.operation_rank(), + self.table_name(), + key_fingerprint(self.key()) + ) } } diff --git a/src/sqlite_repo/mod.rs b/src/sqlite_repo/mod.rs index 214b3c2..1b1f589 100644 --- a/src/sqlite_repo/mod.rs +++ b/src/sqlite_repo/mod.rs @@ -20,12 +20,11 @@ use crate::entity::{Entity, EventRecord}; use crate::outbox::{OutboxMessage, OutboxMessageStatus}; use crate::outbox_worker::{ ensure_active_claim, AsyncOutboxStore, ClaimOutboxMessages, OutboxClaimRef, - OutboxPublishFailureAction, }; use crate::read_model::{ - validate_key, ColumnDef, ColumnType, ReadModelAdapterCapabilities, ReadModelCommitOutcome, - ReadModelError, ReadModelIncludeRows, ReadModelLoadGraph, ReadModelLoadRequest, - ReadModelQueryCapabilities, ReadModelWritePlan, RowValue, + ColumnDef, ColumnType, ReadModelAdapterCapabilities, ReadModelCommitOutcome, ReadModelError, + ReadModelLoadGraph, ReadModelLoadRequest, ReadModelQueryCapabilities, ReadModelWritePlan, + RowValue, }; use crate::repository::{ reject_duplicate_outbox_messages, reject_duplicate_streams, @@ -36,9 +35,8 @@ use crate::repository::{ }; use crate::snapshot::SnapshotRecord; use crate::sqlx_repo::read_model::{ - apply_read_model_write_plan_in_tx, begin_read_model_tx, commit_read_model_tx, - empty_string_as_none, load_relational_row_by_key, load_relationship_rows, quote_identifier, - remember_read_model_schemas, resolve_registered_read_model_schemas, + apply_read_model_write_plan_in_tx, commit_read_model_write_plan, empty_string_as_none, + load_read_model_graph, quote_identifier, remember_read_model_schemas, sql_read_model_capabilities, validate_sql_write_plan, }; use crate::sqlx_repo::{ @@ -478,13 +476,7 @@ impl ReadModelWritePlanStore for SqliteRepository { &self, plan: ReadModelWritePlan, ) -> impl Future> + Send + '_ { - async move { - validate_sql_write_plan(&plan)?; - let mut tx = begin_read_model_tx(&self.pool).await?; - let outcome = apply_read_model_write_plan_in_tx(&mut tx, plan).await?; - commit_read_model_tx(tx).await?; - Ok(outcome) - } + async move { commit_read_model_write_plan(&self.pool, plan).await } } } @@ -498,36 +490,13 @@ impl RelationalReadModelQueryStore for SqliteRepository { request: ReadModelLoadRequest, ) -> impl Future> + Send + '_ { async move { - request.validate_for_query_capabilities(&self.read_model_query_capabilities())?; - - let (root_schema, include_specs) = - resolve_registered_read_model_schemas(&self.read_model_schemas, &request)?; - validate_key(&root_schema, &request.key)?; - - let Some(root) = - load_relational_row_by_key(&self.pool, &root_schema, &request.key).await? - else { - return Ok(ReadModelLoadGraph::default()); - }; - - let mut includes = BTreeMap::new(); - for spec in include_specs { - let loaded_rows = - load_relationship_rows(&self.pool, &root_schema, &root.data, &spec).await?; - includes.insert( - spec.name, - ReadModelIncludeRows { - relationship: spec.relationship, - target_schema: spec.target_schema, - rows: loaded_rows, - }, - ); - } - - Ok(ReadModelLoadGraph { - root: Some(root), - includes, - }) + load_read_model_graph( + &self.pool, + &self.read_model_schemas, + request, + self.read_model_query_capabilities(), + ) + .await } } } @@ -833,30 +802,6 @@ impl AsyncOutboxStore for SqliteOutboxStore { .await } } - - fn record_failure_async<'a>( - &'a self, - claim: &'a OutboxClaimRef, - error: &'a str, - max_attempts: u32, - ) -> impl Future> + Send + 'a { - async move { - let message = outbox_message_by_id_pool(&self.pool, &claim.message_id) - .await? - .ok_or_else(|| RepositoryError::NotFound { - id: claim.message_id.clone(), - })?; - ensure_active_claim(&message, Some(claim), SystemTime::now())?; - - if message.attempts >= max_attempts { - self.fail_async(claim, error).await?; - Ok(OutboxPublishFailureAction::Failed) - } else { - self.release_async(claim, error).await?; - Ok(OutboxPublishFailureAction::Released) - } - } - } } impl SnapshotStore for SqliteRepository { diff --git a/src/sqlx_repo/read_model.rs b/src/sqlx_repo/read_model.rs index 7687026..8866e0f 100644 --- a/src/sqlx_repo/read_model.rs +++ b/src/sqlx_repo/read_model.rs @@ -12,6 +12,7 @@ //! one-line-per-method impls — so the SQL-building logic exists once rather than //! being mirrored (and risking silent drift) in `postgres_repo`/`sqlite_repo`. +use std::collections::BTreeMap; use std::sync::RwLock; use sqlx::{Database, Encode, Executor, IntoArguments, QueryBuilder, Row, Transaction, Type}; @@ -19,9 +20,10 @@ use sqlx::{Database, Encode, Executor, IntoArguments, QueryBuilder, Row, Transac use crate::read_model::{ key_fingerprint, validate_key, validate_row_values, ColumnDef, DeleteRowMutation, ExpectedVersion, PatchMode, PatchRowMutation, ReadModelAdapterCapabilities, - ReadModelCommitOutcome, ReadModelError, ReadModelLoadRequest, ReadModelMutation, - ReadModelSchema, ReadModelWritePlan, RelationshipDef, RelationshipKind, RowKey, RowMutation, - RowValue, RowValues, RowWriteMode, Versioned, + ReadModelCommitOutcome, ReadModelError, ReadModelIncludeRows, ReadModelLoadGraph, + ReadModelLoadRequest, ReadModelMutation, ReadModelQueryCapabilities, ReadModelSchema, + ReadModelWritePlan, RelationshipDef, RelationshipKind, RowKey, RowMutation, RowValue, + RowValues, RowWriteMode, Versioned, }; use crate::table::TableSchemaRegistry; @@ -409,6 +411,24 @@ pub(crate) async fn commit_read_model_tx( .map_err(|err| read_model_storage_error(DB::BACKEND, "commit transaction", err)) } +pub(crate) async fn commit_read_model_write_plan( + pool: &sqlx::Pool, + plan: ReadModelWritePlan, +) -> Result +where + DB: SqlxReadModelBackend, + for<'c> &'c mut ::Connection: Executor<'c, Database = DB>, + ::Arguments: IntoArguments, + for<'q> i64: Encode<'q, DB> + Type + sqlx::Decode<'q, DB>, + for<'r> &'r str: sqlx::ColumnIndex<::Row>, +{ + validate_sql_write_plan(&plan)?; + let mut tx = begin_read_model_tx(pool).await?; + let outcome = apply_read_model_write_plan_in_tx(&mut tx, plan).await?; + commit_read_model_tx(tx).await?; + Ok(outcome) +} + pub(crate) async fn apply_read_model_write_plan_in_tx( tx: &mut Transaction<'_, DB>, plan: ReadModelWritePlan, @@ -981,6 +1001,47 @@ where } } +pub(crate) async fn load_read_model_graph( + pool: &sqlx::Pool, + schemas: &RwLock, + request: ReadModelLoadRequest, + capabilities: ReadModelQueryCapabilities, +) -> Result +where + DB: SqlxReadModelBackend, + for<'c> &'c sqlx::Pool: Executor<'c, Database = DB>, + ::Arguments: IntoArguments, + for<'q> i64: Encode<'q, DB> + Type + sqlx::Decode<'q, DB>, + for<'r> &'r str: sqlx::ColumnIndex<::Row>, +{ + request.validate_for_query_capabilities(&capabilities)?; + + let (root_schema, include_specs) = resolve_registered_read_model_schemas(schemas, &request)?; + validate_key(&root_schema, &request.key)?; + + let Some(root) = load_relational_row_by_key(pool, &root_schema, &request.key).await? else { + return Ok(ReadModelLoadGraph::default()); + }; + + let mut includes = BTreeMap::new(); + for spec in include_specs { + let rows = load_relationship_rows(pool, &root_schema, &root.data, &spec).await?; + includes.insert( + spec.name, + ReadModelIncludeRows { + relationship: spec.relationship, + target_schema: spec.target_schema, + rows, + }, + ); + } + + Ok(ReadModelLoadGraph { + root: Some(root), + includes, + }) +} + async fn load_has_many_rows( pool: &sqlx::Pool, root_schema: &ReadModelSchema, diff --git a/tests/bomberman/main.rs b/tests/bomberman/main.rs index 15ac990..8938964 100644 --- a/tests/bomberman/main.rs +++ b/tests/bomberman/main.rs @@ -208,10 +208,11 @@ async fn player_killed_by_bomb() { .contains(&"player:p2".to_string())); // Verify outbox message was created (PlayerKilled) - use distributed::{OutboxMessageStatus, OutboxStore}; + use distributed::{AsyncOutboxStore, OutboxMessageStatus}; let pending = repo2 .outbox_store() - .messages_by_status(OutboxMessageStatus::Pending) + .messages_by_status_async(OutboxMessageStatus::Pending) + .await .unwrap(); assert!(!pending.is_empty()); assert_eq!(pending[0].event_type, "player.killed"); diff --git a/tests/distributed_read_model/main.rs b/tests/distributed_read_model/main.rs index a6b67b8..6eb9779 100644 --- a/tests/distributed_read_model/main.rs +++ b/tests/distributed_read_model/main.rs @@ -1,18 +1,3 @@ -//! Distributed read-model example over an event-driven seat checkout flow. -//! -//! Deployment shape: -//! - the **checkout saga service** owns the `CheckoutSaga` aggregate and its outbox; -//! - the **seat inventory service** owns the `Seat` aggregate and its outbox; -//! - coordinator subscribers translate domain events into aggregate method calls; -//! - the **projection service** consumes the same bus and reconciles normalized -//! `checkouts`, `checkout_steps`, and `seats` rows in a shared read store; -//! - a **query service** reads the projected graph through primary-key loads -//! plus `has_many` / `belongs_to` relationship includes. -//! -//! Commands are present-tense requests. Events are past-tense facts. The saga -//! is an aggregate that records checkout-process facts and emits its own events; -//! it does not directly issue commands to the seat aggregate. - mod checkout; mod checkout_saga_service; #[cfg(feature = "postgres")] @@ -85,8 +70,6 @@ fn unique_id(prefix: &str) -> String { format!("{prefix}-{nanos}-{sequence}") } -// Used by the gated sqlite/postgres flow tests; the matrix uses the per-step -// helpers directly, so this is unused in a default (no-feature) build. #[allow(dead_code)] async fn run_persistent_checkout_flow( checkout_repo: R, @@ -133,36 +116,13 @@ async fn run_persistent_checkout_flow( .await .expect("checkout read model load should succeed") .expect("checkout should be projected"); - assert_eq!(checkout.seat_id, ids.seat_id); - assert_eq!(checkout.seat_category, ids.category); - assert_eq!(checkout.status, CHECKOUT_SEAT_RESERVED); - assert_eq!(checkout.screen_message, SEAT_RESERVED_MESSAGE); - assert_eq!( - checkout - .seat - .as_ref() - .expect("checkout should include seat") - .status, - SEAT_RESERVED - ); - - let mut steps: Vec<&str> = checkout - .steps - .iter() - .map(|step| step.step.as_str()) - .collect(); - steps.sort(); - assert_eq!( - steps, - vec!["seat_reservation_completed", "seat_reserved", "started"] - ); + assert_checkout_screen(&checkout, &ids); let seat = load_seat(&read_repo, &ids.seat_id) .await .expect("seat read model load should succeed") .expect("seat should be projected"); - assert_eq!(seat.status, SEAT_RESERVED); - assert_eq!(seat.checkout_id, ids.checkout_id); + assert_projected_seat(&seat, &ids); let loaded_checkout = checkout_repo .clone() @@ -460,9 +420,36 @@ where .map(|root| SeatView::from_row(root.data).expect("seat row should hydrate"))) } -/// Bridge a HashMap-backed service's pending outbox onto the async bus — the -/// new-transport equivalent of the old `OutboxWorkerThread`: claim → publish → -/// complete, so each event is forwarded exactly once. +fn assert_checkout_screen(checkout: &CheckoutView, ids: &FlowIds) { + assert_eq!(checkout.seat_id, ids.seat_id); + assert_eq!(checkout.seat_category, ids.category); + assert_eq!(checkout.status, CHECKOUT_SEAT_RESERVED); + assert_eq!(checkout.screen_message, SEAT_RESERVED_MESSAGE); + assert_eq!( + checkout + .seat + .as_ref() + .expect("checkout should include seat") + .status, + SEAT_RESERVED + ); + let mut steps: Vec<&str> = checkout + .steps + .iter() + .map(|step| step.step.as_str()) + .collect(); + steps.sort(); + assert_eq!( + steps, + vec!["seat_reservation_completed", "seat_reserved", "started"] + ); +} + +fn assert_projected_seat(seat: &SeatView, ids: &FlowIds) { + assert_eq!(seat.status, SEAT_RESERVED); + assert_eq!(seat.checkout_id, ids.checkout_id); +} + async fn publish_pending_outbox( outbox: &distributed::HashMapOutboxStore, bus: &distributed::bus::InMemoryBus, @@ -494,14 +481,15 @@ async fn publish_pending_outbox( } } -/// The original gold-standard choreography, now driven over the async -/// `InMemoryBus` instead of the legacy `InMemoryQueue` / `OutboxWorkerThread` / -/// `bus::Subscribable` wiring. Same services, same projection + query, same -/// assertions — only the transport changed. #[tokio::test] async fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { use distributed::bus::InMemoryBus; + let ids = FlowIds { + checkout_id: "checkout-1".to_string(), + seat_id: "A-7".to_string(), + category: "balcony".to_string(), + }; let checkout_store = HashMapRepository::new(); let checkout_service = checkout_saga_service::service(checkout_store.clone().queued().aggregate()); @@ -514,13 +502,12 @@ async fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { let bus = InMemoryBus::new(); - // Commands: add the seat, start the checkout (each writes its own outbox). dispatch( &seat_service, seat_command::ADD, AddSeat { - seat_id: "A-7".to_string(), - category: "balcony".to_string(), + seat_id: ids.seat_id.clone(), + category: ids.category.clone(), }, ) .await; @@ -528,15 +515,13 @@ async fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { &checkout_service, checkout_command::START, StartCheckout { - checkout_id: "checkout-1".to_string(), - seat_id: "A-7".to_string(), - seat_category: "balcony".to_string(), + checkout_id: ids.checkout_id.clone(), + seat_id: ids.seat_id.clone(), + seat_category: ids.category.clone(), }, ) .await; - // Hop 1: SeatAdded + CheckoutStarted reach the bus; the projection records the - // opening state and the seat service reacts to the checkout by reserving. publish_pending_outbox(&seat_store.outbox_store(), &bus).await; publish_pending_outbox(&checkout_store.outbox_store(), &bus).await; bus.subscribe(projection_svc.clone(), RunOptions::idempotent()) @@ -546,7 +531,6 @@ async fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { .await .expect("seat service reacts to the started checkout"); - // Hop 2: SeatReserved reaches the bus; the saga records it; projection updates. publish_pending_outbox(&seat_store.outbox_store(), &bus).await; bus.subscribe(projection_svc.clone(), RunOptions::idempotent()) .await @@ -555,70 +539,46 @@ async fn seat_checkout_saga_reserves_seat_and_projects_user_screen() { .await .expect("saga records the seat reservation"); - // Hop 3: SeatReservationCompleted reaches the bus; the projection finalizes. publish_pending_outbox(&checkout_store.outbox_store(), &bus).await; bus.subscribe(projection_svc.clone(), RunOptions::idempotent()) .await .expect("projection drains the completion"); let checkout = query_service - .checkout_screen("checkout-1") + .checkout_screen(&ids.checkout_id) .await .expect("checkout query should succeed") .expect("checkout should be projected"); - assert_eq!(checkout.seat_id, "A-7"); - assert_eq!(checkout.seat_category, "balcony"); - assert_eq!(checkout.status, CHECKOUT_SEAT_RESERVED); - assert_eq!(checkout.screen_message, SEAT_RESERVED_MESSAGE); - assert_eq!( - checkout - .seat - .as_ref() - .expect("checkout should include seat") - .status, - SEAT_RESERVED - ); - - let mut steps: Vec<&str> = checkout - .steps - .iter() - .map(|step| step.step.as_str()) - .collect(); - steps.sort(); - assert_eq!( - steps, - vec!["seat_reservation_completed", "seat_reserved", "started"] - ); + assert_checkout_screen(&checkout, &ids); let seat = query_service - .seat("A-7") + .seat(&ids.seat_id) .await .expect("seat query should succeed") .expect("seat should be projected"); - assert_eq!(seat.status, SEAT_RESERVED); - assert_eq!(seat.checkout_id, "checkout-1"); + assert_projected_seat(&seat, &ids); let checkout_saga = checkout_store .clone() .queued() .aggregate::() - .peek("checkout-1") + .peek(&ids.checkout_id) .await .unwrap() .unwrap(); assert_eq!(checkout_saga.status, CHECKOUT_SEAT_RESERVED); - assert_eq!(checkout_saga.reserved_seat_id, "A-7"); + assert_eq!(checkout_saga.reserved_seat_id, ids.seat_id); let seat = seat_store .clone() .queued() .aggregate::() - .peek("A-7") + .peek(&ids.seat_id) .await .unwrap() .unwrap(); assert_eq!(seat.status, SEAT_RESERVED); - assert_eq!(seat.checkout_id, "checkout-1"); + assert_eq!(seat.checkout_id, ids.checkout_id); } #[cfg(feature = "sqlite")] @@ -757,21 +717,12 @@ async fn checkout_commands_can_be_grpc_service() { assert_eq!(saga.status, checkout::CHECKOUT_STARTED); } -// =================================================================== -// Transport × persistence matrix -// -// The same seat-checkout scenario over every async bus transport and every -// persistence backend. No sync path: the domain flow, projection, and query run -// on the async repository `R`, and the events travel over the `Bus` facade `B`. -// =================================================================== - use std::collections::HashMap as StdHashMap; use std::sync::{Arc as StdArc, Mutex as StdMutex}; use distributed::bus::{Bus, BusConsumer, RunOptions}; use distributed::microsvc::{Message, MessageKind}; -/// The four checkout events in flow (causal) order, by CloudEvent/event type. const FLOW_EVENT_TYPES: [&str; 4] = [ seat_event::ADDED, checkout_event::STARTED, @@ -779,7 +730,6 @@ const FLOW_EVENT_TYPES: [&str; 4] = [ checkout_event::SEAT_RESERVATION_COMPLETED, ]; -/// Messages the transport delivered to the projection sink: (name, id, payload). type Collected = StdArc)>>>; fn record_message(collected: &Collected, message: &Message) { @@ -790,8 +740,6 @@ fn record_message(collected: &Collected, message: &Message) { )); } -/// A subscriber service that records every checkout event it receives — the -/// transport sink the bus drains into. Subscribes to all four event names. fn build_collector() -> (StdArc>, Collected) { let collected: Collected = StdArc::new(StdMutex::new(Vec::new())); let (c1, c2, c3, c4) = ( @@ -824,10 +772,6 @@ fn build_collector() -> (StdArc>, Collected) { (StdArc::new(service), collected) } -/// Generic end-to-end matrix cell: run the seat-checkout domain flow + read-model -/// projection + query on persistence `repo`, routing the events over transport -/// `bus`. `collector`/`collected` are the bus's projection sink (the caller binds -/// the subscription first for transports that require it, e.g. RabbitMQ). async fn run_checkout_over_bus( bus: B, collector: StdArc>, @@ -845,7 +789,6 @@ async fn run_checkout_over_bus( + Sync + 'static, { - // 1. Domain flow on the persistence backend → the four causal events. let seat_added = add_seat(&repo, &ids.seat_id, &ids.category).await; let checkout_started = start_checkout(&repo, &ids.checkout_id, &ids.seat_id, &ids.category).await; @@ -858,7 +801,6 @@ async fn run_checkout_over_bus( reservation_completed, ]; - // 2. Publish every event over the transport. for event in &events { let message = Message::new( event.event_type.clone(), @@ -871,16 +813,13 @@ async fn run_checkout_over_bus( .expect("event should publish over the bus"); } - // 3. Drain the transport into the projection sink. bus.subscribe(collector, RunOptions::idempotent()) .await .expect("subscriber should drain the bus"); - // 4-5. Project the transport-delivered events in causal order, then assert. project_and_assert_checkout(&repo, &ids, &delivered_map(&collected)).await; } -/// Collapse the recorded deliveries into a `name -> (id, payload)` map. fn delivered_map(collected: &Collected) -> StdHashMap)> { collected .lock() @@ -890,9 +829,6 @@ fn delivered_map(collected: &Collected) -> StdHashMap)> .collect() } -/// Project the events the transport delivered (in causal order) into `repo`'s -/// read models, then query the graph and assert the user-facing checkout screen. -/// Shared by every transport cell (pull buses and the Knative HTTP path). async fn project_and_assert_checkout( repo: &R, ids: &FlowIds, @@ -913,35 +849,13 @@ async fn project_and_assert_checkout( .await .expect("checkout read model load should succeed") .expect("checkout should be projected"); - assert_eq!(checkout.seat_id, ids.seat_id); - assert_eq!(checkout.seat_category, ids.category); - assert_eq!(checkout.status, CHECKOUT_SEAT_RESERVED); - assert_eq!(checkout.screen_message, SEAT_RESERVED_MESSAGE); - assert_eq!( - checkout - .seat - .as_ref() - .expect("checkout should include seat") - .status, - SEAT_RESERVED - ); - let mut steps: Vec<&str> = checkout - .steps - .iter() - .map(|step| step.step.as_str()) - .collect(); - steps.sort(); - assert_eq!( - steps, - vec!["seat_reservation_completed", "seat_reserved", "started"] - ); + assert_checkout_screen(&checkout, ids); let seat = load_seat(repo, &ids.seat_id) .await .expect("seat read model load should succeed") .expect("seat should be projected"); - assert_eq!(seat.status, SEAT_RESERVED); - assert_eq!(seat.checkout_id, ids.checkout_id); + assert_projected_seat(&seat, ids); } fn matrix_ids(tag: &str) -> FlowIds { @@ -952,23 +866,28 @@ fn matrix_ids(tag: &str) -> FlowIds { } } -/// In-memory persistence × in-memory transport — the always-on matrix cell. +async fn run_matrix_cell(bus: B, repo: R, tag: &str) +where + B: Bus + BusConsumer, + R: Clone + + GetStream + + ReadModelWritePlanStore + + RelationalReadModelQueryStore + + TransactionalCommit + + Send + + Sync + + 'static, +{ + let (collector, collected) = build_collector(); + run_checkout_over_bus(bus, collector, collected, repo, matrix_ids(tag)).await; +} + #[tokio::test] async fn matrix_in_memory_persistence_over_in_memory_bus() { use distributed::bus::InMemoryBus; - let (collector, collected) = build_collector(); - run_checkout_over_bus( - InMemoryBus::new(), - collector, - collected, - inmem_matrix_repo(), - matrix_ids("inmem-inmem"), - ) - .await; + run_matrix_cell(InMemoryBus::new(), inmem_matrix_repo(), "inmem-inmem").await; } -// ---- Persistence fixtures (read-model schemas registered/bootstrapped) ---- - fn inmem_matrix_repo() -> HashMapRepository { let repo = HashMapRepository::new(); register_schemas(repo.model_store()).expect("read-model schemas should register"); @@ -987,12 +906,6 @@ async fn sqlite_matrix_repo() -> SqliteRepository { repo } -// ---- Knative (HTTP / CloudEvents) transport cell ---- -// -// Knative produce = POST CloudEvents to a broker-ingress; consume = the platform -// delivers them over HTTP to `cloud_events_router`. Here a local router serves the -// projection sink, so the same scenario runs over the Knative transport with no -// broker. (HTTP/gRPC command ingress is this same Knative surface.) #[cfg(feature = "http")] async fn run_checkout_over_knative(repo: R, ids: FlowIds) where @@ -1019,7 +932,6 @@ where .expect("knative ingress should serve"); }); - // events_broker "" + namespace "" => POST to the router root ("/"). let bus = KnativeBus::new(format!("http://{addr}"), "", "matrix-source", "", ""); let seat_added = add_seat(&repo, &ids.seat_id, &ids.category).await; @@ -1048,23 +960,14 @@ where server.abort(); } -// =================== Matrix cells =================== -// -// Transport axis: InMemoryBus, NatsBus, RabbitBus, KafkaBus, PostgresBus, Knative. -// Persistence axis: HashMapRepository, SqliteRepository, PostgresRepository. -// Broker/DB cells skip when their env var is unset. - #[cfg(feature = "sqlite")] #[tokio::test] async fn matrix_sqlite_persistence_over_in_memory_bus() { use distributed::bus::InMemoryBus; - let (collector, collected) = build_collector(); - run_checkout_over_bus( + run_matrix_cell( InMemoryBus::new(), - collector, - collected, sqlite_matrix_repo().await, - matrix_ids("sqlite-inmem"), + "sqlite-inmem", ) .await; } @@ -1106,13 +1009,10 @@ async fn matrix_in_memory_persistence_over_nats_bus() { return; } let ns = unique_id("ns").to_lowercase(); - let (collector, collected) = build_collector(); - run_checkout_over_bus( + run_matrix_cell( nats_matrix_bus(&ns).await, - collector, - collected, inmem_matrix_repo(), - matrix_ids("inmem-nats"), + "inmem-nats", ) .await; } @@ -1124,13 +1024,10 @@ async fn matrix_sqlite_persistence_over_nats_bus() { return; } let ns = unique_id("ns").to_lowercase(); - let (collector, collected) = build_collector(); - run_checkout_over_bus( + run_matrix_cell( nats_matrix_bus(&ns).await, - collector, - collected, sqlite_matrix_repo().await, - matrix_ids("sqlite-nats"), + "sqlite-nats", ) .await; } @@ -1219,13 +1116,10 @@ async fn matrix_in_memory_persistence_over_kafka_bus() { return; } let ns = unique_id("ns"); - let (collector, collected) = build_collector(); - run_checkout_over_bus( + run_matrix_cell( kafka_matrix_bus(&ns).await, - collector, - collected, inmem_matrix_repo(), - matrix_ids("inmem-kafka"), + "inmem-kafka", ) .await; } @@ -1237,13 +1131,10 @@ async fn matrix_sqlite_persistence_over_kafka_bus() { return; } let ns = unique_id("ns"); - let (collector, collected) = build_collector(); - run_checkout_over_bus( + run_matrix_cell( kafka_matrix_bus(&ns).await, - collector, - collected, sqlite_matrix_repo().await, - matrix_ids("sqlite-kafka"), + "sqlite-kafka", ) .await; } @@ -1263,15 +1154,7 @@ async fn matrix_in_memory_persistence_over_postgres_bus() { let bus_pool = schema.repository().await.pool().clone(); let bus = PostgresBus::new(bus_pool).group("matrix"); bus.ensure_tables().await.expect("postgres bus tables"); - let (collector, collected) = build_collector(); - run_checkout_over_bus( - bus, - collector, - collected, - inmem_matrix_repo(), - matrix_ids("inmem-pgbus"), - ) - .await; + run_matrix_cell(bus, inmem_matrix_repo(), "inmem-pgbus").await; } #[cfg(feature = "postgres")] @@ -1291,19 +1174,9 @@ async fn matrix_postgres_persistence_over_in_memory_bus() { repo.bootstrap_table_schema_for_dev(®istry) .await .expect("read-model schema should bootstrap"); - let (collector, collected) = build_collector(); - run_checkout_over_bus( - InMemoryBus::new(), - collector, - collected, - repo, - matrix_ids("pg-inmem"), - ) - .await; + run_matrix_cell(InMemoryBus::new(), repo, "pg-inmem").await; } -// ---- Remaining matrix cells: Postgres persistence + Postgres-bus pairings ---- - #[cfg(feature = "postgres")] async fn postgres_matrix_repo() -> Option<( postgres::PostgresTestSchema, @@ -1342,15 +1215,7 @@ async fn matrix_sqlite_persistence_over_postgres_bus() { let Some(bus) = postgres_matrix_bus().await else { return; }; - let (collector, collected) = build_collector(); - run_checkout_over_bus( - bus, - collector, - collected, - sqlite_matrix_repo().await, - matrix_ids("sqlite-pgbus"), - ) - .await; + run_matrix_cell(bus, sqlite_matrix_repo().await, "sqlite-pgbus").await; } #[cfg(feature = "postgres")] @@ -1361,8 +1226,7 @@ async fn matrix_postgres_persistence_over_postgres_bus() { else { return; }; - let (collector, collected) = build_collector(); - run_checkout_over_bus(bus, collector, collected, repo, matrix_ids("pg-pgbus")).await; + run_matrix_cell(bus, repo, "pg-pgbus").await; } #[cfg(all(feature = "postgres", feature = "nats"))] @@ -1375,15 +1239,7 @@ async fn matrix_postgres_persistence_over_nats_bus() { return; }; let ns = unique_id("ns").to_lowercase(); - let (collector, collected) = build_collector(); - run_checkout_over_bus( - nats_matrix_bus(&ns).await, - collector, - collected, - repo, - matrix_ids("pg-nats"), - ) - .await; + run_matrix_cell(nats_matrix_bus(&ns).await, repo, "pg-nats").await; } #[cfg(all(feature = "postgres", feature = "rabbitmq"))] @@ -1411,15 +1267,7 @@ async fn matrix_postgres_persistence_over_kafka_bus() { return; }; let ns = unique_id("ns"); - let (collector, collected) = build_collector(); - run_checkout_over_bus( - kafka_matrix_bus(&ns).await, - collector, - collected, - repo, - matrix_ids("pg-kafka"), - ) - .await; + run_matrix_cell(kafka_matrix_bus(&ns).await, repo, "pg-kafka").await; } #[cfg(all(feature = "postgres", feature = "http"))] diff --git a/tests/microsvc/convention.rs b/tests/microsvc/convention.rs index d6902b6..b5329e0 100644 --- a/tests/microsvc/convention.rs +++ b/tests/microsvc/convention.rs @@ -8,7 +8,7 @@ //! Registration uses the `register_handlers!` macro. use distributed::microsvc::{Service, Session}; -use distributed::{AggregateBuilder, HashMapRepository, OutboxStore, Queueable}; +use distributed::{AggregateBuilder, AsyncOutboxStore, HashMapRepository, Queueable}; use serde_json::json; use crate::handlers; @@ -108,7 +108,7 @@ async fn create_persists_outbox_message() { assert_eq!(counter.value, 0); // Outbox message was persisted - let pending = inner.outbox_store().pending().unwrap(); + let pending = inner.outbox_store().pending_async().await.unwrap(); assert_eq!(pending.len(), 1); assert_eq!(pending[0].event_type, "counter.initialized"); } @@ -136,7 +136,8 @@ async fn duplicate_create_leaves_single_outbox_message() { .repo() .inner() .outbox_store() - .pending() + .pending_async() + .await .unwrap(); assert_eq!(pending.len(), 1); } @@ -169,7 +170,7 @@ async fn increment_persists_outbox_message() { // Both outbox messages were persisted let inner = service.repo().repo().inner(); - let pending = inner.outbox_store().pending().unwrap(); + let pending = inner.outbox_store().pending_async().await.unwrap(); assert_eq!(pending.len(), 2); let mut event_types: Vec<&str> = pending.iter().map(|m| m.event_type.as_str()).collect(); event_types.sort(); diff --git a/tests/persistent_repository_conformance/outbox.rs b/tests/persistent_repository_conformance/outbox.rs index 5acf8b6..9b7eb8d 100644 --- a/tests/persistent_repository_conformance/outbox.rs +++ b/tests/persistent_repository_conformance/outbox.rs @@ -11,10 +11,6 @@ use distributed::{ use super::scenario::unique_id; use super::seat::Seat; -/// A publisher that fails its first `fail_first` publish attempts (returning a -/// retryable transport error) and succeeds thereafter, recording every message -/// id it actually delivered. Used to prove the outbox keeps a row until it is -/// genuinely delivered, and that it is delivered exactly once. struct FlakyPublisher { fail_first: usize, attempts: Mutex, @@ -66,16 +62,7 @@ where { let seat_id = unique_id("outbox-seat"); let message_id = unique_id("outbox-message"); - let mut seat = added_seat(&seat_id); - let message = OutboxMessage::create(&message_id, "seat.added", b"{}".to_vec()) - .expect("outbox message should be valid"); - - repo.clone() - .aggregate::() - .outbox(message) - .commit(&mut seat) - .await - .expect("aggregate and outbox should commit atomically"); + commit_outbox_for_seat(&repo, seat_id.clone(), &message_id, "seat.added").await; let stored = find_outbox_by_id(&outbox, &message_id) .await @@ -106,16 +93,13 @@ where R: GetStream + TransactionalCommit + Clone + Send + Sync + 'static, { let duplicate_message_id = unique_id("duplicate-outbox"); - let mut existing_seat = added_seat(&unique_id("existing-seat")); - let existing_message = - OutboxMessage::create(&duplicate_message_id, "seat.added", b"{}".to_vec()) - .expect("existing outbox message should be valid"); - repo.clone() - .aggregate::() - .outbox(existing_message) - .commit(&mut existing_seat) - .await - .expect("initial outbox commit should succeed"); + commit_outbox_for_seat( + &repo, + unique_id("existing-seat"), + &duplicate_message_id, + "seat.added", + ) + .await; let rollback_seat_id = unique_id("rollback-seat"); let rollback_identity = StreamIdentity::new(Seat::aggregate_type(), &rollback_seat_id) @@ -188,12 +172,10 @@ where .expect("winner commit should succeed"); let message_id = unique_id("rollback-outbox-message"); - let message = OutboxMessage::create(&message_id, "seat.reserved", b"{}".to_vec()) - .expect("outbox message should be valid"); let err = repo .clone() .aggregate::() - .outbox(message) + .outbox(outbox_message(&message_id, "seat.reserved")) .commit(&mut stale) .await .expect_err("stale aggregate should reject the batch"); @@ -208,33 +190,26 @@ where S: AsyncOutboxStore + Send + Sync, { let complete_message_id = unique_id("complete-outbox"); - let mut complete_seat = added_seat(&unique_id("complete-seat")); - let complete_message = - OutboxMessage::create(&complete_message_id, "seat.added", b"{}".to_vec()) - .expect("complete outbox message should be valid"); - repo.clone() - .aggregate::() - .outbox(complete_message) - .commit(&mut complete_seat) - .await - .expect("complete message should be stored"); - - let claimed = outbox - .claim_async(ClaimOutboxMessages::new( - "worker-a", - 1, - Duration::from_secs(60), - )) - .await - .expect("claim should succeed"); - assert_eq!(claimed.len(), 1); - assert_eq!(claimed[0].id(), complete_message_id); + commit_outbox_for_seat( + &repo, + unique_id("complete-seat"), + &complete_message_id, + "seat.added", + ) + .await; + + let claimed = claim_one( + &outbox, + ClaimOutboxMessages::new("worker-a", 1, Duration::from_secs(60)), + ) + .await; + assert_eq!(claimed.id(), complete_message_id); let wrong_claim = OutboxClaimRef { - message_id: claimed[0].id().to_string(), + message_id: claimed.id().to_string(), worker_id: "worker-b".into(), - leased_until: claimed[0].leased_until.expect("claim should have lease"), - attempt: claimed[0].attempts, + leased_until: claimed.leased_until.expect("claim should have lease"), + attempt: claimed.attempts, }; let stale_err = outbox .complete_async(&wrong_claim) @@ -242,7 +217,7 @@ where .expect_err("wrong worker should not complete a claim"); assert!(matches!(stale_err, RepositoryError::InvalidState { .. })); - let claim = OutboxClaimRef::from_message(&claimed[0]).expect("claim should be valid"); + let claim = OutboxClaimRef::from_message(&claimed).expect("claim should be valid"); outbox .complete_async(&claim) .await @@ -253,25 +228,20 @@ where assert_eq!(published.status, OutboxMessageStatus::Published); let retry_message_id = unique_id("retry-outbox"); - let mut retry_seat = added_seat(&unique_id("retry-seat")); - let retry_message = OutboxMessage::create(&retry_message_id, "seat.added", b"{}".to_vec()) - .expect("retry outbox message should be valid"); - repo.clone() - .aggregate::() - .outbox(retry_message) - .commit(&mut retry_seat) - .await - .expect("retry message should be stored"); - - let claimed = outbox - .claim_async(ClaimOutboxMessages::new( - "worker-r", - 1, - Duration::from_secs(60), - )) - .await - .expect("retry claim should succeed"); - let claim = OutboxClaimRef::from_message(&claimed[0]).expect("claim should be valid"); + commit_outbox_for_seat( + &repo, + unique_id("retry-seat"), + &retry_message_id, + "seat.added", + ) + .await; + + let claimed = claim_one( + &outbox, + ClaimOutboxMessages::new("worker-r", 1, Duration::from_secs(60)), + ) + .await; + let claim = OutboxClaimRef::from_message(&claimed).expect("claim should be valid"); let action = outbox .record_failure_async(&claim, "first failure", 2) .await @@ -285,20 +255,17 @@ where assert_eq!(released.attempts, 1); assert_eq!(released.last_error.as_deref(), Some("first failure")); - let claimed = outbox - .claim_async(ClaimOutboxMessages::new( - "worker-r", - 1, - Duration::from_secs(60), - )) - .await - .expect("second retry claim should succeed"); + let claimed = claim_one( + &outbox, + ClaimOutboxMessages::new("worker-r", 1, Duration::from_secs(60)), + ) + .await; let stale_err = outbox .complete_async(&claim) .await .expect_err("stale attempt should not complete a later claim"); assert!(matches!(stale_err, RepositoryError::InvalidState { .. })); - let claim = OutboxClaimRef::from_message(&claimed[0]).expect("claim should be valid"); + let claim = OutboxClaimRef::from_message(&claimed).expect("claim should be valid"); let action = outbox .record_failure_async(&claim, "second failure", 2) .await @@ -321,37 +288,25 @@ where let wanted_id = unique_id("wanted-outbox"); let other_id = unique_id("other-outbox"); for (message_id, seat_id) in [(&wanted_id, "wanted-seat"), (&other_id, "other-seat")] { - let mut seat = added_seat(&unique_id(seat_id)); - let message = OutboxMessage::create(message_id, "seat.added", b"{}".to_vec()) - .expect("outbox message should be valid"); - repo.clone() - .aggregate::() - .outbox(message) - .commit(&mut seat) - .await - .expect("message should be stored"); + commit_outbox_for_seat(&repo, unique_id(seat_id), message_id, "seat.added").await; } - // Claiming by explicit id claims only the requested row (claim order among - // an explicit id set is unspecified across backends). - let claimed = outbox - .claim_async(ClaimOutboxMessages::for_ids( + let claimed = claim_one( + &outbox, + ClaimOutboxMessages::for_ids( "immediate-worker", vec![wanted_id.clone()], Duration::from_secs(60), - )) - .await - .expect("claim by id should succeed"); - assert_eq!(claimed.len(), 1); - assert_eq!(claimed[0].id(), wanted_id); + ), + ) + .await; + assert_eq!(claimed.id(), wanted_id); - // The unrequested row remains claimable by a normal poll. let other = find_outbox_by_id(&outbox, &other_id) .await .expect("other message should exist"); assert_eq!(other.status, OutboxMessageStatus::Pending); - // A raced/missing id yields an empty claim, not an error. let empty = outbox .claim_async(ClaimOutboxMessages::for_ids( "immediate-worker", @@ -362,9 +317,6 @@ where .expect("claiming a missing id should not error"); assert!(empty.is_empty()); - // A requested id that is leased by another worker must be skipped, not - // stolen: this is the claim-safety property of the by-id path (it exercises - // the SQLite per-id conditional UPDATE and the Postgres claimability CTE). let leased = outbox .claim_async(ClaimOutboxMessages::for_ids( "worker-b", @@ -384,48 +336,25 @@ where assert_eq!(still_owned.worker_id.as_deref(), Some("immediate-worker")); } -/// Red-team crash recovery: worker A claims a row with a short lease and then -/// "crashes" — it never completes or releases the claim. Once the lease expires, -/// worker B must be able to reclaim the same row (the claim predicate treats an -/// expired in-flight row as claimable). B then completes it. Finally A's *late* -/// settle attempts (it woke up with a stale claim) must be fenced: neither -/// `complete_async` nor `release_async` from the original lease may mutate the -/// row B now owns/published. -/// -/// This proves the SQL `claimed_until <= now` expiry predicate end-to-end — -/// today only the in-memory store has a unit test for lease-expiry reclaim -/// (`src/outbox_worker/store.rs::claim_includes_expired_in_flight_messages`). -/// -/// Note: lease expiry is inherently wall-clock based, so this scenario uses a -/// short real lease and a sleep just past it. There is no barrier/channel -/// substitute for the passage of lease time through the public store API. pub async fn expired_outbox_lease_is_reclaimed_by_second_worker(repo: R, outbox: S) where R: GetStream + TransactionalCommit + Clone + Send + Sync + 'static, S: AsyncOutboxStore + Send + Sync, { let message_id = unique_id("crash-lease-outbox"); - let mut seat = added_seat(&unique_id("crash-lease-seat")); - let message = OutboxMessage::create(&message_id, "seat.added", b"{}".to_vec()) - .expect("outbox message should be valid"); - repo.clone() - .aggregate::() - .outbox(message) - .commit(&mut seat) - .await - .expect("message should be stored"); + commit_outbox_for_seat( + &repo, + unique_id("crash-lease-seat"), + &message_id, + "seat.added", + ) + .await; - // Worker A claims with a short lease, then "crashes" (never settles). let lease = Duration::from_secs(1); - let claimed_a = outbox - .claim_async(ClaimOutboxMessages::new("worker-a", 1, lease)) - .await - .expect("worker A claim should succeed"); - assert_eq!(claimed_a.len(), 1, "worker A claims the only pending row"); - assert_eq!(claimed_a[0].id(), message_id); - let stale_claim_a = OutboxClaimRef::from_message(&claimed_a[0]).expect("A's claim is valid"); + let claimed_a = claim_one(&outbox, ClaimOutboxMessages::new("worker-a", 1, lease)).await; + assert_eq!(claimed_a.id(), message_id); + let stale_claim_a = OutboxClaimRef::from_message(&claimed_a).expect("A's claim is valid"); - // While the lease is live, worker B must NOT be able to steal the row. let live = outbox .claim_async(ClaimOutboxMessages::new("worker-b", 1, lease)) .await @@ -435,33 +364,21 @@ where "an unexpired lease must not be reclaimable by another worker" ); - // Let the lease expire (no settle from A — simulated crash). tokio::time::sleep(Duration::from_millis(1_200)).await; - // Worker B reclaims the now-expired in-flight row. - let claimed_b = outbox - .claim_async(ClaimOutboxMessages::new( - "worker-b", - 1, - Duration::from_secs(60), - )) - .await - .expect("worker B claim after expiry should succeed"); - assert_eq!( - claimed_b.len(), - 1, - "the expired lease must be reclaimable by a second worker" - ); - assert_eq!(claimed_b[0].id(), message_id); - assert_eq!(claimed_b[0].worker_id.as_deref(), Some("worker-b")); + let claimed_b = claim_one( + &outbox, + ClaimOutboxMessages::new("worker-b", 1, Duration::from_secs(60)), + ) + .await; + assert_eq!(claimed_b.id(), message_id); + assert_eq!(claimed_b.worker_id.as_deref(), Some("worker-b")); assert!( - claimed_b[0].attempts > claimed_a[0].attempts, + claimed_b.attempts > claimed_a.attempts, "reclaim increments the attempt counter (fences A's stale claim)" ); - let claim_b = OutboxClaimRef::from_message(&claimed_b[0]).expect("B's claim is valid"); + let claim_b = OutboxClaimRef::from_message(&claimed_b).expect("B's claim is valid"); - // Worker A wakes up with its stale lease and tries to settle. Both complete - // and release must be rejected — A no longer owns the row. let late_complete = outbox .complete_async(&stale_claim_a) .await @@ -479,7 +396,6 @@ where "stale-worker release should be InvalidState, got {late_release:?}" ); - // Worker B (the rightful owner) completes successfully. outbox .complete_async(&claim_b) .await @@ -494,15 +410,6 @@ where ); } -/// Publish-on-commit durability: a row committed alongside its aggregate must -/// survive transient publish failures and stay claimable until it is genuinely -/// delivered, then end Published — and the bus must see exactly one delivery. -/// -/// The dispatcher runs one pass per `dispatch_ids` call. The first two passes -/// fail to publish (row goes InFlight on claim → back to Pending on -/// `record_failure`, attempts incrementing); the third succeeds and the row -/// becomes Published. This pins the at-least-once-store / exactly-once-delivery -/// boundary that publish-on-commit relies on. pub async fn publish_failure_after_commit_retains_outbox_row_until_delivered( repo: R, outbox: S, @@ -511,18 +418,14 @@ pub async fn publish_failure_after_commit_retains_outbox_row_until_delivered() - .outbox(message) - .commit(&mut seat) - .await - .expect("aggregate and outbox should commit atomically"); + commit_outbox_for_seat( + &repo, + unique_id("publish-retry-seat"), + &message_id, + "seat.added", + ) + .await; - // max_attempts is high enough that the two failures only release (never - // permanently fail) the row. let dispatcher = OutboxDispatcher::new( outbox.clone(), FlakyPublisher::new(2), @@ -532,7 +435,6 @@ pub async fn publish_failure_after_commit_retains_outbox_row_until_delivered(repo: &R, seat_id: String, message_id: &str, event_type: &str) +where + R: TransactionalCommit + Clone + Send + Sync + 'static, +{ + let mut seat = added_seat(&seat_id); + repo.clone() + .aggregate::() + .outbox(outbox_message(message_id, event_type)) + .commit(&mut seat) + .await + .expect("aggregate and outbox should commit atomically"); +} + +fn outbox_message(id: &str, event_type: &str) -> OutboxMessage { + OutboxMessage::create(id, event_type, b"{}".to_vec()).expect("outbox message should be valid") +} + +async fn claim_one(outbox: &S, request: ClaimOutboxMessages) -> OutboxMessage +where + S: AsyncOutboxStore + Send + Sync, +{ + let claimed = outbox + .claim_async(request) + .await + .expect("claim should succeed"); + assert_eq!(claimed.len(), 1); + claimed.into_iter().next().expect("one claimed message") +} + fn added_seat(id: &str) -> Seat { let mut seat = Seat::default(); seat.add(id.to_string(), "floor".to_string()) diff --git a/tests/sourced_snapshot/main.rs b/tests/sourced_snapshot/main.rs index 670c51e..5cac621 100644 --- a/tests/sourced_snapshot/main.rs +++ b/tests/sourced_snapshot/main.rs @@ -2,7 +2,7 @@ mod aggregates; use aggregates::*; use distributed::{ - Aggregate, AggregateBuilder, HashMapRepository, OutboxMessage, OutboxStore, SnapshotStore, + Aggregate, AggregateBuilder, AsyncOutboxStore, HashMapRepository, OutboxMessage, SnapshotStore, Snapshottable, StreamIdentity, }; @@ -260,7 +260,7 @@ async fn domain_event_commits_with_outbox() { let loaded = repo.get("t1").await.unwrap().unwrap(); assert_eq!(loaded.snapshot().task, "Ship it"); - let pending = repo.repo().outbox_store().pending().unwrap(); + let pending = repo.repo().outbox_store().pending_async().await.unwrap(); assert_eq!(pending.len(), 1); assert!(pending[0].is_pending()); } diff --git a/tests/sql_lock_manager/main.rs b/tests/sql_lock_manager/main.rs index c5f66dd..78dc20e 100644 --- a/tests/sql_lock_manager/main.rs +++ b/tests/sql_lock_manager/main.rs @@ -354,29 +354,33 @@ mod sqlite_backend { (db, SqliteLockManager::new(pool)) } - #[tokio::test] - async fn acquire_contend_release() { - let (_db, manager) = manager().await; - scenario_acquire_contend_release(&manager).await; + async fn managers() -> (TempDb, SqliteLockManager, SqliteLockManager) { + let db = TempDb::new(); + let m1 = SqliteLockManager::new(db.pool().await); + let m2 = SqliteLockManager::new(db.pool().await); + (db, m1, m2) } - #[tokio::test] - async fn distinct_keys_do_not_contend() { - let (_db, manager) = manager().await; - scenario_distinct_keys_do_not_contend(&manager).await; + macro_rules! single_manager_test { + ($name:ident, $scenario:ident) => { + #[tokio::test] + async fn $name() { + let (_db, manager) = manager().await; + $scenario(&manager).await; + } + }; } - #[tokio::test] - async fn same_handle_per_key() { - let (_db, manager) = manager().await; - scenario_same_handle_per_key(&manager).await; - } + single_manager_test!(acquire_contend_release, scenario_acquire_contend_release); + single_manager_test!( + distinct_keys_do_not_contend, + scenario_distinct_keys_do_not_contend + ); + single_manager_test!(same_handle_per_key, scenario_same_handle_per_key); - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + #[tokio::test] async fn two_managers_serialize() { - let db = TempDb::new(); - let m1 = SqliteLockManager::new(db.pool().await); - let m2 = SqliteLockManager::new(db.pool().await); + let (db, m1, m2) = managers().await; scenario_two_managers_serialize(m1, m2).await; drop(db); } @@ -410,9 +414,7 @@ mod sqlite_backend { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn two_managers_race_free_key() { - let db = TempDb::new(); - let m1 = SqliteLockManager::new(db.pool().await); - let m2 = SqliteLockManager::new(db.pool().await); + let (db, m1, m2) = managers().await; scenario_two_managers_race_free_key(m1, m2).await; drop(db); } @@ -445,9 +447,7 @@ mod sqlite_backend { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cancelled_acquire_releases_gate() { - let db = TempDb::new(); - let holder = SqliteLockManager::new(db.pool().await); - let other = SqliteLockManager::new(db.pool().await); + let (db, holder, other) = managers().await; scenario_cancelled_acquire_releases_gate(&holder, &other).await; drop(db); } @@ -473,41 +473,43 @@ mod postgres_backend { PostgresTestSchema::create_from_env("locks", SKIP).await } - #[tokio::test] - async fn acquire_contend_release() { - let Some(schema) = schema().await else { - return; - }; + async fn manager() -> Option<(PostgresTestSchema, PostgresLockManager)> { + let schema = schema().await?; let manager = PostgresLockManager::new(schema.repository().await.pool().clone()); - scenario_acquire_contend_release(&manager).await; + Some((schema, manager)) } - #[tokio::test] - async fn distinct_keys_do_not_contend() { - let Some(schema) = schema().await else { - return; - }; - let manager = PostgresLockManager::new(schema.repository().await.pool().clone()); - scenario_distinct_keys_do_not_contend(&manager).await; + async fn managers() -> Option<(PostgresTestSchema, PostgresLockManager, PostgresLockManager)> { + let schema = schema().await?; + let m1 = PostgresLockManager::new(schema.repository().await.pool().clone()); + let m2 = PostgresLockManager::new(schema.repository().await.pool().clone()); + Some((schema, m1, m2)) } - #[tokio::test] - async fn same_handle_per_key() { - let Some(schema) = schema().await else { - return; + macro_rules! single_manager_test { + ($name:ident, $scenario:ident) => { + #[tokio::test] + async fn $name() { + let Some((_schema, manager)) = manager().await else { + return; + }; + $scenario(&manager).await; + } }; - let manager = PostgresLockManager::new(schema.repository().await.pool().clone()); - scenario_same_handle_per_key(&manager).await; } - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + single_manager_test!(acquire_contend_release, scenario_acquire_contend_release); + single_manager_test!( + distinct_keys_do_not_contend, + scenario_distinct_keys_do_not_contend + ); + single_manager_test!(same_handle_per_key, scenario_same_handle_per_key); + + #[tokio::test] async fn two_managers_serialize() { - let Some(schema) = schema().await else { + let Some((_schema, m1, m2)) = managers().await else { return; }; - // Two pools on the same schema = two independent connections (processes). - let m1 = PostgresLockManager::new(schema.repository().await.pool().clone()); - let m2 = PostgresLockManager::new(schema.repository().await.pool().clone()); scenario_two_managers_serialize(m1, m2).await; } @@ -537,20 +539,17 @@ mod postgres_backend { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn queued_abort_releases() { - let Some(schema) = schema().await else { + let Some((_schema, manager)) = manager().await else { return; }; - let manager = PostgresLockManager::new(schema.repository().await.pool().clone()); scenario_queued_abort_releases(manager).await; } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn two_managers_race_free_key() { - let Some(schema) = schema().await else { + let Some((_schema, m1, m2)) = managers().await else { return; }; - let m1 = PostgresLockManager::new(schema.repository().await.pool().clone()); - let m2 = PostgresLockManager::new(schema.repository().await.pool().clone()); scenario_two_managers_race_free_key(m1, m2).await; } @@ -585,11 +584,9 @@ mod postgres_backend { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cancelled_acquire_releases_gate() { - let Some(schema) = schema().await else { + let Some((_schema, holder, other)) = managers().await else { return; }; - let holder = PostgresLockManager::new(schema.repository().await.pool().clone()); - let other = PostgresLockManager::new(schema.repository().await.pool().clone()); scenario_cancelled_acquire_releases_gate(&holder, &other).await; } } diff --git a/tests/todos/main.rs b/tests/todos/main.rs index ad8f552..f77131f 100644 --- a/tests/todos/main.rs +++ b/tests/todos/main.rs @@ -2,9 +2,10 @@ mod aggregate; use aggregate::{Todo, TodoSnapshot}; use distributed::{ - AggregateBuilder, AsyncLock, AsyncLockManager, ClaimOutboxMessages, CommitBuilderExt, - EventEmitter, HashMapRepository, LocalEmitterPublisher, LogPublisher, OutboxClaimRef, - OutboxMessage, OutboxMessageStatus, OutboxStore, OutboxWorker, Queueable, RepositoryError, + AggregateBuilder, AsyncLock, AsyncLockManager, AsyncOutboxStore, ClaimOutboxMessages, + CommitBuilderExt, DrainResult, EventEmitter, HashMapRepository, LocalEmitterPublisher, + LogPublisher, OutboxClaimRef, OutboxMessage, OutboxMessageStatus, OutboxPublisher, + OutboxWorker, Queueable, RepositoryError, }; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{mpsc, Arc, Mutex}; @@ -18,7 +19,48 @@ fn next_id() -> String { format!("todo-{}", id) } -fn complete_published_outbox( +fn initialized_todo(user_id: &str, task: &str) -> (Todo, String) { + let mut todo = Todo::new(); + let id = next_id(); + todo.initialize(id.clone(), user_id.to_string(), task.to_string()) + .unwrap(); + (todo, id) +} + +fn todo_outbox_message( + id: &str, + suffix: &str, + event_type: &str, + snapshot: &TodoSnapshot, +) -> OutboxMessage { + OutboxMessage::encode(format!("{id}:{suffix}"), event_type, snapshot).unwrap() +} + +async fn claim_and_process( + repo: &HashMapRepository, + worker: &mut OutboxWorker

, + worker_id: &str, + batch_size: usize, +) -> (DrainResult, Vec, Vec) { + let mut claimed = repo + .outbox_store() + .claim_async(ClaimOutboxMessages::new( + worker_id, + batch_size, + Duration::from_secs(30), + )) + .await + .unwrap(); + let claims = claimed + .iter() + .map(OutboxClaimRef::from_message) + .collect::, _>>() + .unwrap(); + let result = worker.process_batch(&mut claimed).unwrap(); + (result, claimed, claims) +} + +async fn complete_published_outbox( repo: &HashMapRepository, messages: &[OutboxMessage], claims: &[OutboxClaimRef], @@ -26,12 +68,12 @@ fn complete_published_outbox( let store = repo.outbox_store(); for (message, claim) in messages.iter().zip(claims) { if message.is_published() { - store.complete(claim).unwrap(); + store.complete_async(claim).await.unwrap(); } } } -fn load_outbox_message(repo: &HashMapRepository, id: &str) -> OutboxMessage { +async fn load_outbox_message(repo: &HashMapRepository, id: &str) -> OutboxMessage { let store = repo.outbox_store(); for status in [ OutboxMessageStatus::Pending, @@ -40,7 +82,8 @@ fn load_outbox_message(repo: &HashMapRepository, id: &str) -> OutboxMessage { OutboxMessageStatus::Failed, ] { if let Some(message) = store - .messages_by_status(status) + .messages_by_status_async(status) + .await .unwrap() .into_iter() .find(|message| message.id() == id) @@ -55,23 +98,8 @@ fn load_outbox_message(repo: &HashMapRepository, id: &str) -> OutboxMessage { async fn todos() { let repo = HashMapRepository::new().queued().aggregate::(); - // Create a new Todo + Outbox messages - let mut todo = Todo::new(); - let id1 = next_id(); - todo.initialize( - id1.clone(), - "user1".to_string(), - "Buy groceries".to_string(), - ) - .unwrap(); - - // Add an outbox event for the initialization - let init_message = OutboxMessage::encode( - format!("{}:init", id1), - "todo.initialized", - &todo.snapshot(), - ) - .unwrap(); + let (mut todo, id1) = initialized_todo("user1", "Buy groceries"); + let init_message = todo_outbox_message(&id1, "init", "todo.initialized", &todo.snapshot()); // Commit the Todo + Outbox message to the repository repo.outbox(init_message) @@ -81,67 +109,60 @@ async fn todos() { // Verify the outbox event was captured { - let pending = repo.repo().inner().outbox_store().pending().unwrap(); + let pending = repo + .repo() + .inner() + .outbox_store() + .pending_async() + .await + .unwrap(); assert_eq!(pending.len(), 1); assert_eq!(pending[0].event_type, "todo.initialized"); } // Retrieve the Todo from the repository and complete it, then commit again - if let Some(mut retrieved_todo) = repo.get(&id1).await.unwrap() { - retrieved_todo.complete().unwrap(); - - // Add an outbox event for the completion - let complete_message = OutboxMessage::encode( - format!("{}:complete", id1), - "todo.completed", - &retrieved_todo.snapshot(), - ) - .unwrap(); - - repo.outbox(complete_message) - .commit(&mut retrieved_todo) - .await - .expect("completed todo outbox commit should succeed"); - - // Verify we now have 2 outbox events - { - let pending = repo.repo().inner().outbox_store().pending().unwrap(); - assert_eq!(pending.len(), 2); - assert!(pending - .iter() - .any(|msg| msg.event_type == "todo.initialized")); - assert!(pending.iter().any(|msg| msg.event_type == "todo.completed")); - } + let mut retrieved_todo = repo.get(&id1).await.unwrap().expect("Todo not found"); + retrieved_todo.complete().unwrap(); + let complete_message = todo_outbox_message( + &id1, + "complete", + "todo.completed", + &retrieved_todo.snapshot(), + ); - if let Some(completed_todo) = repo.get(&id1).await.unwrap() { - assert!(completed_todo.snapshot().id == id1); - assert!(completed_todo.snapshot().user_id == "user1"); - assert!(completed_todo.snapshot().task == "Buy groceries"); - assert!(completed_todo.snapshot().completed); + repo.outbox(complete_message) + .commit(&mut retrieved_todo) + .await + .expect("completed todo outbox commit should succeed"); - repo.abort(&completed_todo).await.unwrap(); - } else { - panic!("Updated Todo not found"); - } - } else { - panic!("Todo not found"); + { + let pending = repo + .repo() + .inner() + .outbox_store() + .pending_async() + .await + .unwrap(); + assert_eq!(pending.len(), 2); + assert!(pending + .iter() + .any(|msg| msg.event_type == "todo.initialized")); + assert!(pending.iter().any(|msg| msg.event_type == "todo.completed")); } - let mut todo2 = Todo::new(); - let id2 = next_id(); - todo2 - .initialize(id2.clone(), "user1".to_string(), "Buy Sauna".to_string()) - .unwrap(); + let completed_todo = repo + .get(&id1) + .await + .unwrap() + .expect("Updated Todo not found"); + assert!(completed_todo.snapshot().id == id1); + assert!(completed_todo.snapshot().user_id == "user1"); + assert!(completed_todo.snapshot().task == "Buy groceries"); + assert!(completed_todo.snapshot().completed); + repo.abort(&completed_todo).await.unwrap(); - let mut todo3 = Todo::new(); - let id3 = next_id(); - todo3 - .initialize( - id3.clone(), - "user2".to_string(), - "Chew bubblegum".to_string(), - ) - .unwrap(); + let (mut todo2, id2) = initialized_todo("user1", "Buy Sauna"); + let (mut todo3, id3) = initialized_todo("user2", "Chew bubblegum"); // Commit multiple Todos to the repository repo.commit_all(&mut [&mut todo2, &mut todo3]) @@ -160,10 +181,7 @@ async fn todos() { #[tokio::test] async fn get_commit_roundtrip() { let repo = HashMapRepository::new().queued().aggregate::(); - let mut todo = Todo::new(); - let id = next_id(); - todo.initialize(id.clone(), "user1".to_string(), "Roundtrip".to_string()) - .unwrap(); + let (mut todo, id) = initialized_todo("user1", "Roundtrip"); repo.commit(&mut todo).await.unwrap(); @@ -178,25 +196,8 @@ async fn get_commit_roundtrip() { async fn get_all_commit_all_roundtrip() { let repo = HashMapRepository::new().queued().aggregate::(); - let mut todo1 = Todo::new(); - let id1 = next_id(); - todo1 - .initialize( - id1.clone(), - "user1".to_string(), - "todo.first_recorded".to_string(), - ) - .unwrap(); - - let mut todo2 = Todo::new(); - let id2 = next_id(); - todo2 - .initialize( - id2.clone(), - "user2".to_string(), - "todo.second_recorded".to_string(), - ) - .unwrap(); + let (mut todo1, id1) = initialized_todo("user1", "todo.first_recorded"); + let (mut todo2, id2) = initialized_todo("user2", "todo.second_recorded"); repo.commit_all(&mut [&mut todo1, &mut todo2]) .await @@ -232,18 +233,14 @@ async fn get_all_commit_all_roundtrip() { #[tokio::test] async fn outbox_records_persisted() { let repo = HashMapRepository::new(); - let mut todo = Todo::new(); - let id = next_id(); - todo.initialize(id.clone(), "user1".to_string(), "Outbox demo".to_string()) - .unwrap(); + let (mut todo, id) = initialized_todo("user1", "Outbox demo"); let snapshot = todo.snapshot(); - let message = - OutboxMessage::encode(format!("{}:init", id), "todo.initialized", &snapshot).unwrap(); + let message = todo_outbox_message(&id, "init", "todo.initialized", &snapshot); repo.outbox(message).commit(&mut todo).await.unwrap(); // Check pending outbox messages - let pending = repo.outbox_store().pending().unwrap(); + let pending = repo.outbox_store().pending_async().await.unwrap(); assert_eq!(pending.len(), 1); assert_eq!(pending[0].event_type, "todo.initialized"); @@ -257,21 +254,11 @@ async fn outbox_records_persisted() { #[tokio::test] async fn outbox_worker_log_publisher() { let repo = HashMapRepository::new(); - let mut todo = Todo::new(); - let id = next_id(); - todo.initialize( - id.clone(), - "user1".to_string(), - "Outbox log publisher".to_string(), - ) - .unwrap(); - let snapshot = todo.snapshot(); - let message = - OutboxMessage::encode(format!("{}:init", id), "todo.initialized", &snapshot).unwrap(); + let (mut todo, id) = initialized_todo("user1", "Outbox log publisher"); + let message = todo_outbox_message(&id, "init", "todo.initialized", &todo.snapshot()); let message_id = message.id().to_string(); repo.outbox(message).commit(&mut todo).await.unwrap(); - // Create worker with new API let buffer = Arc::new(Mutex::new(Vec::new())); let publisher = LogPublisher::with_buffer(Arc::clone(&buffer)); let mut worker = OutboxWorker::new(publisher) @@ -279,47 +266,24 @@ async fn outbox_worker_log_publisher() { .with_batch_size(10) .with_max_attempts(3); - // Claim pending messages and process - let store = repo.outbox_store(); - let mut claimed = store - .claim(ClaimOutboxMessages::new( - "logger-1", - 10, - Duration::from_secs(30), - )) - .unwrap(); - let claims = claimed - .iter() - .map(OutboxClaimRef::from_message) - .collect::, _>>() - .unwrap(); - let result = worker.process_batch(&mut claimed).unwrap(); + let (result, claimed, claims) = claim_and_process(&repo, &mut worker, "logger-1", 10).await; assert_eq!(result.completed, 1); - complete_published_outbox(&repo, &claimed, &claims); + complete_published_outbox(&repo, &claimed, &claims).await; let lines = buffer.lock().unwrap(); assert_eq!(lines.len(), 1); assert!(lines[0].contains("todo.initialized")); // Check record is marked as published - let published = load_outbox_message(&repo, &message_id); + let published = load_outbox_message(&repo, &message_id).await; assert!(published.is_published()); } #[tokio::test] async fn outbox_worker_local_emitter_publisher() { let repo = HashMapRepository::new(); - let mut todo = Todo::new(); - let id = next_id(); - todo.initialize( - id.clone(), - "user1".to_string(), - "Outbox local emitter".to_string(), - ) - .unwrap(); - let snapshot = todo.snapshot(); - let message = - OutboxMessage::encode(format!("{}:init", id), "todo.initialized", &snapshot).unwrap(); + let (mut todo, id) = initialized_todo("user1", "Outbox local emitter"); + let message = todo_outbox_message(&id, "init", "todo.initialized", &todo.snapshot()); repo.outbox(message).commit(&mut todo).await.unwrap(); let mut emitter = EventEmitter::new(); @@ -334,23 +298,9 @@ async fn outbox_worker_local_emitter_publisher() { .with_batch_size(10) .with_max_attempts(3); - // Claim pending messages and process - let store = repo.outbox_store(); - let mut claimed = store - .claim(ClaimOutboxMessages::new( - "emitter-1", - 10, - Duration::from_secs(30), - )) - .unwrap(); - let claims = claimed - .iter() - .map(OutboxClaimRef::from_message) - .collect::, _>>() - .unwrap(); - let result = worker.process_batch(&mut claimed).unwrap(); + let (result, claimed, claims) = claim_and_process(&repo, &mut worker, "emitter-1", 10).await; assert_eq!(result.completed, 1); - complete_published_outbox(&repo, &claimed, &claims); + complete_published_outbox(&repo, &claimed, &claims).await; // LocalEmitterPublisher converts bytes to lossy string, so we just verify something was received let payload = rx.recv_timeout(Duration::from_secs(1)).unwrap(); @@ -360,10 +310,7 @@ async fn outbox_worker_local_emitter_publisher() { #[tokio::test] async fn abort_releases_lock_after_get() { let repo = Arc::new(HashMapRepository::new().queued().aggregate::()); - let mut todo = Todo::new(); - let id = next_id(); - todo.initialize(id.clone(), "user1".to_string(), "Abort get".to_string()) - .unwrap(); + let (mut todo, id) = initialized_todo("user1", "Abort get"); repo.commit(&mut todo).await.unwrap(); let locked = repo.get(&id).await.unwrap().unwrap(); @@ -389,26 +336,10 @@ async fn abort_releases_lock_after_get() { #[tokio::test] async fn abort_releases_lock_after_get_all() { let repo = Arc::new(HashMapRepository::new().queued().aggregate::()); - let mut todo1 = Todo::new(); - let id1 = next_id(); - todo1 - .initialize( - id1.clone(), - "user1".to_string(), - "Abort get_all 1".to_string(), - ) - .unwrap(); + let (mut todo1, id1) = initialized_todo("user1", "Abort get_all 1"); repo.commit(&mut todo1).await.unwrap(); - let mut todo2 = Todo::new(); - let id2 = next_id(); - todo2 - .initialize( - id2.clone(), - "user2".to_string(), - "Abort get_all 2".to_string(), - ) - .unwrap(); + let (mut todo2, id2) = initialized_todo("user2", "Abort get_all 2"); repo.commit(&mut todo2).await.unwrap(); let locked = repo.get_all(&[&id1, &id2]).await.unwrap(); @@ -437,21 +368,10 @@ async fn abort_releases_lock_after_get_all() { #[tokio::test] async fn queued_repo_blocks_get_until_commit() { let repo = Arc::new(HashMapRepository::new().queued().aggregate::()); - let mut todo = Todo::new(); - let id = next_id(); - todo.initialize(id.clone(), "user1".to_string(), "Queue test".to_string()) - .unwrap(); + let (mut todo, id) = initialized_todo("user1", "Queue test"); repo.commit(&mut todo).await.unwrap(); - let mut other_todo = Todo::new(); - let other_id = next_id(); - other_todo - .initialize( - other_id.clone(), - "user2".to_string(), - "Independent queue".to_string(), - ) - .unwrap(); + let (mut other_todo, other_id) = initialized_todo("user2", "Independent queue"); repo.commit(&mut other_todo).await.unwrap(); let (tx_started, rx_started) = mpsc::channel(); @@ -555,14 +475,7 @@ async fn manual_lock_reports_failure_when_already_held() { #[tokio::test] async fn commit_failure_keeps_lock_until_abort() { let repo = Arc::new(HashMapRepository::new().queued().aggregate::()); - let mut todo = Todo::new(); - let id = next_id(); - todo.initialize( - id.clone(), - "user1".to_string(), - "Commit failure lock".to_string(), - ) - .unwrap(); + let (mut todo, id) = initialized_todo("user1", "Commit failure lock"); repo.commit(&mut todo).await.unwrap(); let mut locked = repo.get(&id).await.unwrap().unwrap(); @@ -609,23 +522,13 @@ async fn commit_failure_keeps_lock_until_abort() { #[tokio::test] async fn outbox_worker_process_next_with_commit() { let repo = HashMapRepository::new(); - let mut todo = Todo::new(); - let id = next_id(); - todo.initialize( - id.clone(), - "user1".to_string(), - "Process next test".to_string(), - ) - .unwrap(); + let (mut todo, id) = initialized_todo("user1", "Process next test"); let snapshot = todo.snapshot(); // Queue 3 messages - let message1 = - OutboxMessage::encode(format!("{}:1", id), "todo.first_recorded", &snapshot).unwrap(); - let message2 = - OutboxMessage::encode(format!("{}:2", id), "todo.second_recorded", &snapshot).unwrap(); - let message3 = - OutboxMessage::encode(format!("{}:3", id), "todo.third_recorded", &snapshot).unwrap(); + let message1 = todo_outbox_message(&id, "1", "todo.first_recorded", &snapshot); + let message2 = todo_outbox_message(&id, "2", "todo.second_recorded", &snapshot); + let message3 = todo_outbox_message(&id, "3", "todo.third_recorded", &snapshot); let message_ids = vec![ message1.id().to_string(), @@ -653,11 +556,12 @@ async fn outbox_worker_process_next_with_commit() { loop { let store = repo.outbox_store(); let mut claimed = store - .claim(ClaimOutboxMessages::new( + .claim_async(ClaimOutboxMessages::new( "safe-worker", 1, Duration::from_secs(30), )) + .await .unwrap(); if claimed.is_empty() { break; @@ -669,15 +573,15 @@ async fn outbox_worker_process_next_with_commit() { .unwrap(); let result = worker.process_batch(&mut claimed).unwrap(); processed += result.completed + result.released + result.failed; - complete_published_outbox(&repo, &claimed, &claims); + complete_published_outbox(&repo, &claimed, &claims).await; } assert_eq!(processed, 3); for id in &message_ids { - let message = load_outbox_message(&repo, id); + let message = load_outbox_message(&repo, id).await; assert!(message.is_published()); } - assert_eq!(repo.outbox_store().pending().unwrap().len(), 0); + assert_eq!(repo.outbox_store().pending_async().await.unwrap().len(), 0); let lines = buffer.lock().unwrap(); assert_eq!(lines.len(), 3); @@ -724,22 +628,10 @@ async fn metadata_flows_from_entity_through_outbox_to_publisher() { let publisher = LogPublisher::with_buffer(Arc::clone(&buffer)); let mut worker = OutboxWorker::new(publisher).with_worker_id("meta-worker"); - let store = repo.repo().outbox_store(); - let mut claimed = store - .claim(ClaimOutboxMessages::new( - "meta-worker", - 10, - Duration::from_secs(30), - )) - .unwrap(); - let claims = claimed - .iter() - .map(OutboxClaimRef::from_message) - .collect::, _>>() - .unwrap(); - let result = worker.process_batch(&mut claimed).unwrap(); + let (result, claimed, claims) = + claim_and_process(repo.repo(), &mut worker, "meta-worker", 10).await; assert_eq!(result.completed, 1); - complete_published_outbox(repo.repo(), &claimed, &claims); + complete_published_outbox(repo.repo(), &claimed, &claims).await; // 6. Verify the publisher received metadata let lines = buffer.lock().unwrap(); diff --git a/tests/transport_conformance/mod.rs b/tests/transport_conformance/mod.rs index 59c12af..e1da7fb 100644 --- a/tests/transport_conformance/mod.rs +++ b/tests/transport_conformance/mod.rs @@ -209,17 +209,20 @@ pub fn event_message(name: &str, id: Option<&str>) -> Message { message } +fn recording_source(messages: Vec) -> (Arc, Arc>, FakeSource) { + let recorder = Recorder::new(); + let service = recording_service(&recorder); + let source = FakeSource::new(recorder.clone(), messages); + (recorder, service, source) +} + // ============================================================================= // Source-runner contract // ============================================================================= pub async fn source_dispatches_before_ack() { - let recorder = Recorder::new(); - let service = recording_service(&recorder); - let source = FakeSource::new( - recorder.clone(), - vec![event_message("delivery.succeeded", Some("m1"))], - ); + let (recorder, service, source) = + recording_source(vec![event_message("delivery.succeeded", Some("m1"))]); run_source(service, source, RunOptions::idempotent()) .await .unwrap(); @@ -231,12 +234,8 @@ pub async fn source_dispatches_before_ack() { } pub async fn source_retryable_failure_nacks_without_ack() { - let recorder = Recorder::new(); - let service = recording_service(&recorder); - let source = FakeSource::new( - recorder.clone(), - vec![event_message("delivery.retry_requested", Some("m1"))], - ); + let (recorder, service, source) = + recording_source(vec![event_message("delivery.retry_requested", Some("m1"))]); run_source(service, source, RunOptions::idempotent()) .await .unwrap(); @@ -250,12 +249,10 @@ pub async fn source_retryable_failure_nacks_without_ack() { } pub async fn source_permanent_failure_dead_letters_by_default() { - let recorder = Recorder::new(); - let service = recording_service(&recorder); - let source = FakeSource::new( - recorder.clone(), - vec![event_message("delivery.permanently_failed", Some("m1"))], - ); + let (recorder, service, source) = recording_source(vec![event_message( + "delivery.permanently_failed", + Some("m1"), + )]); run_source(service, source, RunOptions::idempotent()) .await .unwrap(); @@ -266,15 +263,10 @@ pub async fn source_permanent_failure_dead_letters_by_default() { } pub async fn source_permanent_failure_stops_under_stop_policy() { - let recorder = Recorder::new(); - let service = recording_service(&recorder); - let source = FakeSource::new( - recorder.clone(), - vec![ - event_message("delivery.permanently_failed", Some("m1")), - event_message("delivery.succeeded", Some("m2")), - ], - ); + let (recorder, service, source) = recording_source(vec![ + event_message("delivery.permanently_failed", Some("m1")), + event_message("delivery.succeeded", Some("m2")), + ]); let outcome = run_source( service, source, @@ -290,12 +282,8 @@ pub async fn source_permanent_failure_stops_under_stop_policy() { } pub async fn source_unhandled_message_is_acked_and_ignored() { - let recorder = Recorder::new(); - let service = recording_service(&recorder); - let source = FakeSource::new( - recorder.clone(), - vec![event_message("unrelated", Some("m1"))], - ); + let (recorder, service, source) = + recording_source(vec![event_message("unrelated", Some("m1"))]); run_source(service, source, RunOptions::idempotent()) .await .unwrap(); @@ -304,13 +292,9 @@ pub async fn source_unhandled_message_is_acked_and_ignored() { } pub async fn source_inbox_mode_rejects_missing_stable_id() { - let recorder = Recorder::new(); - let service = recording_service(&recorder); // No id on the message; inbox mode requires a stable id. - let source = FakeSource::new( - recorder.clone(), - vec![event_message("delivery.succeeded", None)], - ); + let (recorder, service, source) = + recording_source(vec![event_message("delivery.succeeded", None)]); run_source(service, source, RunOptions::inbox(())) .await .unwrap(); @@ -321,12 +305,8 @@ pub async fn source_inbox_mode_rejects_missing_stable_id() { } pub async fn source_inbox_mode_dispatches_with_stable_id() { - let recorder = Recorder::new(); - let service = recording_service(&recorder); - let source = FakeSource::new( - recorder.clone(), - vec![event_message("delivery.succeeded", Some("m1"))], - ); + let (recorder, service, source) = + recording_source(vec![event_message("delivery.succeeded", Some("m1"))]); run_source(service, source, RunOptions::inbox(())) .await .unwrap(); @@ -337,26 +317,18 @@ pub async fn source_inbox_mode_dispatches_with_stable_id() { } pub async fn source_propagates_recv_errors() { - let recorder = Recorder::new(); - let service = recording_service(&recorder); - let source = FakeSource::new( - recorder.clone(), - vec![event_message("delivery.succeeded", Some("m1"))], - ) - .with_recv_error(); + let (recorder, service, source) = + recording_source(vec![event_message("delivery.succeeded", Some("m1"))]); + let source = source.with_recv_error(); let outcome = run_source(service, source, RunOptions::idempotent()).await; assert!(outcome.is_err(), "recv errors must not be swallowed"); assert!(recorder.events().is_empty()); } pub async fn source_propagates_settle_errors() { - let recorder = Recorder::new(); - let service = recording_service(&recorder); - let source = FakeSource::new( - recorder.clone(), - vec![event_message("delivery.succeeded", Some("m1"))], - ) - .with_settle_failure(); + let (recorder, service, source) = + recording_source(vec![event_message("delivery.succeeded", Some("m1"))]); + let source = source.with_settle_failure(); let outcome = run_source(service, source, RunOptions::idempotent()).await; assert!(outcome.is_err(), "settle errors must not be swallowed"); // The ack was attempted before the error surfaced. @@ -378,23 +350,28 @@ async fn store_outbox(repo: &HashMapRepository, id: &str) -> String { id.to_string() } -fn outbox_status(repo: &HashMapRepository, id: &str) -> Option { - use distributed::OutboxStore; +async fn outbox_status(repo: &HashMapRepository, id: &str) -> Option { + use distributed::AsyncOutboxStore; let store = repo.outbox_store(); - [ + for status in [ OutboxMessageStatus::Pending, OutboxMessageStatus::InFlight, OutboxMessageStatus::Published, OutboxMessageStatus::Failed, ] .into_iter() - .find(|status| { - store - .messages_by_status(status.clone()) + { + if store + .messages_by_status_async(status.clone()) + .await .unwrap() .iter() .any(|message| message.id() == id) - }) + { + return Some(status); + } + } + None } fn dispatcher( @@ -426,7 +403,7 @@ pub async fn dispatcher_completes_only_after_publish_success() { vec!["evt-1".to_string()] ); assert_eq!( - outbox_status(&repo, &id), + outbox_status(&repo, &id).await, Some(OutboxMessageStatus::Published) ); } @@ -443,7 +420,7 @@ pub async fn dispatcher_unknown_outcome_stays_retryable() { assert_eq!(outcome.published, 0); assert_eq!(outcome.released, 1); assert_eq!( - outbox_status(&repo, &id), + outbox_status(&repo, &id).await, Some(OutboxMessageStatus::Pending), "row must stay retryable" ); @@ -462,12 +439,12 @@ pub async fn dispatcher_claims_explicit_ids_before_publish() { assert_eq!(outcome.claimed, 1); assert_eq!(outcome.published, 1); assert_eq!( - outbox_status(&repo, &wanted), + outbox_status(&repo, &wanted).await, Some(OutboxMessageStatus::Published) ); // The unrequested row is untouched (claimed before publish, by id). assert_eq!( - outbox_status(&repo, &other), + outbox_status(&repo, &other).await, Some(OutboxMessageStatus::Pending) ); } From 0b34010d986bec713a10b1f7119edeae49f4e4fe Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 13 Jun 2026 14:43:26 -0500 Subject: [PATCH 2/4] refactor: make outbox APIs async-only --- README.md | 2 +- src/lib.rs | 2 +- src/microsvc/dependencies.rs | 3 +- src/microsvc/runtime.rs | 14 +- src/outbox/commit.rs | 6 +- src/outbox_worker/mod.rs | 16 +-- src/outbox_worker/outbox_dispatch.rs | 12 +- src/outbox_worker/outbox_source.rs | 26 ++-- src/outbox_worker/publish_hook.rs | 8 +- src/outbox_worker/publisher.rs | 128 ++++++++---------- src/outbox_worker/store.rs | 89 ++++++------ src/outbox_worker/worker.rs | 100 +++++++------- src/postgres_repo/mod.rs | 16 +-- src/sqlite_repo/mod.rs | 16 +-- tests/bomberman/main.rs | 4 +- tests/distributed_read_model/main.rs | 14 +- tests/distributed_read_model_board/main.rs | 8 +- tests/durable_enqueue_sqlite/main.rs | 10 +- tests/microsvc/convention.rs | 8 +- .../inbox.rs | 8 +- .../outbox.rs | 51 ++++--- tests/postgres_repository/main.rs | 8 +- tests/postgres_transport/main.rs | 4 +- tests/repository_api/main.rs | 6 +- tests/sagas/microsvc_saga.rs | 8 +- tests/sourced_snapshot/main.rs | 4 +- tests/sqlite_repository/main.rs | 8 +- tests/todos/main.rs | 40 ++---- tests/transport_conformance/mod.rs | 6 +- 29 files changed, 290 insertions(+), 335 deletions(-) diff --git a/README.md b/README.md index e478a75..4733eea 100644 --- a/README.md +++ b/README.md @@ -284,7 +284,7 @@ Every infrastructure concern in `distributed` follows the same pattern: an **asy | Messaging | `Bus` + `BusConsumer` | `InMemoryBus` | `NatsBus`, `PostgresBus`, `RabbitBus`, `KafkaBus`, `KnativeBus` | | Read model rows | `ReadModelWritePlanStore` + `RelationalReadModelQueryStore` | `InMemoryReadModelStore` | Postgres, SQLite | | Snapshot store | `SnapshotStore` | `InMemorySnapshotStore` | Postgres, SQLite, … | -| Outbox publishing | `AsyncMessagePublisher` (production; the extension point) — sync `OutboxPublisher` is dev/test only | `LogPublisher` (dev/test) | Any `AsyncMessagePublisher` (e.g. `BusPublisher` over a real `Bus`) | +| Outbox publishing | `OutboxStore` + async `AsyncMessagePublisher` / `OutboxPublisher` | `LogPublisher` (dev/test) | Any `AsyncMessagePublisher` (e.g. `BusPublisher` over a real `Bus`) | | Locking | `AsyncLock` + `AsyncLockManager` | `InMemoryAsyncLockManager` | `PostgresLockManager`, `SqliteLockManager` (durable leases), Redis, … | All in-memory defaults are `Clone` and `Send + Sync`, so they work in single-task tests and multi-task servers alike. When you're ready for production, implement the trait for your infrastructure and plug it in — handler code does not change. diff --git a/src/lib.rs b/src/lib.rs index d5a1bb3..fe2e987 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -77,7 +77,6 @@ pub use outbox::{ // Outbox Worker: drain and publish concerns pub use outbox_worker::{ - AsyncOutboxStore, ClaimOutboxMessages, // Worker DrainResult, @@ -87,6 +86,7 @@ pub use outbox_worker::{ OutboxClaimRef, OutboxPublishFailureAction, OutboxPublisher, + OutboxStore, OutboxWorker, ProcessOneResult, }; diff --git a/src/microsvc/dependencies.rs b/src/microsvc/dependencies.rs index 2de1b70..8ebf227 100644 --- a/src/microsvc/dependencies.rs +++ b/src/microsvc/dependencies.rs @@ -2,7 +2,6 @@ use crate::aggregate::AggregateRepository; use crate::outbox::OutboxPublisherConfig; -use crate::outbox_worker::AsyncOutboxStore; use crate::repository::{ReadModelWritePlanStore, RelationalReadModelQueryStore, Repository}; /// Dependency capability for services that expose an aggregate repository. @@ -59,7 +58,7 @@ impl HasOutboxStore for RepoReadModelDependencies { /// (`AggregateRepository` -> `QueuedRepository` -> the leaf SQL/in-memory repo). pub trait HasOutboxStore { /// The concrete outbox store this repository produces. - type OutboxStore: AsyncOutboxStore; + type OutboxStore: crate::outbox_worker::OutboxStore; /// Produce a handle to the durable outbox store. fn outbox_store(&self) -> Self::OutboxStore; diff --git a/src/microsvc/runtime.rs b/src/microsvc/runtime.rs index b86b1c2..c511b27 100644 --- a/src/microsvc/runtime.rs +++ b/src/microsvc/runtime.rs @@ -139,7 +139,7 @@ mod tests { use crate::bus::{Bus, InMemoryBus, RunOptions}; use crate::microsvc::{Context, HandlerError, HasOutboxStore, Service, Session}; - use crate::outbox_worker::AsyncOutboxStore; + use crate::outbox_worker::OutboxStore; use crate::{ sourced, AggregateBuilder, AggregateRepository, Entity, HashMapRepository, OutboxMessage, OutboxMessageStatus, Queueable, QueuedRepository, Snapshot, @@ -180,12 +180,12 @@ mod tests { assert_eq!(receipt.outbox_message_ids(), ["evt-1".to_string()]); let published = store - .messages_by_status_async(OutboxMessageStatus::Published) + .messages_by_status(OutboxMessageStatus::Published) .await .unwrap(); assert_eq!(published.len(), 1, "row should be published at commit time"); assert_eq!(published[0].id(), "evt-1"); - assert!(store.pending_async().await.unwrap().is_empty()); + assert!(store.pending().await.unwrap().is_empty()); } type TouchRepo = AggregateRepository, Dummy>; @@ -217,12 +217,12 @@ mod tests { let store = service.repo().outbox_store(); let published = store - .messages_by_status_async(OutboxMessageStatus::Published) + .messages_by_status(OutboxMessageStatus::Published) .await .unwrap(); assert_eq!(published.len(), 1, "row should be published immediately"); assert_eq!(published[0].id(), "evt-1"); - assert!(store.pending_async().await.unwrap().is_empty()); + assert!(store.pending().await.unwrap().is_empty()); } #[tokio::test] @@ -244,7 +244,7 @@ mod tests { service.run(RunOptions::idempotent()).await.unwrap(); let published = store - .messages_by_status_async(OutboxMessageStatus::Published) + .messages_by_status(OutboxMessageStatus::Published) .await .unwrap(); assert_eq!( @@ -303,7 +303,7 @@ mod tests { let store = service.repo().outbox_store(); let published = store - .messages_by_status_async(OutboxMessageStatus::Published) + .messages_by_status(OutboxMessageStatus::Published) .await .unwrap(); assert_eq!( diff --git a/src/outbox/commit.rs b/src/outbox/commit.rs index 94eaec0..7e7a8c3 100644 --- a/src/outbox/commit.rs +++ b/src/outbox/commit.rs @@ -208,7 +208,7 @@ impl AggregateRepository { #[cfg(test)] mod tests { use super::*; - use crate::{sourced, AggregateBuilder, AsyncOutboxStore, Entity, HashMapRepository}; + use crate::{sourced, AggregateBuilder, Entity, HashMapRepository, OutboxStore}; use std::sync::Mutex; #[derive(Default)] @@ -267,7 +267,7 @@ mod tests { assert!(receipt.has_outbox_messages()); assert_eq!(receipt.outbox_message_ids(), ["msg-1".to_string()]); - let pending = repo.repo().outbox_store().pending_async().await.unwrap(); + let pending = repo.repo().outbox_store().pending().await.unwrap(); assert_eq!(pending.len(), 1); assert_eq!(pending[0].id(), "msg-1"); } @@ -404,7 +404,7 @@ mod tests { ); // 2) outbox row present (pending — no bus attached here) - let pending = repo.repo().outbox_store().pending_async().await.unwrap(); + let pending = repo.repo().outbox_store().pending().await.unwrap(); assert_eq!(pending.len(), 1); assert_eq!(pending[0].id(), "evt-c1"); diff --git a/src/outbox_worker/mod.rs b/src/outbox_worker/mod.rs index 7ea3078..6aecc73 100644 --- a/src/outbox_worker/mod.rs +++ b/src/outbox_worker/mod.rs @@ -3,10 +3,10 @@ //! This module provides the worker infrastructure for processing outbox messages. //! //! Items: -//! - `AsyncOutboxStore` - Store operations for claiming and completing messages +//! - `OutboxStore` - Store operations for claiming and completing messages //! - `OutboxDispatcher` / `BusPublisher` - the async production drain path -//! - `OutboxWorker` - synchronous dev/test message processor -//! - `OutboxPublisher` - synchronous dev/test publish trait +//! - `OutboxWorker` - async loaded-message processor +//! - `OutboxPublisher` - async loaded-message publish trait //! - `LogPublisher` - simple logging publisher for tests //! - `LocalEmitterPublisher` - In-process event emitter (requires `emitter` feature) //! @@ -19,16 +19,16 @@ //! ## Example //! //! ```ignore -//! use distributed::{AsyncOutboxStore, ClaimOutboxMessages, OutboxClaimRef}; +//! use distributed::{OutboxStore, ClaimOutboxMessages, OutboxClaimRef}; //! use std::time::Duration; //! //! let worker_id = "worker-1"; //! let messages = outbox -//! .claim_async(ClaimOutboxMessages::new(worker_id, 10, Duration::from_secs(60))) +//! .claim(ClaimOutboxMessages::new(worker_id, 10, Duration::from_secs(60))) //! .await?; //! for msg in messages { //! let claim = OutboxClaimRef::from_message(&msg)?; -//! outbox.complete_async(&claim).await?; +//! outbox.complete(&claim).await?; //! } //! ``` @@ -48,9 +48,7 @@ pub use publisher::{LogPublisher, LogPublisherError, OutboxPublisher}; // Repository helpers #[cfg(any(feature = "postgres", feature = "sqlite"))] pub(crate) use store::ensure_active_claim; -pub use store::{ - AsyncOutboxStore, ClaimOutboxMessages, OutboxClaimRef, OutboxPublishFailureAction, -}; +pub use store::{ClaimOutboxMessages, OutboxClaimRef, OutboxPublishFailureAction, OutboxStore}; // Worker pub use worker::{DrainResult, OutboxWorker, ProcessOneResult}; diff --git a/src/outbox_worker/outbox_dispatch.rs b/src/outbox_worker/outbox_dispatch.rs index 3bcb1d4..a7d9ce9 100644 --- a/src/outbox_worker/outbox_dispatch.rs +++ b/src/outbox_worker/outbox_dispatch.rs @@ -12,7 +12,7 @@ use std::time::Duration; -use super::{AsyncOutboxStore, ClaimOutboxMessages, OutboxClaimRef, OutboxPublishFailureAction}; +use super::{ClaimOutboxMessages, OutboxClaimRef, OutboxPublishFailureAction, OutboxStore}; use crate::bus::{AsyncMessagePublisher, Message, MessageKind, TransportError, TransportErrorKind}; use crate::outbox::OutboxMessage; use crate::repository::RepositoryError; @@ -149,7 +149,7 @@ pub struct OutboxDispatcher { impl OutboxDispatcher where - S: AsyncOutboxStore, + S: OutboxStore, P: AsyncMessagePublisher, { /// Create a dispatcher. `worker_id` scopes claims (use a synthetic id such as @@ -190,7 +190,7 @@ where ) -> Result { let request = ClaimOutboxMessages::for_ids(self.worker_id.clone(), ids.to_vec(), self.lease); - let claimed = self.store.claim_async(request).await?; + let claimed = self.store.claim(request).await?; let mut outcome = self.dispatch_claimed(claimed).await?; outcome.requested = ids.len(); Ok(outcome) @@ -202,7 +202,7 @@ where batch_size: usize, ) -> Result { let request = ClaimOutboxMessages::new(self.worker_id.clone(), batch_size, self.lease); - let claimed = self.store.claim_async(request).await?; + let claimed = self.store.claim(request).await?; let mut outcome = self.dispatch_claimed(claimed).await?; outcome.requested = batch_size; Ok(outcome) @@ -224,13 +224,13 @@ where let transport_message = Message::from(&message); match self.publisher.publish(transport_message).await { Ok(()) => { - self.store.complete_async(&claim).await?; + self.store.complete(&claim).await?; outcome.published += 1; } Err(publish_error) => { match self .store - .record_failure_async(&claim, &publish_error.to_string(), self.max_attempts) + .record_failure(&claim, &publish_error.to_string(), self.max_attempts) .await? { OutboxPublishFailureAction::Released => outcome.released += 1, diff --git a/src/outbox_worker/outbox_source.rs b/src/outbox_worker/outbox_source.rs index ac4ff43..42525b0 100644 --- a/src/outbox_worker/outbox_source.rs +++ b/src/outbox_worker/outbox_source.rs @@ -1,6 +1,6 @@ //! Outbox-backed durable receive. //! -//! [`OutboxSource`] turns any [`AsyncOutboxStore`] into an [`AsyncMessageSource`]: +//! [`OutboxSource`] turns any [`OutboxStore`] into an [`AsyncMessageSource`]: //! it claims durable rows (`FOR UPDATE SKIP LOCKED` + lease in the SQL stores), //! maps each to a canonical [`Message`], and settles by row status — //! ack→complete, nack→release-for-retry, dead-letter/park→fail (the terminal @@ -18,7 +18,7 @@ use std::collections::VecDeque; use std::sync::Arc; use std::time::Duration; -use super::{AsyncOutboxStore, ClaimOutboxMessages, OutboxClaimRef}; +use super::{ClaimOutboxMessages, OutboxClaimRef, OutboxStore}; use crate::bus::{AsyncMessageSource, Message, ReceivedMessage, TransportError}; use crate::outbox::OutboxMessage; @@ -27,7 +27,7 @@ pub const DEFAULT_OUTBOX_SOURCE_LEASE: Duration = Duration::from_secs(30); /// Default number of rows claimed per `recv` refill. pub const DEFAULT_OUTBOX_SOURCE_BATCH: usize = 16; -/// An [`AsyncMessageSource`] backed by an [`AsyncOutboxStore`]. +/// An [`AsyncMessageSource`] backed by an [`OutboxStore`]. pub struct OutboxSource { store: Arc, worker_id: String, @@ -40,7 +40,7 @@ pub struct OutboxSource { impl OutboxSource where - S: AsyncOutboxStore, + S: OutboxStore, { /// Create a source. `worker_id` scopes claims; `max_attempts` is the /// retryable-failure ceiling before a row is failed. @@ -102,13 +102,13 @@ where impl AsyncMessageSource for OutboxSource where - S: AsyncOutboxStore, + S: OutboxStore, { type Received = ReceivedOutboxMessage; async fn recv(&mut self) -> Result, TransportError> { if self.buffer.is_empty() { - let claimed = self.store.claim_async(self.claim_request()).await?; + let claimed = self.store.claim(self.claim_request()).await?; self.buffer.extend(claimed); } match self.buffer.pop_front() { @@ -136,7 +136,7 @@ pub struct ReceivedOutboxMessage { impl ReceivedMessage for ReceivedOutboxMessage where - S: AsyncOutboxStore, + S: OutboxStore, { fn message(&self) -> &Message { &self.message @@ -144,28 +144,28 @@ where /// Complete the row (transport delivery succeeded). async fn ack(self) -> Result<(), TransportError> { - self.store.complete_async(&self.claim).await?; + self.store.complete(&self.claim).await?; Ok(()) } /// Release for retry, or fail once the attempt ceiling is reached. async fn nack(self, reason: &str) -> Result<(), TransportError> { self.store - .record_failure_async(&self.claim, reason, self.max_attempts) + .record_failure(&self.claim, reason, self.max_attempts) .await?; Ok(()) } /// Fail the row terminally (the outbox `Failed` status is the DLQ/archive). async fn dead_letter(self, reason: &str) -> Result<(), TransportError> { - self.store.fail_async(&self.claim, reason).await?; + self.store.fail(&self.claim, reason).await?; Ok(()) } /// Park terminally for manual inspection (same `Failed` status as dead-letter /// in the outbox's state model). async fn park(self, reason: &str) -> Result<(), TransportError> { - self.store.fail_async(&self.claim, reason).await?; + self.store.fail(&self.claim, reason).await?; Ok(()) } } @@ -176,7 +176,7 @@ mod tests { use crate::bus::{run_source, RunOptions}; use crate::microsvc::Service; use crate::{ - AsyncOutboxStore, CommitBatch, HashMapRepository, OutboxMessage, OutboxMessageStatus, + CommitBatch, HashMapRepository, OutboxMessage, OutboxMessageStatus, OutboxStore, TransactionalCommit, }; use serde_json::json; @@ -218,7 +218,7 @@ mod tests { ] .into_iter() .find(|status| { - block_on(store.messages_by_status_async(status.clone())) + block_on(store.messages_by_status(status.clone())) .unwrap() .iter() .any(|m| m.id() == id) diff --git a/src/outbox_worker/publish_hook.rs b/src/outbox_worker/publish_hook.rs index 3469d70..440704a 100644 --- a/src/outbox_worker/publish_hook.rs +++ b/src/outbox_worker/publish_hook.rs @@ -14,7 +14,7 @@ use crate::bus::{AsyncMessagePublisher, Message}; use crate::outbox::{OutboxMessage, OutboxPublishHook}; use crate::repository::RepositoryError; -use super::{AsyncOutboxStore, OutboxClaimRef}; +use super::{OutboxClaimRef, OutboxStore}; /// Publishes committed outbox rows through `publisher` and settles their claims /// in `store`. The `store` must be the same outbox store the commit wrote to. @@ -38,7 +38,7 @@ impl BusOutboxPublishHook { impl OutboxPublishHook for BusOutboxPublishHook where - S: AsyncOutboxStore, + S: OutboxStore, P: AsyncMessagePublisher, { fn publish_claimed<'a>( @@ -49,10 +49,10 @@ where let claim = OutboxClaimRef::from_message(&claimed)?; let message = Message::from(&claimed); match self.publisher.publish(message).await { - Ok(()) => self.store.complete_async(&claim).await, + Ok(()) => self.store.complete(&claim).await, Err(error) => self .store - .record_failure_async(&claim, &error.to_string(), self.max_attempts) + .record_failure(&claim, &error.to_string(), self.max_attempts) .await .map(|_action| ()), } diff --git a/src/outbox_worker/publisher.rs b/src/outbox_worker/publisher.rs index 5f320b3..7dd10e8 100644 --- a/src/outbox_worker/publisher.rs +++ b/src/outbox_worker/publisher.rs @@ -1,33 +1,19 @@ +#![expect( + clippy::manual_async_fn, + reason = "trait method returns impl Future to preserve public future bounds explicitly" +)] + use std::collections::HashMap; use std::fmt; +use std::future::Future; use std::sync::{Arc, Mutex}; #[cfg(feature = "emitter")] use crate::EventEmitter; -/// Synchronous publisher trait used by the in-process [`OutboxWorker`] — a -/// **dev/test** drain loop. -/// -/// # Which publisher path to use -/// -/// This crate has two outbox-publishing paths, and they are not parallel -/// hierarchies — they sit at different layers: -/// -/// - **Production / the extension point: [`AsyncMessagePublisher`]**. The async -/// [`OutboxDispatcher`] drains durable rows and publishes them through an -/// `AsyncMessagePublisher`. [`BusPublisher`] adapts any [`Bus`] into one, and -/// `service.with_bus(bus)` wires this path automatically so -/// `repo.outbox(msg).commit(agg)` publishes on commit. Implement -/// `AsyncMessagePublisher` to plug in a custom transport. -/// - **Dev/test only: this trait + [`OutboxWorker`] + [`LogPublisher`]**. A -/// synchronous, in-process processor with no async runtime and no real -/// transport. Useful in unit tests and examples that just need to observe -/// that rows drain; not the production drain path. +/// Publisher trait used by [`OutboxWorker`] to process loaded outbox messages. /// /// [`AsyncMessagePublisher`]: crate::bus::AsyncMessagePublisher -/// [`OutboxDispatcher`]: crate::OutboxDispatcher -/// [`BusPublisher`]: crate::BusPublisher -/// [`Bus`]: crate::bus::Bus /// [`OutboxWorker`]: crate::OutboxWorker /// [`LogPublisher`]: crate::LogPublisher pub trait OutboxPublisher { @@ -36,12 +22,12 @@ pub trait OutboxPublisher { /// Publish an event with the given type, payload bytes, and metadata. /// The publisher is responsible for converting the payload to the appropriate format /// (e.g., decoding bitcode and re-encoding to JSON for CloudEvents). - fn publish( - &mut self, - event_type: &str, - payload: &[u8], - metadata: &HashMap, - ) -> Result<(), Self::Error>; + fn publish<'a>( + &'a mut self, + event_type: &'a str, + payload: &'a [u8], + metadata: &'a HashMap, + ) -> impl Future> + 'a; } #[derive(Debug, Clone, PartialEq, Eq)] @@ -59,13 +45,7 @@ impl fmt::Display for LogPublisherError { impl std::error::Error for LogPublisherError {} -/// A simple **dev/test** publisher that logs events to stdout or a buffer. -/// -/// It is the in-memory default for the synchronous [`OutboxPublisher`] / -/// [`OutboxWorker`] processor. For production publishing implement -/// [`AsyncMessagePublisher`](crate::bus::AsyncMessagePublisher) (or use -/// [`BusPublisher`](crate::BusPublisher) over a real [`Bus`](crate::bus::Bus)); -/// see [`OutboxPublisher`] for the full comparison of the two paths. +/// A simple publisher that logs events to stdout or a buffer. pub struct LogPublisher { buffer: Option>>>, } @@ -91,28 +71,30 @@ impl LogPublisher { impl OutboxPublisher for LogPublisher { type Error = LogPublisherError; - fn publish( - &mut self, - event_type: &str, - payload: &[u8], - metadata: &HashMap, - ) -> Result<(), Self::Error> { - let payload_str = String::from_utf8_lossy(payload); - let meta_str = if metadata.is_empty() { - String::new() - } else { - format!(" meta={:?}", metadata) - }; - let line = format!("[OUTBOX] {} {}{}", event_type, payload_str, meta_str); - if let Some(buffer) = &self.buffer { - let mut buffer = buffer - .lock() - .map_err(|_| LogPublisherError::BufferPoisoned)?; - buffer.push(line); - } else { - println!("{}", line); + fn publish<'a>( + &'a mut self, + event_type: &'a str, + payload: &'a [u8], + metadata: &'a HashMap, + ) -> impl Future> + 'a { + async move { + let payload_str = String::from_utf8_lossy(payload); + let meta_str = if metadata.is_empty() { + String::new() + } else { + format!(" meta={:?}", metadata) + }; + let line = format!("[OUTBOX] {} {}{}", event_type, payload_str, meta_str); + if let Some(buffer) = &self.buffer { + let mut buffer = buffer + .lock() + .map_err(|_| LogPublisherError::BufferPoisoned)?; + buffer.push(line); + } else { + println!("{}", line); + } + Ok(()) } - Ok(()) } } @@ -134,16 +116,17 @@ impl LocalEmitterPublisher { impl OutboxPublisher for LocalEmitterPublisher { type Error = std::convert::Infallible; - fn publish( - &mut self, - event_type: &str, - payload: &[u8], - _metadata: &HashMap, - ) -> Result<(), Self::Error> { - // Convert bytes to string for the event emitter (assumes UTF-8) - let payload_str = String::from_utf8_lossy(payload).into_owned(); - self.emitter.emit(event_type, payload_str); - Ok(()) + fn publish<'a>( + &'a mut self, + event_type: &'a str, + payload: &'a [u8], + _metadata: &'a HashMap, + ) -> impl Future> + 'a { + async move { + let payload_str = String::from_utf8_lossy(payload).into_owned(); + self.emitter.emit(event_type, payload_str); + Ok(()) + } } } @@ -151,17 +134,19 @@ impl OutboxPublisher for LocalEmitterPublisher { mod tests { use super::*; - #[test] - fn log_publisher_to_buffer() { + #[tokio::test] + async fn log_publisher_to_buffer() { let buffer = Arc::new(Mutex::new(Vec::new())); let mut publisher = LogPublisher::with_buffer(buffer.clone()); let empty = HashMap::new(); publisher .publish("UserCreated", br#"{"id":"123"}"#, &empty) + .await .unwrap(); publisher .publish("UserUpdated", br#"{"id":"123","name":"Alice"}"#, &empty) + .await .unwrap(); let logs = buffer.lock().unwrap(); @@ -170,14 +155,17 @@ mod tests { assert!(logs[1].contains("UserUpdated")); } - #[test] - fn log_publisher_includes_metadata() { + #[tokio::test] + async fn log_publisher_includes_metadata() { let buffer = Arc::new(Mutex::new(Vec::new())); let mut publisher = LogPublisher::with_buffer(buffer.clone()); let mut meta = HashMap::new(); meta.insert("correlation_id".to_string(), "req-123".to_string()); - publisher.publish("UserCreated", br#"{}"#, &meta).unwrap(); + publisher + .publish("UserCreated", br#"{}"#, &meta) + .await + .unwrap(); let logs = buffer.lock().unwrap(); assert!(logs[0].contains("correlation_id")); diff --git a/src/outbox_worker/store.rs b/src/outbox_worker/store.rs index a4511dc..2b250b0 100644 --- a/src/outbox_worker/store.rs +++ b/src/outbox_worker/store.rs @@ -94,44 +94,41 @@ impl OutboxClaimRef { } /// Async store capability for claiming and updating durable outbox messages. -pub trait AsyncOutboxStore: Send + Sync { - fn messages_by_status_async( +pub trait OutboxStore: Send + Sync { + fn messages_by_status( &self, status: OutboxMessageStatus, ) -> impl Future, RepositoryError>> + Send + '_; - fn pending_async( + fn pending( &self, ) -> impl Future, RepositoryError>> + Send + '_ { - async move { - self.messages_by_status_async(OutboxMessageStatus::Pending) - .await - } + async move { self.messages_by_status(OutboxMessageStatus::Pending).await } } - fn claim_async<'a>( + fn claim<'a>( &'a self, request: ClaimOutboxMessages, ) -> impl Future, RepositoryError>> + Send + 'a; - fn complete_async<'a>( + fn complete<'a>( &'a self, claim: &'a OutboxClaimRef, ) -> impl Future> + Send + 'a; - fn release_async<'a>( + fn release<'a>( &'a self, claim: &'a OutboxClaimRef, error: &'a str, ) -> impl Future> + Send + 'a; - fn fail_async<'a>( + fn fail<'a>( &'a self, claim: &'a OutboxClaimRef, error: &'a str, ) -> impl Future> + Send + 'a; - fn record_failure_async<'a>( + fn record_failure<'a>( &'a self, claim: &'a OutboxClaimRef, error: &'a str, @@ -139,10 +136,10 @@ pub trait AsyncOutboxStore: Send + Sync { ) -> impl Future> + Send + 'a { async move { if claim.attempt >= max_attempts { - self.fail_async(claim, error).await?; + self.fail(claim, error).await?; Ok(OutboxPublishFailureAction::Failed) } else { - self.release_async(claim, error).await?; + self.release(claim, error).await?; Ok(OutboxPublishFailureAction::Released) } } @@ -237,8 +234,8 @@ impl HashMapOutboxStore { } } -impl AsyncOutboxStore for HashMapOutboxStore { - fn messages_by_status_async( +impl OutboxStore for HashMapOutboxStore { + fn messages_by_status( &self, status: OutboxMessageStatus, ) -> impl Future, RepositoryError>> + Send + '_ { @@ -258,7 +255,7 @@ impl AsyncOutboxStore for HashMapOutboxStore { } } - fn claim_async<'a>( + fn claim<'a>( &'a self, request: ClaimOutboxMessages, ) -> impl Future, RepositoryError>> + Send + 'a { @@ -303,7 +300,7 @@ impl AsyncOutboxStore for HashMapOutboxStore { } } - fn complete_async<'a>( + fn complete<'a>( &'a self, claim: &'a OutboxClaimRef, ) -> impl Future> + Send + 'a { @@ -316,7 +313,7 @@ impl AsyncOutboxStore for HashMapOutboxStore { } } - fn release_async<'a>( + fn release<'a>( &'a self, claim: &'a OutboxClaimRef, error: &'a str, @@ -330,7 +327,7 @@ impl AsyncOutboxStore for HashMapOutboxStore { } } - fn fail_async<'a>( + fn fail<'a>( &'a self, claim: &'a OutboxClaimRef, error: &'a str, @@ -380,7 +377,7 @@ mod tests { let store = repo.outbox_store(); let claimed = store - .claim_async(ClaimOutboxMessages::new( + .claim(ClaimOutboxMessages::new( "worker-2", 1, Duration::from_secs(60), @@ -409,7 +406,7 @@ mod tests { let store = repo.outbox_store(); let claimed = store - .claim_async(ClaimOutboxMessages::new( + .claim(ClaimOutboxMessages::new( "worker-2", 1, Duration::from_secs(60), @@ -435,7 +432,7 @@ mod tests { let claimed = repo .outbox_store() - .claim_async(ClaimOutboxMessages::new( + .claim(ClaimOutboxMessages::new( "worker-1", 1, Duration::from_secs(60), @@ -467,7 +464,7 @@ mod tests { let claimed = repo .outbox_store() - .claim_async(ClaimOutboxMessages::for_ids( + .claim(ClaimOutboxMessages::for_ids( "worker-1", vec!["msg-b".to_string(), "msg-c".to_string()], Duration::from_secs(60), @@ -498,7 +495,7 @@ mod tests { // not an error. let claimed = repo .outbox_store() - .claim_async(ClaimOutboxMessages::for_ids( + .claim(ClaimOutboxMessages::for_ids( "worker-1", vec!["msg-a".to_string(), "missing".to_string()], Duration::from_secs(60), @@ -545,7 +542,7 @@ mod tests { let worker_a = thread::spawn(move || { barrier_a.wait(); let rt = tokio::runtime::Runtime::new().unwrap(); - rt.block_on(store_a.claim_async(ClaimOutboxMessages::new( + rt.block_on(store_a.claim(ClaimOutboxMessages::new( "worker-a", 1, Duration::from_secs(60), @@ -556,7 +553,7 @@ mod tests { let worker_b = thread::spawn(move || { barrier_b.wait(); let rt = tokio::runtime::Runtime::new().unwrap(); - rt.block_on(store_b.claim_async(ClaimOutboxMessages::new( + rt.block_on(store_b.claim(ClaimOutboxMessages::new( "worker-b", 1, Duration::from_secs(60), @@ -582,7 +579,7 @@ mod tests { let store = repo.outbox_store(); let claimed = store - .claim_async(ClaimOutboxMessages::new( + .claim(ClaimOutboxMessages::new( "worker-1", 1, Duration::from_secs(60), @@ -591,7 +588,7 @@ mod tests { .unwrap(); let claim = OutboxClaimRef::from_message(&claimed[0]).unwrap(); let action = store - .record_failure_async(&claim, "first failure", 2) + .record_failure(&claim, "first failure", 2) .await .unwrap(); assert_eq!(action, OutboxPublishFailureAction::Released); @@ -602,7 +599,7 @@ mod tests { assert_eq!(stored.last_error.as_deref(), Some("first failure")); let claimed = store - .claim_async(ClaimOutboxMessages::new( + .claim(ClaimOutboxMessages::new( "worker-1", 1, Duration::from_secs(60), @@ -611,7 +608,7 @@ mod tests { .unwrap(); let claim = OutboxClaimRef::from_message(&claimed[0]).unwrap(); let action = store - .record_failure_async(&claim, "second failure", 2) + .record_failure(&claim, "second failure", 2) .await .unwrap(); assert_eq!(action, OutboxPublishFailureAction::Failed); @@ -635,13 +632,11 @@ mod tests { }; let is_missing = |err: RepositoryError| matches!(&err, RepositoryError::NotFound { id } if id == "missing"); - assert!(is_missing(store.complete_async(&claim).await.unwrap_err())); - assert!(is_missing( - store.release_async(&claim, "error").await.unwrap_err() - )); + assert!(is_missing(store.complete(&claim).await.unwrap_err())); assert!(is_missing( - store.fail_async(&claim, "error").await.unwrap_err() + store.release(&claim, "error").await.unwrap_err() )); + assert!(is_missing(store.fail(&claim, "error").await.unwrap_err())); } #[tokio::test] @@ -652,7 +647,7 @@ mod tests { let store = repo.outbox_store(); let claimed = store - .claim_async(ClaimOutboxMessages::new( + .claim(ClaimOutboxMessages::new( "worker-1", 1, Duration::from_secs(60), @@ -661,7 +656,7 @@ mod tests { .unwrap(); let mut claim = OutboxClaimRef::from_message(&claimed[0]).unwrap(); claim.worker_id = "worker-2".into(); - let err = store.complete_async(&claim).await.unwrap_err(); + let err = store.complete(&claim).await.unwrap_err(); assert!(matches!(err, RepositoryError::InvalidState { .. })); let mut expired = OutboxMessage::create("msg-2", "Event", b"{}".to_vec()).unwrap(); @@ -671,7 +666,7 @@ mod tests { let expired_id = store_message(&repo, expired).await; let expired = load_message(&repo, &expired_id); let claim = OutboxClaimRef::from_message(&expired).unwrap(); - let err = store.complete_async(&claim).await.unwrap_err(); + let err = store.complete(&claim).await.unwrap_err(); assert!(matches!(err, RepositoryError::InvalidState { .. })); } @@ -683,7 +678,7 @@ mod tests { let store = repo.outbox_store(); let claimed = store - .claim_async(ClaimOutboxMessages::new( + .claim(ClaimOutboxMessages::new( "worker-1", 1, Duration::from_secs(60), @@ -691,10 +686,10 @@ mod tests { .await .unwrap(); let stale_claim = OutboxClaimRef::from_message(&claimed[0]).unwrap(); - store.release_async(&stale_claim, "retry").await.unwrap(); + store.release(&stale_claim, "retry").await.unwrap(); let claimed = store - .claim_async(ClaimOutboxMessages::new( + .claim(ClaimOutboxMessages::new( "worker-1", 1, Duration::from_secs(60), @@ -703,9 +698,9 @@ mod tests { .unwrap(); let current_claim = OutboxClaimRef::from_message(&claimed[0]).unwrap(); - let err = store.complete_async(&stale_claim).await.unwrap_err(); + let err = store.complete(&stale_claim).await.unwrap_err(); assert!(matches!(err, RepositoryError::InvalidState { .. })); - store.complete_async(¤t_claim).await.unwrap(); + store.complete(¤t_claim).await.unwrap(); } #[tokio::test] @@ -716,7 +711,7 @@ mod tests { let store = repo.outbox_store(); let claimed = store - .claim_async(ClaimOutboxMessages::new( + .claim(ClaimOutboxMessages::new( "worker-1", 1, Duration::from_secs(60), @@ -724,9 +719,9 @@ mod tests { .await .unwrap(); let claim = OutboxClaimRef::from_message(&claimed[0]).unwrap(); - store.complete_async(&claim).await.unwrap(); + store.complete(&claim).await.unwrap(); - let err = store.complete_async(&claim).await.unwrap_err(); + let err = store.complete(&claim).await.unwrap_err(); assert!(matches!(err, RepositoryError::InvalidState { .. })); } } diff --git a/src/outbox_worker/worker.rs b/src/outbox_worker/worker.rs index 5ca7686..c171a43 100644 --- a/src/outbox_worker/worker.rs +++ b/src/outbox_worker/worker.rs @@ -28,16 +28,7 @@ pub struct ProcessOneResult { pub failed: bool, } -/// Synchronous, in-process worker for processing outbox messages — a **dev/test** -/// drain loop. -/// -/// The repository is responsible for claiming pending messages. The worker -/// processes messages that are already loaded and (optionally) claimed via a -/// synchronous [`OutboxPublisher`]. -/// -/// For the production drain path use the async [`OutboxDispatcher`] with an -/// [`AsyncMessagePublisher`] (e.g. [`BusPublisher`] over a real [`Bus`]); see -/// [`OutboxPublisher`] for the full comparison. +/// Async worker for processing loaded outbox messages through an [`OutboxPublisher`]. /// /// [`OutboxDispatcher`]: crate::OutboxDispatcher /// [`AsyncMessagePublisher`]: crate::bus::AsyncMessagePublisher @@ -104,7 +95,7 @@ impl OutboxWorker

{ /// If the message is pending, it will be claimed by this worker before /// publishing. Store-backed worker loops should persist the outcome with /// the outbox store completion or failure APIs. - pub fn process_message( + pub async fn process_message( &mut self, message: &mut OutboxMessage, ) -> SourcedResult { @@ -122,51 +113,54 @@ impl OutboxWorker

{ return Ok(ProcessOneResult::default()); } - let result = - match self - .publisher - .publish(&message.event_type, &message.payload, &message.metadata) - { - Ok(()) => { - message.complete()?; + let result = match self + .publisher + .publish(&message.event_type, &message.payload, &message.metadata) + .await + { + Ok(()) => { + message.complete()?; + ProcessOneResult { + did_work: true, + claimed, + completed: true, + ..Default::default() + } + } + Err(err) => { + let error_msg = err.to_string(); + if message.attempts >= self.max_attempts { + message.fail(error_msg)?; ProcessOneResult { did_work: true, claimed, - completed: true, + failed: true, ..Default::default() } - } - Err(err) => { - let error_msg = err.to_string(); - if message.attempts >= self.max_attempts { - message.fail(error_msg)?; - ProcessOneResult { - did_work: true, - claimed, - failed: true, - ..Default::default() - } - } else { - message.release(error_msg)?; - ProcessOneResult { - did_work: true, - claimed, - released: true, - ..Default::default() - } + } else { + message.release(error_msg)?; + ProcessOneResult { + did_work: true, + claimed, + released: true, + ..Default::default() } } - }; + } + }; Ok(result) } /// Process a batch of outbox messages. - pub fn process_batch(&mut self, messages: &mut [OutboxMessage]) -> SourcedResult { + pub async fn process_batch( + &mut self, + messages: &mut [OutboxMessage], + ) -> SourcedResult { let mut result = DrainResult::default(); for message in messages.iter_mut().take(self.batch_size) { - let processed = self.process_message(message)?; + let processed = self.process_message(message).await?; if processed.claimed { result.claimed += 1; } @@ -204,19 +198,19 @@ mod tests { assert_eq!(worker.max_attempts, 2); } - #[test] - fn process_message_noop_for_published() { + #[tokio::test] + async fn process_message_noop_for_published() { let mut message = OutboxMessage::create("msg-1", "Event", b"{}".to_vec()).unwrap(); message.claim_for("worker", Duration::from_secs(1)).unwrap(); message.complete().unwrap(); let mut worker = OutboxWorker::new(LogPublisher::default()); - let result = worker.process_message(&mut message).unwrap(); + let result = worker.process_message(&mut message).await.unwrap(); assert!(!result.did_work); } - #[test] - fn process_message_passes_metadata_to_publisher() { + #[tokio::test] + async fn process_message_passes_metadata_to_publisher() { use std::sync::{Arc, Mutex}; let buffer = Arc::new(Mutex::new(Vec::new())); @@ -226,7 +220,7 @@ mod tests { let mut message = OutboxMessage::create("msg-1", "UserCreated", b"{}".to_vec()).unwrap(); message.set_correlation_id("req-abc"); - let result = worker.process_message(&mut message).unwrap(); + let result = worker.process_message(&mut message).await.unwrap(); assert!(result.completed); let logs = buffer.lock().unwrap(); @@ -234,19 +228,19 @@ mod tests { assert!(logs[0].contains("req-abc")); } - #[test] - fn process_batch_counts_pending_messages_claimed_by_this_call() { + #[tokio::test] + async fn process_batch_counts_pending_messages_claimed_by_this_call() { let mut messages = vec![OutboxMessage::create("msg-1", "Event", b"{}".to_vec()).unwrap()]; let mut worker = OutboxWorker::new(LogPublisher::default()); - let result = worker.process_batch(&mut messages).unwrap(); + let result = worker.process_batch(&mut messages).await.unwrap(); assert_eq!(result.claimed, 1); assert_eq!(result.completed, 1); } - #[test] - fn process_batch_does_not_count_already_in_flight_messages_as_claimed() { + #[tokio::test] + async fn process_batch_does_not_count_already_in_flight_messages_as_claimed() { let mut message = OutboxMessage::create("msg-1", "Event", b"{}".to_vec()).unwrap(); message .claim_for("other-worker", Duration::from_secs(1)) @@ -254,7 +248,7 @@ mod tests { let mut messages = vec![message]; let mut worker = OutboxWorker::new(LogPublisher::default()); - let result = worker.process_batch(&mut messages).unwrap(); + let result = worker.process_batch(&mut messages).await.unwrap(); assert_eq!(result.claimed, 0); assert_eq!(result.completed, 1); diff --git a/src/postgres_repo/mod.rs b/src/postgres_repo/mod.rs index 5ec666a..06e8bcf 100644 --- a/src/postgres_repo/mod.rs +++ b/src/postgres_repo/mod.rs @@ -19,9 +19,7 @@ use sqlx::{PgPool, Postgres, QueryBuilder, Row, Transaction}; use crate::entity::Entity; use crate::entity::EventRecord; use crate::outbox::{OutboxMessage, OutboxMessageStatus}; -use crate::outbox_worker::{ - ensure_active_claim, AsyncOutboxStore, ClaimOutboxMessages, OutboxClaimRef, -}; +use crate::outbox_worker::{ensure_active_claim, ClaimOutboxMessages, OutboxClaimRef, OutboxStore}; use crate::read_model::{ ColumnDef, ColumnType, ReadModelAdapterCapabilities, ReadModelCommitOutcome, ReadModelError, ReadModelLoadGraph, ReadModelLoadRequest, ReadModelQueryCapabilities, ReadModelWritePlan, @@ -520,8 +518,8 @@ impl RelationalReadModelQueryStore for PostgresRepository { } } -impl AsyncOutboxStore for PostgresOutboxStore { - fn messages_by_status_async( +impl OutboxStore for PostgresOutboxStore { + fn messages_by_status( &self, status: OutboxMessageStatus, ) -> impl Future, RepositoryError>> + Send + '_ { @@ -536,7 +534,7 @@ impl AsyncOutboxStore for PostgresOutboxStore { } } - fn claim_async<'a>( + fn claim<'a>( &'a self, request: ClaimOutboxMessages, ) -> impl Future, RepositoryError>> + Send + 'a { @@ -627,7 +625,7 @@ impl AsyncOutboxStore for PostgresOutboxStore { } } - fn complete_async<'a>( + fn complete<'a>( &'a self, claim: &'a OutboxClaimRef, ) -> impl Future> + Send + 'a { @@ -676,7 +674,7 @@ impl AsyncOutboxStore for PostgresOutboxStore { } } - fn release_async<'a>( + fn release<'a>( &'a self, claim: &'a OutboxClaimRef, error: &'a str, @@ -728,7 +726,7 @@ impl AsyncOutboxStore for PostgresOutboxStore { } } - fn fail_async<'a>( + fn fail<'a>( &'a self, claim: &'a OutboxClaimRef, error: &'a str, diff --git a/src/sqlite_repo/mod.rs b/src/sqlite_repo/mod.rs index 1b1f589..e1b76ea 100644 --- a/src/sqlite_repo/mod.rs +++ b/src/sqlite_repo/mod.rs @@ -18,9 +18,7 @@ use sqlx::{QueryBuilder, Row, Sqlite, SqlitePool, Transaction}; use crate::entity::{Entity, EventRecord}; use crate::outbox::{OutboxMessage, OutboxMessageStatus}; -use crate::outbox_worker::{ - ensure_active_claim, AsyncOutboxStore, ClaimOutboxMessages, OutboxClaimRef, -}; +use crate::outbox_worker::{ensure_active_claim, ClaimOutboxMessages, OutboxClaimRef, OutboxStore}; use crate::read_model::{ ColumnDef, ColumnType, ReadModelAdapterCapabilities, ReadModelCommitOutcome, ReadModelError, ReadModelLoadGraph, ReadModelLoadRequest, ReadModelQueryCapabilities, ReadModelWritePlan, @@ -501,8 +499,8 @@ impl RelationalReadModelQueryStore for SqliteRepository { } } -impl AsyncOutboxStore for SqliteOutboxStore { - fn messages_by_status_async( +impl OutboxStore for SqliteOutboxStore { + fn messages_by_status( &self, status: OutboxMessageStatus, ) -> impl Future, RepositoryError>> + Send + '_ { @@ -528,7 +526,7 @@ impl AsyncOutboxStore for SqliteOutboxStore { } } - fn claim_async<'a>( + fn claim<'a>( &'a self, request: ClaimOutboxMessages, ) -> impl Future, RepositoryError>> + Send + 'a { @@ -649,7 +647,7 @@ impl AsyncOutboxStore for SqliteOutboxStore { } } - fn complete_async<'a>( + fn complete<'a>( &'a self, claim: &'a OutboxClaimRef, ) -> impl Future> + Send + 'a { @@ -698,7 +696,7 @@ impl AsyncOutboxStore for SqliteOutboxStore { } } - fn release_async<'a>( + fn release<'a>( &'a self, claim: &'a OutboxClaimRef, error: &'a str, @@ -751,7 +749,7 @@ impl AsyncOutboxStore for SqliteOutboxStore { } } - fn fail_async<'a>( + fn fail<'a>( &'a self, claim: &'a OutboxClaimRef, error: &'a str, diff --git a/tests/bomberman/main.rs b/tests/bomberman/main.rs index 8938964..e694427 100644 --- a/tests/bomberman/main.rs +++ b/tests/bomberman/main.rs @@ -208,10 +208,10 @@ async fn player_killed_by_bomb() { .contains(&"player:p2".to_string())); // Verify outbox message was created (PlayerKilled) - use distributed::{AsyncOutboxStore, OutboxMessageStatus}; + use distributed::{OutboxMessageStatus, OutboxStore}; let pending = repo2 .outbox_store() - .messages_by_status_async(OutboxMessageStatus::Pending) + .messages_by_status(OutboxMessageStatus::Pending) .await .unwrap(); assert!(!pending.is_empty()); diff --git a/tests/distributed_read_model/main.rs b/tests/distributed_read_model/main.rs index 6eb9779..2d21392 100644 --- a/tests/distributed_read_model/main.rs +++ b/tests/distributed_read_model/main.rs @@ -26,7 +26,7 @@ use distributed::microsvc::{Context, Service, Session}; #[cfg(feature = "sqlite")] use distributed::SqliteRepository; use distributed::{ - AggregateBuilder, AsyncOutboxStore, CommitBuilderExt, GetStream, OutboxMessage, ReadModelError, + AggregateBuilder, CommitBuilderExt, GetStream, OutboxMessage, OutboxStore, ReadModelError, ReadModelWritePlanBuilder, ReadModelWritePlanStore, RelationalReadModel, RelationalReadModelIncludes, RelationalReadModelQueryStore, TransactionalCommit, }; @@ -87,8 +87,8 @@ async fn run_persistent_checkout_flow( + Send + Sync + 'static, - CheckoutOutbox: AsyncOutboxStore + Send + Sync, - SeatOutbox: AsyncOutboxStore + Send + Sync, + CheckoutOutbox: OutboxStore + Send + Sync, + SeatOutbox: OutboxStore + Send + Sync, { let seat_added = add_seat(&seat_repo, &ids.seat_id, &ids.category).await; assert_pending(&seat_outbox, &seat_added).await; @@ -367,10 +367,10 @@ where #[allow(dead_code)] async fn assert_pending(store: &S, message: &OutboxMessage) where - S: AsyncOutboxStore + Send + Sync, + S: OutboxStore + Send + Sync, { let pending = store - .pending_async() + .pending() .await .expect("pending outbox messages should load"); assert!( @@ -455,7 +455,7 @@ async fn publish_pending_outbox( bus: &distributed::bus::InMemoryBus, ) { let claimed = outbox - .claim_async(distributed::ClaimOutboxMessages::new( + .claim(distributed::ClaimOutboxMessages::new( "matrix-outbox-bridge", 64, Duration::from_secs(60), @@ -475,7 +475,7 @@ async fn publish_pending_outbox( let claim = distributed::OutboxClaimRef::from_message(&message) .expect("claimed message should yield a claim ref"); outbox - .complete_async(&claim) + .complete(&claim) .await .expect("forwarded outbox message should complete"); } diff --git a/tests/distributed_read_model_board/main.rs b/tests/distributed_read_model_board/main.rs index 49ab202..c36c15e 100644 --- a/tests/distributed_read_model_board/main.rs +++ b/tests/distributed_read_model_board/main.rs @@ -20,8 +20,8 @@ use board_service::{AddCard, MoveCard, OpenBoard, RemoveCard}; use distributed::bus::{Bus, BusConsumer, InMemoryBus, RunOptions}; use distributed::microsvc::{Message, MessageKind, Service, Session}; use distributed::{ - AggregateBuilder, AsyncOutboxStore, ClaimOutboxMessages, HashMapOutboxStore, HashMapRepository, - InMemoryReadModelStore, OutboxClaimRef, Queueable, + AggregateBuilder, ClaimOutboxMessages, HashMapOutboxStore, HashMapRepository, + InMemoryReadModelStore, OutboxClaimRef, OutboxStore, Queueable, }; use projections_service::{load_board, service as build_projection}; use query_service::BoardQueryService; @@ -49,7 +49,7 @@ where /// (bitcode), and the projection decodes them with `BitcodePayloadCodec`. async fn publish_pending_outbox(outbox: &HashMapOutboxStore, bus: &InMemoryBus) { let claimed = outbox - .claim_async(ClaimOutboxMessages::new( + .claim(ClaimOutboxMessages::new( "board-outbox-bridge", 64, Duration::from_secs(60), @@ -69,7 +69,7 @@ async fn publish_pending_outbox(outbox: &HashMapOutboxStore, bus: &InMemoryBus) .expect("board event should publish to the bus"); let claim = OutboxClaimRef::from_message(&message).expect("claimed message yields a ref"); outbox - .complete_async(&claim) + .complete(&claim) .await .expect("forwarded event should complete"); } diff --git a/tests/durable_enqueue_sqlite/main.rs b/tests/durable_enqueue_sqlite/main.rs index 73ba160..95ae99d 100644 --- a/tests/durable_enqueue_sqlite/main.rs +++ b/tests/durable_enqueue_sqlite/main.rs @@ -11,8 +11,8 @@ use serde_json::{json, Value}; use distributed::bus::{Bus, InMemoryBus, RunOptions}; use distributed::microsvc::{Context, HandlerError, HasOutboxStore, Service, Session}; use distributed::{ - sourced, AggregateBuilder, AggregateRepository, AsyncOutboxStore, Entity, OutboxMessage, - OutboxMessageStatus, Queueable, QueuedRepository, SqliteRepository, + sourced, AggregateBuilder, AggregateRepository, Entity, OutboxMessage, OutboxMessageStatus, + OutboxStore, Queueable, QueuedRepository, SqliteRepository, }; #[derive(Default)] @@ -66,13 +66,13 @@ async fn commit_publishes_immediately_over_sqlite() { let store = service.repo().outbox_store(); let published = store - .messages_by_status_async(OutboxMessageStatus::Published) + .messages_by_status(OutboxMessageStatus::Published) .await .unwrap(); assert_eq!(published.len(), 1, "row should be published immediately"); assert_eq!(published[0].id(), "evt-c1"); assert!( - store.pending_async().await.unwrap().is_empty(), + store.pending().await.unwrap().is_empty(), "nothing should be left for the poller" ); } @@ -93,7 +93,7 @@ async fn run_consumes_command_and_publishes_over_sqlite() { service.run(RunOptions::idempotent()).await.unwrap(); let published = store - .messages_by_status_async(OutboxMessageStatus::Published) + .messages_by_status(OutboxMessageStatus::Published) .await .unwrap(); assert_eq!(published.len(), 1); diff --git a/tests/microsvc/convention.rs b/tests/microsvc/convention.rs index b5329e0..aa2b5bb 100644 --- a/tests/microsvc/convention.rs +++ b/tests/microsvc/convention.rs @@ -8,7 +8,7 @@ //! Registration uses the `register_handlers!` macro. use distributed::microsvc::{Service, Session}; -use distributed::{AggregateBuilder, AsyncOutboxStore, HashMapRepository, Queueable}; +use distributed::{AggregateBuilder, HashMapRepository, OutboxStore, Queueable}; use serde_json::json; use crate::handlers; @@ -108,7 +108,7 @@ async fn create_persists_outbox_message() { assert_eq!(counter.value, 0); // Outbox message was persisted - let pending = inner.outbox_store().pending_async().await.unwrap(); + let pending = inner.outbox_store().pending().await.unwrap(); assert_eq!(pending.len(), 1); assert_eq!(pending[0].event_type, "counter.initialized"); } @@ -136,7 +136,7 @@ async fn duplicate_create_leaves_single_outbox_message() { .repo() .inner() .outbox_store() - .pending_async() + .pending() .await .unwrap(); assert_eq!(pending.len(), 1); @@ -170,7 +170,7 @@ async fn increment_persists_outbox_message() { // Both outbox messages were persisted let inner = service.repo().repo().inner(); - let pending = inner.outbox_store().pending_async().await.unwrap(); + let pending = inner.outbox_store().pending().await.unwrap(); assert_eq!(pending.len(), 2); let mut event_types: Vec<&str> = pending.iter().map(|m| m.event_type.as_str()).collect(); event_types.sort(); diff --git a/tests/persistent_repository_conformance/inbox.rs b/tests/persistent_repository_conformance/inbox.rs index 1ff63b0..334ce41 100644 --- a/tests/persistent_repository_conformance/inbox.rs +++ b/tests/persistent_repository_conformance/inbox.rs @@ -2,7 +2,7 @@ //! in-memory, SQLite, and Postgres backends. use distributed::{ - AsyncOutboxStore, CommitBatch, InboxReceipt, InboxStore, OutboxMessage, OutboxMessageStatus, + CommitBatch, InboxReceipt, InboxStore, OutboxMessage, OutboxMessageStatus, OutboxStore, RepositoryError, TransactionalCommit, }; @@ -15,7 +15,7 @@ fn batch_with(outbox: Vec, receipts: Vec) -> Commit batch } -async fn outbox_present(outbox: &S, id: &str) -> bool { +async fn outbox_present(outbox: &S, id: &str) -> bool { for status in [ OutboxMessageStatus::Pending, OutboxMessageStatus::InFlight, @@ -23,7 +23,7 @@ async fn outbox_present(outbox: &S, id: &str) OutboxMessageStatus::Failed, ] { let messages = outbox - .messages_by_status_async(status) + .messages_by_status(status) .await .expect("outbox status lookup should succeed"); if messages.iter().any(|m| m.id() == id) { @@ -40,7 +40,7 @@ async fn outbox_present(outbox: &S, id: &str) pub async fn inbox_records_dedupes_and_fences_with_real_effects(repo: R, outbox: S) where R: InboxStore + TransactionalCommit + Clone + Send + Sync + 'static, - S: AsyncOutboxStore + Send + Sync, + S: OutboxStore + Send + Sync, { let consumer = unique_id("consumer"); let m1 = unique_id("msg"); diff --git a/tests/persistent_repository_conformance/outbox.rs b/tests/persistent_repository_conformance/outbox.rs index 9b7eb8d..f1c662b 100644 --- a/tests/persistent_repository_conformance/outbox.rs +++ b/tests/persistent_repository_conformance/outbox.rs @@ -3,9 +3,9 @@ use std::time::Duration; use distributed::bus::{AsyncMessagePublisher, Message, TransportError}; use distributed::{ - Aggregate, AggregateBuilder, AsyncOutboxStore, ClaimOutboxMessages, GetStream, OutboxClaimRef, - OutboxDispatcher, OutboxMessage, OutboxMessageStatus, OutboxPublishFailureAction, - RepositoryError, StreamIdentity, TransactionalCommit, + Aggregate, AggregateBuilder, ClaimOutboxMessages, GetStream, OutboxClaimRef, OutboxDispatcher, + OutboxMessage, OutboxMessageStatus, OutboxPublishFailureAction, OutboxStore, RepositoryError, + StreamIdentity, TransactionalCommit, }; use super::scenario::unique_id; @@ -58,7 +58,7 @@ impl AsyncMessagePublisher for FlakyPublisher { pub async fn high_level_outbox_commit_persists_row_without_stream(repo: R, outbox: S) where R: GetStream + TransactionalCommit + Clone + Send + Sync + 'static, - S: AsyncOutboxStore + Send + Sync, + S: OutboxStore + Send + Sync, { let seat_id = unique_id("outbox-seat"); let message_id = unique_id("outbox-message"); @@ -132,7 +132,7 @@ where pub async fn aggregate_conflict_rolls_back_outbox(repo: R, outbox: S) where R: GetStream + TransactionalCommit + Clone + Send + Sync + 'static, - S: AsyncOutboxStore + Send + Sync, + S: OutboxStore + Send + Sync, { let seat_id = unique_id("conflict-outbox-seat"); let seat_repo = repo.clone().aggregate::(); @@ -187,7 +187,7 @@ where pub async fn worker_claim_complete_and_retry_lifecycle(repo: R, outbox: S) where R: GetStream + TransactionalCommit + Clone + Send + Sync + 'static, - S: AsyncOutboxStore + Send + Sync, + S: OutboxStore + Send + Sync, { let complete_message_id = unique_id("complete-outbox"); commit_outbox_for_seat( @@ -212,14 +212,14 @@ where attempt: claimed.attempts, }; let stale_err = outbox - .complete_async(&wrong_claim) + .complete(&wrong_claim) .await .expect_err("wrong worker should not complete a claim"); assert!(matches!(stale_err, RepositoryError::InvalidState { .. })); let claim = OutboxClaimRef::from_message(&claimed).expect("claim should be valid"); outbox - .complete_async(&claim) + .complete(&claim) .await .expect("owning worker should complete the claim"); let published = find_outbox_by_id(&outbox, &complete_message_id) @@ -243,7 +243,7 @@ where .await; let claim = OutboxClaimRef::from_message(&claimed).expect("claim should be valid"); let action = outbox - .record_failure_async(&claim, "first failure", 2) + .record_failure(&claim, "first failure", 2) .await .expect("first failure should be recorded"); assert_eq!(action, OutboxPublishFailureAction::Released); @@ -261,13 +261,13 @@ where ) .await; let stale_err = outbox - .complete_async(&claim) + .complete(&claim) .await .expect_err("stale attempt should not complete a later claim"); assert!(matches!(stale_err, RepositoryError::InvalidState { .. })); let claim = OutboxClaimRef::from_message(&claimed).expect("claim should be valid"); let action = outbox - .record_failure_async(&claim, "second failure", 2) + .record_failure(&claim, "second failure", 2) .await .expect("second failure should be recorded"); assert_eq!(action, OutboxPublishFailureAction::Failed); @@ -283,7 +283,7 @@ where pub async fn worker_claim_by_ids_claims_only_requested(repo: R, outbox: S) where R: GetStream + TransactionalCommit + Clone + Send + Sync + 'static, - S: AsyncOutboxStore + Send + Sync, + S: OutboxStore + Send + Sync, { let wanted_id = unique_id("wanted-outbox"); let other_id = unique_id("other-outbox"); @@ -308,7 +308,7 @@ where assert_eq!(other.status, OutboxMessageStatus::Pending); let empty = outbox - .claim_async(ClaimOutboxMessages::for_ids( + .claim(ClaimOutboxMessages::for_ids( "immediate-worker", vec![unique_id("never-stored")], Duration::from_secs(60), @@ -318,7 +318,7 @@ where assert!(empty.is_empty()); let leased = outbox - .claim_async(ClaimOutboxMessages::for_ids( + .claim(ClaimOutboxMessages::for_ids( "worker-b", vec![wanted_id.clone()], Duration::from_secs(60), @@ -339,7 +339,7 @@ where pub async fn expired_outbox_lease_is_reclaimed_by_second_worker(repo: R, outbox: S) where R: GetStream + TransactionalCommit + Clone + Send + Sync + 'static, - S: AsyncOutboxStore + Send + Sync, + S: OutboxStore + Send + Sync, { let message_id = unique_id("crash-lease-outbox"); commit_outbox_for_seat( @@ -356,7 +356,7 @@ where let stale_claim_a = OutboxClaimRef::from_message(&claimed_a).expect("A's claim is valid"); let live = outbox - .claim_async(ClaimOutboxMessages::new("worker-b", 1, lease)) + .claim(ClaimOutboxMessages::new("worker-b", 1, lease)) .await .expect("worker B poll should not error while the lease is live"); assert!( @@ -380,7 +380,7 @@ where let claim_b = OutboxClaimRef::from_message(&claimed_b).expect("B's claim is valid"); let late_complete = outbox - .complete_async(&stale_claim_a) + .complete(&stale_claim_a) .await .expect_err("A's late complete must be fenced"); assert!( @@ -388,7 +388,7 @@ where "stale-worker complete should be InvalidState, got {late_complete:?}" ); let late_release = outbox - .release_async(&stale_claim_a, "A woke up late") + .release(&stale_claim_a, "A woke up late") .await .expect_err("A's late release must be fenced"); assert!( @@ -397,7 +397,7 @@ where ); outbox - .complete_async(&claim_b) + .complete(&claim_b) .await .expect("the reclaiming worker should complete the row"); let published = find_outbox_by_id(&outbox, &message_id) @@ -415,7 +415,7 @@ pub async fn publish_failure_after_commit_retains_outbox_row_until_delivered OutboxMessage { async fn claim_one(outbox: &S, request: ClaimOutboxMessages) -> OutboxMessage where - S: AsyncOutboxStore + Send + Sync, + S: OutboxStore + Send + Sync, { - let claimed = outbox - .claim_async(request) - .await - .expect("claim should succeed"); + let claimed = outbox.claim(request).await.expect("claim should succeed"); assert_eq!(claimed.len(), 1); claimed.into_iter().next().expect("one claimed message") } @@ -524,7 +521,7 @@ fn added_seat(id: &str) -> Seat { async fn find_outbox_by_id(outbox: &S, id: &str) -> Option where - S: AsyncOutboxStore + Send + Sync, + S: OutboxStore + Send + Sync, { for status in [ OutboxMessageStatus::Pending, @@ -533,7 +530,7 @@ where OutboxMessageStatus::Failed, ] { let messages = outbox - .messages_by_status_async(status) + .messages_by_status(status) .await .expect("outbox status lookup should succeed"); if let Some(message) = messages.into_iter().find(|message| message.id() == id) { diff --git a/tests/postgres_repository/main.rs b/tests/postgres_repository/main.rs index a611f09..197fa5b 100644 --- a/tests/postgres_repository/main.rs +++ b/tests/postgres_repository/main.rs @@ -8,8 +8,8 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::time::{SystemTime, UNIX_EPOCH}; use distributed::{ - sourced, Aggregate, AggregateBuilder, AsyncOutboxStore, CommitBatch, Entity, GetStream, - OutboxMessage, OutboxMessageStatus, PostgresRepository, ReadModel, ReadModelWritePlanBuilder, + sourced, Aggregate, AggregateBuilder, CommitBatch, Entity, GetStream, OutboxMessage, + OutboxMessageStatus, OutboxStore, PostgresRepository, ReadModel, ReadModelWritePlanBuilder, ReadModelWritePlanCommitExt, RepositoryError, RowKey, RowPatch, RowValue, SnapshotRecord, SnapshotStore, StreamIdentity, StreamWrite, TableSchemaRegistry, TransactionalCommit, }; @@ -358,7 +358,7 @@ async fn read_model_failure_mid_plan_rolls_back_events_and_outbox() { OutboxMessageStatus::Published, OutboxMessageStatus::Failed, ] { - let rows = outbox.messages_by_status_async(status).await.unwrap(); + let rows = outbox.messages_by_status(status).await.unwrap(); assert!( rows.iter().all(|m| m.id() != outbox_id), "the outbox row must roll back" @@ -652,7 +652,7 @@ async fn outbox_metadata_columns_round_trip_into_message_metadata() { let stored = repo .outbox_store() - .messages_by_status_async(OutboxMessageStatus::Pending) + .messages_by_status(OutboxMessageStatus::Pending) .await .unwrap() .into_iter() diff --git a/tests/postgres_transport/main.rs b/tests/postgres_transport/main.rs index dc51fbc..bffb812 100644 --- a/tests/postgres_transport/main.rs +++ b/tests/postgres_transport/main.rs @@ -22,7 +22,7 @@ use distributed::bus::{ use distributed::microsvc::{Context, Message, MessageKind, Service}; use distributed::OutboxSource; use distributed::{ - AsyncOutboxStore, CommitBatch, OutboxMessage, OutboxMessageStatus, PostgresOutboxStore, + CommitBatch, OutboxMessage, OutboxMessageStatus, OutboxStore, PostgresOutboxStore, PostgresRepository, TransactionalCommit, }; use serde_json::json; @@ -47,7 +47,7 @@ async fn status(store: &PostgresOutboxStore, id: &str) -> Option usize { let claimed = outbox - .claim_async(ClaimOutboxMessages::new( + .claim(ClaimOutboxMessages::new( "saga-outbox-bridge", 64, Duration::from_secs(60), @@ -267,7 +267,7 @@ async fn publish_pending_outbox(outbox: &HashMapOutboxStore, bus: &InMemoryBus) } let claim = OutboxClaimRef::from_message(&message).expect("claimed message yields a ref"); outbox - .complete_async(&claim) + .complete(&claim) .await .expect("forwarded message should complete"); } diff --git a/tests/sourced_snapshot/main.rs b/tests/sourced_snapshot/main.rs index 5cac621..bebc386 100644 --- a/tests/sourced_snapshot/main.rs +++ b/tests/sourced_snapshot/main.rs @@ -2,7 +2,7 @@ mod aggregates; use aggregates::*; use distributed::{ - Aggregate, AggregateBuilder, AsyncOutboxStore, HashMapRepository, OutboxMessage, SnapshotStore, + Aggregate, AggregateBuilder, HashMapRepository, OutboxMessage, OutboxStore, SnapshotStore, Snapshottable, StreamIdentity, }; @@ -260,7 +260,7 @@ async fn domain_event_commits_with_outbox() { let loaded = repo.get("t1").await.unwrap().unwrap(); assert_eq!(loaded.snapshot().task, "Ship it"); - let pending = repo.repo().outbox_store().pending_async().await.unwrap(); + let pending = repo.repo().outbox_store().pending().await.unwrap(); assert_eq!(pending.len(), 1); assert!(pending[0].is_pending()); } diff --git a/tests/sqlite_repository/main.rs b/tests/sqlite_repository/main.rs index 10a652a..2f3dad5 100644 --- a/tests/sqlite_repository/main.rs +++ b/tests/sqlite_repository/main.rs @@ -4,8 +4,8 @@ use std::collections::HashMap; use distributed::table::TableSchemaRegistry; use distributed::{ - sourced, Aggregate, AggregateBuilder, AsyncOutboxStore, CommitBatch, Entity, GetStream, - OutboxMessage, OutboxMessageStatus, ReadModel, ReadModelWritePlanBuilder, + sourced, Aggregate, AggregateBuilder, CommitBatch, Entity, GetStream, OutboxMessage, + OutboxMessageStatus, OutboxStore, ReadModel, ReadModelWritePlanBuilder, ReadModelWritePlanCommitExt, RepositoryError, RowKey, RowPatch, RowValue, SnapshotRecord, SnapshotStore, SqliteRepository, StreamIdentity, StreamWrite, TransactionalCommit, OUTBOX_MESSAGES_TABLE, @@ -305,7 +305,7 @@ async fn read_model_failure_mid_plan_rolls_back_events_and_outbox() { OutboxMessageStatus::Published, OutboxMessageStatus::Failed, ] { - let rows = outbox.messages_by_status_async(status).await.unwrap(); + let rows = outbox.messages_by_status(status).await.unwrap(); assert!( rows.iter().all(|m| m.id() != outbox_id), "the outbox row must roll back with the failed read-model plan" @@ -596,7 +596,7 @@ async fn outbox_metadata_columns_round_trip_into_message_metadata() { let stored = repo .outbox_store() - .messages_by_status_async(OutboxMessageStatus::Pending) + .messages_by_status(OutboxMessageStatus::Pending) .await .unwrap() .into_iter() diff --git a/tests/todos/main.rs b/tests/todos/main.rs index f77131f..dd3bfe9 100644 --- a/tests/todos/main.rs +++ b/tests/todos/main.rs @@ -2,10 +2,10 @@ mod aggregate; use aggregate::{Todo, TodoSnapshot}; use distributed::{ - AggregateBuilder, AsyncLock, AsyncLockManager, AsyncOutboxStore, ClaimOutboxMessages, - CommitBuilderExt, DrainResult, EventEmitter, HashMapRepository, LocalEmitterPublisher, - LogPublisher, OutboxClaimRef, OutboxMessage, OutboxMessageStatus, OutboxPublisher, - OutboxWorker, Queueable, RepositoryError, + AggregateBuilder, AsyncLock, AsyncLockManager, ClaimOutboxMessages, CommitBuilderExt, + DrainResult, EventEmitter, HashMapRepository, LocalEmitterPublisher, LogPublisher, + OutboxClaimRef, OutboxMessage, OutboxMessageStatus, OutboxPublisher, OutboxStore, OutboxWorker, + Queueable, RepositoryError, }; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{mpsc, Arc, Mutex}; @@ -44,7 +44,7 @@ async fn claim_and_process( ) -> (DrainResult, Vec, Vec) { let mut claimed = repo .outbox_store() - .claim_async(ClaimOutboxMessages::new( + .claim(ClaimOutboxMessages::new( worker_id, batch_size, Duration::from_secs(30), @@ -56,7 +56,7 @@ async fn claim_and_process( .map(OutboxClaimRef::from_message) .collect::, _>>() .unwrap(); - let result = worker.process_batch(&mut claimed).unwrap(); + let result = worker.process_batch(&mut claimed).await.unwrap(); (result, claimed, claims) } @@ -68,7 +68,7 @@ async fn complete_published_outbox( let store = repo.outbox_store(); for (message, claim) in messages.iter().zip(claims) { if message.is_published() { - store.complete_async(claim).await.unwrap(); + store.complete(claim).await.unwrap(); } } } @@ -82,7 +82,7 @@ async fn load_outbox_message(repo: &HashMapRepository, id: &str) -> OutboxMessag OutboxMessageStatus::Failed, ] { if let Some(message) = store - .messages_by_status_async(status) + .messages_by_status(status) .await .unwrap() .into_iter() @@ -109,13 +109,7 @@ async fn todos() { // Verify the outbox event was captured { - let pending = repo - .repo() - .inner() - .outbox_store() - .pending_async() - .await - .unwrap(); + let pending = repo.repo().inner().outbox_store().pending().await.unwrap(); assert_eq!(pending.len(), 1); assert_eq!(pending[0].event_type, "todo.initialized"); } @@ -136,13 +130,7 @@ async fn todos() { .expect("completed todo outbox commit should succeed"); { - let pending = repo - .repo() - .inner() - .outbox_store() - .pending_async() - .await - .unwrap(); + let pending = repo.repo().inner().outbox_store().pending().await.unwrap(); assert_eq!(pending.len(), 2); assert!(pending .iter() @@ -240,7 +228,7 @@ async fn outbox_records_persisted() { repo.outbox(message).commit(&mut todo).await.unwrap(); // Check pending outbox messages - let pending = repo.outbox_store().pending_async().await.unwrap(); + let pending = repo.outbox_store().pending().await.unwrap(); assert_eq!(pending.len(), 1); assert_eq!(pending[0].event_type, "todo.initialized"); @@ -556,7 +544,7 @@ async fn outbox_worker_process_next_with_commit() { loop { let store = repo.outbox_store(); let mut claimed = store - .claim_async(ClaimOutboxMessages::new( + .claim(ClaimOutboxMessages::new( "safe-worker", 1, Duration::from_secs(30), @@ -571,7 +559,7 @@ async fn outbox_worker_process_next_with_commit() { .map(OutboxClaimRef::from_message) .collect::, _>>() .unwrap(); - let result = worker.process_batch(&mut claimed).unwrap(); + let result = worker.process_batch(&mut claimed).await.unwrap(); processed += result.completed + result.released + result.failed; complete_published_outbox(&repo, &claimed, &claims).await; } @@ -581,7 +569,7 @@ async fn outbox_worker_process_next_with_commit() { let message = load_outbox_message(&repo, id).await; assert!(message.is_published()); } - assert_eq!(repo.outbox_store().pending_async().await.unwrap().len(), 0); + assert_eq!(repo.outbox_store().pending().await.unwrap().len(), 0); let lines = buffer.lock().unwrap(); assert_eq!(lines.len(), 3); diff --git a/tests/transport_conformance/mod.rs b/tests/transport_conformance/mod.rs index e1da7fb..b8869ca 100644 --- a/tests/transport_conformance/mod.rs +++ b/tests/transport_conformance/mod.rs @@ -6,7 +6,7 @@ //! //! Concrete adapters reuse the pieces here: a real *source* adapter can be //! exercised against [`FakePublisher`], a real *publisher* adapter against -//! [`FakeSource`], and any [`AsyncOutboxStore`] against the dispatcher contract. +//! [`FakeSource`], and any [`OutboxStore`] against the dispatcher contract. //! Other test targets include this module with //! `#[path = "../transport_conformance/mod.rs"] mod conformance;`. #![allow(dead_code)] @@ -351,7 +351,7 @@ async fn store_outbox(repo: &HashMapRepository, id: &str) -> String { } async fn outbox_status(repo: &HashMapRepository, id: &str) -> Option { - use distributed::AsyncOutboxStore; + use distributed::OutboxStore; let store = repo.outbox_store(); for status in [ OutboxMessageStatus::Pending, @@ -362,7 +362,7 @@ async fn outbox_status(repo: &HashMapRepository, id: &str) -> Option Date: Sat, 13 Jun 2026 16:02:32 -0500 Subject: [PATCH 3/4] docs: align async-only API references --- README.md | 3 ++- docs/async-transports.md | 9 ++++----- docs/postgres-event-store.md | 19 +++++-------------- docs/repositories.md | 5 +---- docs/research-and-roadmap.md | 4 ++-- 5 files changed, 14 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 4733eea..344b03a 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,8 @@ It is built with stateless vertical and horizontal scaling in cloud-native envir > **The framework is async-only.** Aggregates, repositories, handlers, the commit > path, and the service bus are all `async`. There is no synchronous repository or > bus API. Persistence adapters (Postgres, SQLite) and transports (NATS, RabbitMQ, -> Kafka, Knative) implement the async traits directly with no blocking shims. +> Kafka, Knative) expose async traits directly; broker/client blocking primitives, +> where unavoidable, stay internal to async transport methods. ## At a Glance diff --git a/docs/async-transports.md b/docs/async-transports.md index 10b8bea..1a64bfa 100644 --- a/docs/async-transports.md +++ b/docs/async-transports.md @@ -1,8 +1,8 @@ # Async Microservice Transports -Distributed (published from the `distributed` crate) keeps the synchronous -in-memory bus intact and adds an async transport layer under -`bus`. The design line is: +Distributed (published from the `distributed` crate) exposes an async-only +transport layer under `bus`. The in-memory bus is the dev/test implementation of +the same async contracts. The design line is: - **`microsvc`** owns handler registration, guards, typed input decoding, dispatch, and handler metadata; @@ -202,6 +202,5 @@ outbox dispatcher, the conformance harness, the Postgres / NATS / RabbitMQ / Kafka adapters, the Knative ingress, and the **bus facade** (`Bus` + `BusConsumer` with `InMemoryBus` / `NatsBus` / `PostgresBus` / `RabbitBus` / `KafkaBus` / `KnativeBus`, each with real-broker competing-vs-fan-out tests). -Still open: migrating the in-repo examples to showcase these APIs and removing the -legacy synchronous bus paths (a breaking change). See +Still open: migrating the in-repo examples to showcase these APIs. See `tasks/transport-docs-examples-cutover`. diff --git a/docs/postgres-event-store.md b/docs/postgres-event-store.md index 665fe70..a889b68 100644 --- a/docs/postgres-event-store.md +++ b/docs/postgres-event-store.md @@ -323,19 +323,10 @@ legacy import path. Newly written Postgres rows must always populate ## Async Posture -Do not hide production Postgres access behind the current synchronous repository -traits by blocking inside Tokio worker threads. The production Postgres -repository should be async-first, preferably backed by `sqlx`. - -Because the current public traits are synchronous, the Postgres implementation is -deferred behind async trait work. Acceptable interim choices: - -- keep `HashMapRepository` as the synchronous behavioral reference; -- design async traits that mirror `Get`, `Commit`, `TransactionalCommit`, - read-model, snapshot, and outbox operations; -- add an explicitly named blocking adapter only for tests or transitional use. - -The blocking adapter must not be the final production contract. +Postgres access is async-first through the public repository, read-model, +snapshot, inbox, lock, and outbox traits. Do not add production Postgres access +behind blocking adapters or synchronous repository shims; SQL I/O belongs in +async `sqlx` paths. ## TypeORM Lineage @@ -398,4 +389,4 @@ Postgres-specific tests should add: - Implementing the actual Postgres repository. - Designing the final read-model table layout. - Designing the outbox message table in detail. -- Replacing synchronous traits in this document. +- Changing the established async repository trait surface. diff --git a/docs/repositories.md b/docs/repositories.md index 0cde9e7..19cbbcd 100644 --- a/docs/repositories.md +++ b/docs/repositories.md @@ -24,13 +24,10 @@ aggregate_type = "...")`, `aggregate!(..., aggregate_type = "..." { ... })`, or identity. The record envelope carries stream identity, covered event version, snapshot payload type/version, payload codec metadata, cache metadata, and timestamp. -- `AsyncOutboxStore` exposes async claim/update operations for durable outbox +- `OutboxStore` exposes async claim/update operations for durable outbox table stores. Aggregate repositories commit outbox rows transactionally, but workers do not hydrate outbox messages through aggregate repositories. -`AsyncOutboxStore` keeps its prefix because `OutboxStore` remains the synchronous -worker-facing API for in-memory/local publisher workflows. - ## In-Memory Reference `HashMapRepository`, `InMemoryReadModelStore`, and `InMemorySnapshotStore` diff --git a/docs/research-and-roadmap.md b/docs/research-and-roadmap.md index 5be0dc2..97b2c27 100644 --- a/docs/research-and-roadmap.md +++ b/docs/research-and-roadmap.md @@ -9,12 +9,12 @@ Based on a review of the Rust ES ecosystem (cqrs-es, disintegrate, esrs, eventua ### Core Improvements #### Async traits -All competing frameworks are async-first. The original traits (`Repository`, `Commit`, `Get`, etc.) are synchronous, which blocks direct integration with: +All competing frameworks are async-first. The current public persistence and transport traits are async-first, which supports direct integration with: - Async databases (sqlx, sea-orm) - Async message brokers (rdkafka, lapin) - Async web frameworks (axum handlers) -Foundation status: stream-aware repository/read-model/snapshot traits now form the async-only persistence surface. See [Repository Boundary](repositories.md). The SQL backends implement those traits directly instead of wrapping database I/O behind synchronous repository traits. +Foundation status: stream-aware repository/read-model/snapshot/outbox traits now form the async-only persistence surface. See [Repository Boundary](repositories.md). The SQL backends implement those traits directly instead of wrapping database I/O behind synchronous repository traits. ### Later From 3d86ed0ba09b3d40372b81d65e004524a713fe00 Mon Sep 17 00:00:00 2001 From: Patrick Lee Scott Date: Sat, 13 Jun 2026 16:16:48 -0500 Subject: [PATCH 4/4] fix: address outbox review comments --- src/outbox_worker/mod.rs | 13 ++++--------- src/outbox_worker/worker.rs | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 9 deletions(-) diff --git a/src/outbox_worker/mod.rs b/src/outbox_worker/mod.rs index 6aecc73..a44d24f 100644 --- a/src/outbox_worker/mod.rs +++ b/src/outbox_worker/mod.rs @@ -19,17 +19,12 @@ //! ## Example //! //! ```ignore -//! use distributed::{OutboxStore, ClaimOutboxMessages, OutboxClaimRef}; +//! use distributed::OutboxDispatcher; //! use std::time::Duration; //! -//! let worker_id = "worker-1"; -//! let messages = outbox -//! .claim(ClaimOutboxMessages::new(worker_id, 10, Duration::from_secs(60))) -//! .await?; -//! for msg in messages { -//! let claim = OutboxClaimRef::from_message(&msg)?; -//! outbox.complete(&claim).await?; -//! } +//! let dispatcher = +//! OutboxDispatcher::new(outbox, publisher, "worker-1", Duration::from_secs(60), 3); +//! let outcome = dispatcher.dispatch_batch(10).await?; //! ``` mod bus_publisher; diff --git a/src/outbox_worker/worker.rs b/src/outbox_worker/worker.rs index c171a43..c9e2b81 100644 --- a/src/outbox_worker/worker.rs +++ b/src/outbox_worker/worker.rs @@ -183,6 +183,23 @@ impl OutboxWorker

{ mod tests { use super::*; use crate::LogPublisher; + use std::collections::HashMap; + use std::future::Future; + + struct FailingPublisher; + + impl OutboxPublisher for FailingPublisher { + type Error = &'static str; + + fn publish<'a>( + &'a mut self, + _event_type: &'a str, + _payload: &'a [u8], + _metadata: &'a HashMap, + ) -> impl Future> + 'a { + async { Err("publish failed") } + } + } #[test] fn worker_builder() { @@ -253,4 +270,22 @@ mod tests { assert_eq!(result.claimed, 0); assert_eq!(result.completed, 1); } + + #[tokio::test] + async fn process_message_fails_when_claimed_attempt_reaches_max_attempts() { + let mut message = OutboxMessage::create("msg-1", "Event", b"{}".to_vec()).unwrap(); + let mut worker = OutboxWorker::new(FailingPublisher).with_max_attempts(2); + + let first = worker.process_message(&mut message).await.unwrap(); + assert!(first.released); + assert!(!first.failed); + assert!(message.is_pending()); + assert_eq!(message.attempts, 1); + + let second = worker.process_message(&mut message).await.unwrap(); + assert!(!second.released); + assert!(second.failed); + assert!(message.is_failed()); + assert_eq!(message.attempts, 2); + } }