From fc9884419f91624129e38bf11f0f5f7a21dc7ccd Mon Sep 17 00:00:00 2001 From: Xiaoyang Han Date: Thu, 8 Oct 2026 08:40:04 +0800 Subject: [PATCH] Bound v3 chunk-map ownership and cancel stalled backend waits at the actual deadline Bound persisted v3 chunk maps and exact receipt generations with primary database owners, capacity admission, resumable collection and final transport-clone retention. Pending cold and warm backend opens, input polls and receipt creates now terminate under the actual remaining owner deadline; empty fragments grant no renewal. Add actual HTTP/storage/database test sources for expiry, withheld final bytes, refund and safe replay. Owned nightly format, canonical candidate whitespace and SQL lexical checks pass; native build/tests, database execution and commit-update performance measurements remain unrun. Truthful backend capabilities are a separately reviewed source-only follow-up dependency. --- .../snapshot_chunk_map_retention_tests.rs | 1405 +++++++++++++++++ src/api/router/snapshot_content.rs | 180 ++- src/api/router/snapshot_content_tests.rs | 104 ++ .../snapshot_persisted_chunk_map_tests.rs | 4 +- src/api/router/snapshot_raw_blob.rs | 44 +- src/api/router/snapshot_raw_blob_tests.rs | 2 +- src/ceres/snapshot/chunks.rs | 2 + src/ceres/snapshot/chunks_stream.rs | 40 +- src/ceres/snapshot/chunks_stream_tests.rs | 6 +- src/ceres/snapshot/content_budget.rs | 4 - ...008_000300_add_mst2_chunk_map_retention.rs | 26 + .../m20261008_000300_chunk_map_retention.sql | 262 +++ src/jupiter/migration/mod.rs | 22 +- src/jupiter/storage/native_chunk_map.rs | 285 ++-- .../storage/native_chunk_map/retention.rs | 1220 ++++++++++++++ src/jupiter/storage/object_storage.rs | 50 + src/orbit/adapter/object.rs | 105 ++ src/orbit_api/error.rs | 6 + src/orbit_api/object_storage.rs | 49 + src/server/http_server.rs | 103 +- 20 files changed, 3753 insertions(+), 166 deletions(-) create mode 100644 src/api/router/snapshot_chunk_map_retention_tests.rs create mode 100644 src/jupiter/migration/m20261008_000300_add_mst2_chunk_map_retention.rs create mode 100644 src/jupiter/migration/m20261008_000300_chunk_map_retention.sql create mode 100644 src/jupiter/storage/native_chunk_map/retention.rs diff --git a/src/api/router/snapshot_chunk_map_retention_tests.rs b/src/api/router/snapshot_chunk_map_retention_tests.rs new file mode 100644 index 00000000..587e4196 --- /dev/null +++ b/src/api/router/snapshot_chunk_map_retention_tests.rs @@ -0,0 +1,1405 @@ +use sea_orm::{DatabaseConnection, DbBackend, Statement}; +use tokio::{sync::Notify, time::timeout}; + +use super::*; +use crate::{ + ceres::snapshot::{ + chunks::{ChunkMapSource, VerifiedSourceChunkMap}, + error::SnapshotErrorCode, + }, + jupiter::storage::{ + native_chunk_map::PostgresChunkMapRepository, object_storage::mock_object_storage, + }, +}; + +fn statement(sql: &str, values: [sea_orm::Value; N]) -> Statement { + Statement::from_sql_and_values(DbBackend::Postgres, sql, values) +} + +fn router(state: &MonoApiServiceState) -> Router { + Router::new().nest("/api/v2", routers(state.clone()).with_state(state.clone())) +} + +fn budgeted_chunks_app( + fixture: &Fixture, + response_budget: &Arc, + scratch_budget: &Arc, +) -> Router { + use axum::{ + extract::{Path as AxumPath, State}, + routing::post, + }; + + use crate::api::router::snapshot_router::{ + content, request::Mst2Bytes, snapshot_auth_middleware, + }; + + let response_budget = response_budget.clone(); + let scratch_budget = scratch_budget.clone(); + Router::new() + .route( + "/api/v2/snapshots/{snapshot_id}/chunks", + post( + move |state: State, + path: AxumPath, + body: Mst2Bytes| { + content::chunks_with_budgets( + state, + path, + body, + content::ChunksBudgets { + response: response_budget.clone(), + scratch: scratch_budget.clone(), + }, + ) + }, + ), + ) + .route_layer(axum::middleware::from_fn_with_state( + fixture.state.clone(), + snapshot_auth_middleware, + )) + .with_state(fixture.state.clone()) +} + +async fn count(db: &DatabaseConnection, table: &str) -> i64 { + db.query_one_raw(statement( + &format!("SELECT count(*) AS count FROM {table}"), + [], + )) + .await + .unwrap() + .unwrap() + .try_get("", "count") + .unwrap() +} + +async fn wait_count(db: &DatabaseConnection, table: &str, expected: i64) { + timeout(Duration::from_secs(10), async { + loop { + if count(db, table).await == expected { + return; + } + tokio::time::sleep(Duration::from_millis(5)).await; + } + }) + .await + .expect("actual durable owner release did not complete"); +} + +async fn age_unowned_candidates(db: &DatabaseConnection) { + db.execute_unprepared("UPDATE mst2_chunk_receipt_generation SET last_progress=pg_catalog.clock_timestamp()-interval '2 hours' WHERE state='LIVE'; UPDATE mst2_chunk_map_lifetime SET last_used=pg_catalog.clock_timestamp()-interval '2 hours' WHERE state='LIVE'").await.unwrap(); +} + +async fn oid_for(fixture: &Fixture, path: &str) -> String { + let handler = MonoApiService::from(&fixture.state); + let main = fixture + .state + .storage + .mono_storage() + .get_main_ref("/project") + .await + .unwrap() + .unwrap(); + let tree = handler.get_tree_by_hash(&main.ref_tree_hash).await.unwrap(); + match resolve_abs_metadata(&handler, &tree, path).await.unwrap() { + MetadataWalkOutcome::FoundFile { oid, .. } => oid, + other => panic!("retention fixture did not resolve: {other:?}"), + } +} + +#[tokio::test] +async fn shared_map_survives_other_source_retirement_while_actual_reader_is_live() { + let fixture = + Fixture::new_in_metadata_family(false, 0, &[("other".to_string(), vec![19; 4096])], true) + .await; + let original = fixture.map("/file").await; + let other = oid_for(&fixture, "/other").await; + let db = fixture.state.storage.mono_storage().get_connection(); + // This G fixture isolates independent source receipts without changing a + // Q-certified file tuple. Two exact facts consume the same actual raw + // bytes independently; map sharing never admits the second source. + db.execute_raw(statement( + "UPDATE mst2_verified_object SET size=$1,raw_sha256=$2 WHERE git_oid=$3", + [ + (fixture.raw.len() as i64).into(), + fixture.digest.to_vec().into(), + other.clone().into(), + ], + )) + .await + .unwrap(); + *fixture.counts.object_size_override.lock().unwrap() = Some(fixture.raw.len() as i64); + *fixture.counts.object_fault.lock().unwrap() = Some(bounded_objects::StreamFault { + oid: other.clone(), + kind: bounded_objects::FaultKind::Parts( + fixture + .raw + .chunks(CHUNK_SIZE as usize) + .map(Bytes::copy_from_slice) + .collect(), + ), + }); + fixture.counts.reset(); + assert_eq!(fixture.map("/other").await["map"], original["map"]); + fixture.counts.assert(1, fixture.raw.len()); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 1); + wait_count(db, "mst2_chunk_reader", 0).await; + assert_eq!(count(db, "mst2_chunk_map_source").await, 2); + assert_eq!(count(db, "mst2_chunk_map").await, 1); + assert_eq!(count(db, "mst2_chunk_map_lifetime").await, 1); + let fact = fixture + .state + .storage + .mono_storage() + .get_verified_blobs(vec![other.clone()]) + .await + .unwrap() + .remove(&other) + .unwrap(); + let source = ChunkMapSource::from_fact(fact, &other).unwrap(); + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + let objects = &fixture.state.storage.git_service.obj_storage; + let reader = repository.read(&source, objects).await.unwrap().unwrap(); + age_unowned_candidates(db).await; + repository.maintain(objects, 64).await.unwrap(); + assert_eq!(count(db, "mst2_chunk_map_source").await, 1); + assert_eq!(count(db, "mst2_chunk_map").await, 1); + assert_eq!(count(db, "mst2_chunk_map_leaf").await, 1); + assert_eq!(count(db, "mst2_chunk_map_node").await, 1); + assert_eq!(fixture.counts.receipt_deletes.load(Ordering::SeqCst), 1); + repository + .selected_page(&reader, 0) + .await + .unwrap() + .verify_chunk(&reader.map, 0, &fixture.raw[..CHUNK_SIZE as usize]) + .unwrap(); + drop(reader); + wait_count(db, "mst2_chunk_reader", 0).await; + age_unowned_candidates(db).await; + repository.maintain(objects, 64).await.unwrap(); + assert_eq!(count(db, "mst2_chunk_map_source").await, 0); + assert_eq!(count(db, "mst2_chunk_map").await, 0); + assert_eq!(fixture.counts.receipt_deletes.load(Ordering::SeqCst), 2); +} + +#[tokio::test] +async fn pending_receipt_is_replayed_before_another_aged_live_victim() { + let fixture = Fixture::new_with_pg_config_directories_and_objects( + false, + 0, + &[("other".to_string(), vec![19; 4096])], + ) + .await; + fixture.map("/file").await; + let other = fixture.map("/other").await; + let db = fixture.state.storage.mono_storage().get_connection(); + wait_count(db, "mst2_chunk_reader", 0).await; + age_unowned_candidates(db).await; + db.execute_raw(statement( + "UPDATE mst2_chunk_receipt_generation g SET state='DELETING' WHERE EXISTS(SELECT 1 FROM mst2_chunk_map_source s WHERE s.receipt_key=g.receipt_key AND s.git_oid=$1)", + [fixture.oid.clone().into()], + )) + .await + .unwrap(); + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + repository + .maintain(&fixture.state.storage.git_service.obj_storage, 1) + .await + .unwrap(); + assert_eq!(count(db, "mst2_chunk_map_source").await, 1); + assert_eq!(count(db, "mst2_chunk_map").await, 1); + assert_eq!(fixture.counts.receipt_deletes.load(Ordering::SeqCst), 1); + fixture.counts.reset(); + assert_eq!(fixture.map("/other").await["map"], other["map"]); + fixture.counts.assert(0, 0); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn actual_http_replays_a_retiring_source_row_before_rebuilding_its_next_generation() { + let fixture = Fixture::new().await; + let original = fixture.map("/file").await; + let db = fixture.state.storage.mono_storage().get_connection(); + wait_count(db, "mst2_chunk_reader", 0).await; + let old_key: String = db + .query_one_raw(statement( + "SELECT receipt_key FROM mst2_chunk_map_source", + [], + )) + .await + .unwrap() + .unwrap() + .try_get("", "receipt_key") + .unwrap(); + db.execute_unprepared( + "UPDATE mst2_chunk_receipt_generation SET state='DELETING' WHERE state='LIVE'", + ) + .await + .unwrap(); + // The source row is deliberately still present; inline replay cannot + // depend on the source-index delete having already finished. + assert_eq!(count(db, "mst2_chunk_map_source").await, 1); + fixture.counts.reset(); + assert_eq!(fixture.map("/file").await["map"], original["map"]); + fixture.counts.assert(1, fixture.raw.len()); + assert_eq!(fixture.counts.receipt_deletes.load(Ordering::SeqCst), 1); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 1); + let new_key: String = db + .query_one_raw(statement( + "SELECT receipt_key FROM mst2_chunk_map_source", + [], + )) + .await + .unwrap() + .unwrap() + .try_get("", "receipt_key") + .unwrap(); + assert_ne!(old_key, new_key); + assert_eq!(count(db, "mst2_chunk_map").await, 1); + fixture.counts.reset(); + fixture.map("/alias").await; + fixture.counts.assert(0, 0); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 0); +} + +#[tokio::test] +async fn actual_warm_body_owner_cancels_never_returning_open_and_next_without_backend_release() { + use crate::ceres::snapshot::content_budget::{MemoryBudget, RANGE_WORK_BYTES}; + + for raw_route in [true, false] { + for stage in 0..3 { + let held_open = stage == 0; + let complete_without_eof = stage == 2; + let fixture = Fixture::new().await; + let map = fixture.map("/file").await; + let db = fixture.state.storage.mono_storage().get_connection(); + wait_count(db, "mst2_chunk_reader", 0).await; + fixture.counts.reset(); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let drops = Arc::new(AtomicUsize::new(0)); + let tail_polls = Arc::new(AtomicUsize::new(0)); + if held_open { + let holds = Some((entered.clone(), release.clone(), drops.clone())); + if raw_route { + *fixture.counts.whole_open_holds.lock().unwrap() = holds; + } else { + *fixture.counts.range_open_holds.lock().unwrap() = holds; + } + } else { + *fixture.counts.object_fault.lock().unwrap() = Some(bounded_objects::StreamFault { + oid: fixture.oid.clone(), + kind: bounded_objects::FaultKind::HeldFragment { + prefix: if complete_without_eof { + Bytes::copy_from_slice(if raw_route { + &fixture.raw + } else { + &fixture.raw[..CHUNK_SIZE as usize] + }) + } else { + Bytes::new() + }, + fragment: None, + entered: entered.clone(), + release: release.clone(), + drops: drops.clone(), + tail_polls: tail_polls.clone(), + }, + }); + } + let response_bytes = if raw_route { + CHUNK_SIZE as usize + } else { + CHUNK_SIZE as usize + 2048 + }; + let response_budget = MemoryBudget::new(response_bytes); + let scratch_budget = MemoryBudget::new(RANGE_WORK_BYTES); + let app = if raw_route { + super::raw_blob::budgeted_app(&fixture, &response_budget, &scratch_budget) + } else { + budgeted_chunks_app(&fixture, &response_budget, &scratch_budget) + }; + let request = if raw_route { + fixture.request("GET", "blob?path=/file", Body::empty()) + } else { + fixture.request( + "POST", + "chunks", + Body::from( + fixture + .chunk_body("/file", map["map"]["map_id"].as_str().unwrap(), "0") + .to_string(), + ), + ) + }; + let expected_raw = fixture.raw.clone(); + let mut task = tokio::spawn(async move { + let response = app.oneshot(request).await.unwrap(); + if raw_route && !held_open { + assert_eq!(response.status(), 200); + let mut body = response.into_body().into_data_stream(); + if complete_without_eof { + let first = body.next().await.unwrap().unwrap(); + assert_eq!(first.as_ref(), &expected_raw[..CHUNK_SIZE as usize]); + drop(first); + } + assert!(body.next().await.unwrap().is_err()); + assert!(body.next().await.is_none()); + } else { + error(response, 410, "LEASE_EXPIRED", false).await; + } + }); + timeout(Duration::from_secs(10), entered.notified()) + .await + .unwrap(); + assert_eq!(count(db, "mst2_chunk_reader").await, 1); + assert_eq!(scratch_budget.used(), RANGE_WORK_BYTES); + assert_eq!( + response_budget.used(), + if raw_route && complete_without_eof { + fixture.raw.len() - CHUNK_SIZE as usize + } else { + response_bytes + } + ); + db.execute_unprepared("UPDATE mst2_chunk_reader SET deadline=pg_catalog.clock_timestamp()+interval '20 milliseconds'").await.unwrap(); + // Never release the backend and never cancel the caller. The + // request must observe actual expiry during its pending await. + let finished = timeout(Duration::from_secs(12), &mut task).await; + if finished.is_err() { + task.abort(); + let _ = task.await; + panic!("actual body owner did not cancel its permanently held backend"); + } + finished.unwrap().unwrap(); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert_eq!(tail_polls.load(Ordering::SeqCst), 0); + let consumed = if complete_without_eof { + if raw_route { + fixture.raw.len() + } else { + CHUNK_SIZE as usize + } + } else { + 0 + }; + assert_eq!(fixture.counts.bytes.load(Ordering::SeqCst), consumed); + assert_eq!(scratch_budget.used(), 0); + assert_eq!(response_budget.used(), 0); + if raw_route { + fixture.counts.assert(1, consumed); + } else { + assert_eq!(fixture.counts.whole.load(Ordering::SeqCst), 0); + assert_eq!(fixture.counts.range.load(Ordering::SeqCst), 1); + } + wait_count(db, "mst2_chunk_reader", 0).await; + assert_eq!(count(db, "mst2_chunk_map_source").await, 1); + drop(release); + } + } +} + +#[tokio::test] +async fn actual_cold_builder_uses_remaining_database_deadline_for_open_and_next_and_refunds_credit() +{ + use crate::ceres::snapshot::content_budget::MemoryBudget; + + for stage in 0..3 { + let held_open = stage == 0; + let complete_without_eof = stage == 2; + let fixture = Fixture::new().await; + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + let source = ChunkMapSource::from_fact(fixture.fact().await, &fixture.oid).unwrap(); + let admission = repository + .admit_install(&source, &fixture.state.storage.git_service.obj_storage) + .await + .unwrap(); + let db = fixture.state.storage.mono_storage().get_connection(); + db.execute_unprepared("UPDATE mst2_chunk_receipt_generation SET deadline=pg_catalog.clock_timestamp()+interval '3 seconds' WHERE state='RESERVED'").await.unwrap(); + admission.test_check_next_owner_operation().await; + fixture.counts.reset(); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let drops = Arc::new(AtomicUsize::new(0)); + let tail_polls = Arc::new(AtomicUsize::new(0)); + if held_open { + *fixture.counts.whole_open_holds.lock().unwrap() = + Some((entered.clone(), release.clone(), drops.clone())); + } else { + *fixture.counts.object_fault.lock().unwrap() = Some(bounded_objects::StreamFault { + oid: fixture.oid.clone(), + kind: bounded_objects::FaultKind::HeldFragment { + prefix: if complete_without_eof { + Bytes::copy_from_slice(&fixture.raw) + } else { + Bytes::new() + }, + fragment: None, + entered: entered.clone(), + release: release.clone(), + drops: drops.clone(), + tail_polls: tail_polls.clone(), + }, + }); + } + let budget = MemoryBudget::new(512 * 1024 * 1024); + let task_budget = budget.clone(); + let handler = MonoApiService::from(&fixture.state); + let mut task = tokio::spawn(async move { + VerifiedSourceChunkMap::verify(&handler, source, &task_budget, &admission).await + }); + timeout(Duration::from_secs(2), entered.notified()) + .await + .unwrap(); + assert!(budget.used() > 0); + let result = timeout(Duration::from_secs(5), &mut task).await; + if result.is_err() { + task.abort(); + let _ = task.await; + panic!("cold source ignored its actual remaining database deadline"); + } + let result = result.unwrap().unwrap(); + let error = match result { + Ok(_) => panic!("incomplete cold source produced trust"), + Err(error) => error, + }; + assert_eq!(error.code, SnapshotErrorCode::LeaseExpired); + assert_eq!(budget.used(), 0); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert_eq!(tail_polls.load(Ordering::SeqCst), 0); + fixture.counts.assert( + 1, + if complete_without_eof { + fixture.raw.len() + } else { + 0 + }, + ); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 0); + assert_eq!(count(db, "mst2_chunk_map").await, 0); + assert_eq!(count(db, "mst2_chunk_map_source").await, 0); + drop(release); + } +} + +#[tokio::test] +async fn actual_receipt_create_and_read_waits_cancel_without_backend_release_or_source_publication() +{ + use crate::ceres::snapshot::content_budget::MemoryBudget; + + for creating in [true, false] { + let fixture = Fixture::new().await; + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + let source = ChunkMapSource::from_fact(fixture.fact().await, &fixture.oid).unwrap(); + let objects = fixture.state.storage.git_service.obj_storage.clone(); + let admission = repository.admit_install(&source, &objects).await.unwrap(); + let budget = MemoryBudget::new(512 * 1024 * 1024); + let handler = MonoApiService::from(&fixture.state); + let verified = VerifiedSourceChunkMap::verify(&handler, source, &budget, &admission) + .await + .unwrap(); + assert!(budget.used() > 0); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let drops = if creating { + *fixture.counts.receipt_write_holds.lock().unwrap() = + Some((entered.clone(), release.clone())); + fixture.counts.receipt_write_wait_drops.clone() + } else { + let drops = Arc::new(AtomicUsize::new(0)); + *fixture.counts.receipt_open_holds.lock().unwrap() = + Some((entered.clone(), release.clone(), drops.clone())); + drops + }; + let mut task = + tokio::spawn(async move { repository.install(verified, &objects, &admission).await }); + timeout(Duration::from_secs(10), entered.notified()) + .await + .unwrap(); + let db = fixture.state.storage.mono_storage().get_connection(); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 1); + if creating { + db.execute_unprepared("UPDATE mst2_chunk_receipt_generation SET deadline=pg_catalog.clock_timestamp()+interval '20 milliseconds' WHERE state='CREATING'").await.unwrap(); + } + // Atomic create uses the actual owner expiry; receipt input also keeps + // its stricter five-second cap. Neither wait needs backend release. + let finished = timeout( + Duration::from_secs(if creating { 12 } else { 7 }), + &mut task, + ) + .await; + if finished.is_err() { + task.abort(); + let _ = task.await; + panic!("receipt wait retained its install workspace indefinitely"); + } + let error = finished.unwrap().unwrap().unwrap_err(); + assert_eq!( + error.code, + if creating { + SnapshotErrorCode::LeaseExpired + } else { + SnapshotErrorCode::TemporaryUnavailable + } + ); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert_eq!(budget.used(), 0); + fixture.counts.assert(1, fixture.raw.len()); + assert_eq!( + fixture.counts.receipt_reads.load(Ordering::SeqCst), + usize::from(!creating) + ); + assert_eq!(count(db, "mst2_chunk_reader").await, 0); + assert_eq!(count(db, "mst2_chunk_map_source").await, 0); + assert_eq!(count(db, "mst2_chunk_map").await, 0); + *fixture.counts.receipt_open_holds.lock().unwrap() = None; + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + repository + .maintain(&fixture.state.storage.git_service.obj_storage, 64) + .await + .unwrap(); + assert_eq!(fixture.counts.receipt_deletes.load(Ordering::SeqCst), 1); + drop(release); + } +} + +#[tokio::test] +async fn held_raw_and_exact_range_do_not_resume_after_actual_reader_expiry() { + for raw_route in [true, false] { + let fixture = Fixture::new().await; + let map = fixture.map("/file").await; + let db = fixture.state.storage.mono_storage().get_connection(); + wait_count(db, "mst2_chunk_reader", 0).await; + fixture.counts.reset(); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let drops = Arc::new(AtomicUsize::new(0)); + let tail_polls = Arc::new(AtomicUsize::new(0)); + *fixture.counts.object_fault.lock().unwrap() = Some(bounded_objects::StreamFault { + oid: fixture.oid.clone(), + kind: bounded_objects::FaultKind::HeldFragment { + prefix: Bytes::new(), + fragment: Some(Bytes::new()), + entered: entered.clone(), + release: release.clone(), + tail_polls: tail_polls.clone(), + drops: drops.clone(), + }, + }); + let app = fixture.app.clone(); + let request = if raw_route { + fixture.request("GET", "blob?path=/file", Body::empty()) + } else { + fixture.request( + "POST", + "chunks", + Body::from( + fixture + .chunk_body("/file", map["map"]["map_id"].as_str().unwrap(), "0") + .to_string(), + ), + ) + }; + let mut task = tokio::spawn(async move { + let response = app.oneshot(request).await.unwrap(); + if raw_route { + assert_eq!(response.status(), 200); + let mut body = response.into_body().into_data_stream(); + assert!(body.next().await.unwrap().is_err()); + assert!(body.next().await.is_none()); + } else { + error(response, 410, "LEASE_EXPIRED", false).await; + } + }); + timeout(Duration::from_secs(10), entered.notified()) + .await + .unwrap(); + assert_eq!(count(db, "mst2_chunk_reader").await, 1); + db.execute_unprepared("UPDATE mst2_chunk_reader SET deadline=pg_catalog.clock_timestamp()+interval '20 milliseconds'").await.unwrap(); + // Exercise the real request-held reader without reaching into it: + // let the existing ten-second local check interval elapse too. + tokio::time::sleep(Duration::from_secs(11)).await; + release.notify_one(); + let completed = timeout(Duration::from_secs(5), &mut task).await; + if completed.is_err() { + task.abort(); + let _ = task.await; + panic!("expired reader continued into another held backend poll"); + } + completed.unwrap().unwrap(); + assert_eq!(tail_polls.load(Ordering::SeqCst), 0); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert_eq!(fixture.counts.bytes.load(Ordering::SeqCst), 0); + wait_count(db, "mst2_chunk_reader", 0).await; + assert_eq!(count(db, "mst2_chunk_map_source").await, 1); + } +} + +#[tokio::test] +async fn production_q_bootstrap_after_retention_preserves_catalog_and_reconstructed_warm_routes() { + use crate::jupiter::storage::qualified_metadata_family::{ + SnapshotMetadataFamily, provision_or_verify_rooted_qualified_family, + }; + + let fixture = Fixture::new_with_pg_config(true).await; + let db = fixture.state.storage.mono_storage().get_connection(); + let namespace = provision_or_verify_rooted_qualified_family(db) + .await + .unwrap(); + assert_eq!( + fixture + .state + .storage + .snapshot_metadata_family(&fixture.lease, true) + .await + .unwrap(), + Some(SnapshotMetadataFamily::Rooted) + ); + let applied: i64 = db + .query_one_raw(statement( + "SELECT count(*) AS count FROM seaql_migrations WHERE version='m20261008_000300_add_mst2_chunk_map_retention'", + [], + )) + .await + .unwrap() + .unwrap() + .try_get("", "count") + .unwrap(); + assert_eq!(applied, 1); + let original = fixture.map("/file").await; + fixture.counts.assert(1, fixture.raw.len()); + wait_count(db, "mst2_chunk_reader", 0).await; + + // This is the production connection path: it reapplies migration checks + // and verifies the complete existing Q authority catalog without a test + // exemption or a refreshed registration fingerprint. + let config = fixture.state.storage.config(); + let connection = crate::jupiter::storage::init::database_connection(&config.database) + .await + .unwrap(); + assert_eq!( + provision_or_verify_rooted_qualified_family(&connection) + .await + .unwrap(), + namespace + ); + let storage = crate::jupiter::storage::Storage::new_with_connection( + config, + Arc::new(connection), + fixture.state.storage.git_service.obj_storage.clone(), + ) + .await + .unwrap(); + let state = MonoApiServiceState { + storage, + ..fixture.state.clone() + }; + assert_eq!( + state + .storage + .snapshot_metadata_family(&fixture.lease, true) + .await + .unwrap(), + Some(SnapshotMetadataFamily::Rooted) + ); + let app = router(&state); + fixture.counts.reset(); + let warm = success_json( + app.clone() + .oneshot(fixture.request("GET", "chunk-map?path=/alias", Body::empty())) + .await + .unwrap(), + ) + .await; + assert_eq!(warm["map"], original["map"]); + fixture.counts.assert(0, 0); + assert_eq!(fixture.counts.receipt_reads.load(Ordering::SeqCst), 1); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 0); + + fixture.counts.reset(); + let response = app + .oneshot( + fixture.request( + "POST", + "chunks", + Body::from( + fixture + .chunk_body("/alias", original["map"]["map_id"].as_str().unwrap(), "0") + .to_string(), + ), + ), + ) + .await + .unwrap(); + assert_eq!(response.status(), 200); + let wire = to_bytes(response.into_body(), 2 * 1024 * 1024) + .await + .unwrap(); + let frames = parse_stream(&wire).unwrap(); + let [Frame::Chunk(chunk), Frame::End(end)] = frames.as_slice() else { + panic!("reconstructed rooted route must emit CHUNK and terminal END"); + }; + assert_eq!(chunk.chunk_bytes, &fixture.raw[..CHUNK_SIZE as usize]); + assert_eq!(chunk.file_content_id, fixture.digest); + assert_eq!(chunk.chunk_index, 0); + assert_eq!(end.request_item_count, 1); + assert_eq!(end.logical_bytes, u64::from(CHUNK_SIZE)); + assert_eq!(fixture.counts.whole.load(Ordering::SeqCst), 0); + assert_eq!(fixture.counts.range.load(Ordering::SeqCst), 1); + assert_eq!( + fixture.counts.bytes.load(Ordering::SeqCst), + CHUNK_SIZE as usize + ); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 0); + drop(wire); + wait_count(db, "mst2_chunk_reader", 0).await; + assert_eq!(count(db, "mst2_chunk_map_source").await, 1); + assert_eq!(count(db, "mst2_chunk_map").await, 1); +} + +#[tokio::test] +async fn one_connection_install_commits_bounded_stages_and_reconstructed_warm_reads() { + let fixture = Fixture::new_with_pg_config(true).await; + let mut config = fixture.state.storage.config().database.clone(); + config.max_connection = 1; + config.min_connection = 1; + let connection = crate::jupiter::storage::init::database_connection(&config) + .await + .unwrap(); + let repository = PostgresChunkMapRepository::new(connection.clone()) + .await + .unwrap(); + assert!( + fixture + .state + .storage + .native_chunk_maps + .set(repository) + .is_ok() + ); + connection.execute_unprepared("CREATE TABLE chunk_stage_audit(relation text NOT NULL, writer_xid bigint NOT NULL); CREATE FUNCTION chunk_stage_audit() RETURNS trigger LANGUAGE plpgsql AS $audit$ BEGIN INSERT INTO chunk_stage_audit VALUES(TG_TABLE_NAME,pg_catalog.txid_current()); RETURN NEW; END $audit$; CREATE TRIGGER chunk_stage_audit AFTER INSERT ON mst2_chunk_map FOR EACH ROW EXECUTE FUNCTION chunk_stage_audit(); CREATE TRIGGER chunk_stage_audit AFTER INSERT ON mst2_chunk_map_leaf FOR EACH ROW EXECUTE FUNCTION chunk_stage_audit(); CREATE TRIGGER chunk_stage_audit AFTER INSERT ON mst2_chunk_map_node FOR EACH ROW EXECUTE FUNCTION chunk_stage_audit(); CREATE TRIGGER chunk_stage_audit AFTER INSERT ON mst2_chunk_map_source FOR EACH ROW EXECUTE FUNCTION chunk_stage_audit()").await.unwrap(); + let map = timeout(Duration::from_secs(10), fixture.map("/file")) + .await + .expect("single-connection staging waited for its own held connection"); + fixture.counts.assert(1, fixture.raw.len()); + let transactions: i64 = connection + .query_one_raw(statement( + "SELECT count(DISTINCT writer_xid) AS count FROM chunk_stage_audit", + [], + )) + .await + .unwrap() + .unwrap() + .try_get("", "count") + .unwrap(); + assert!(transactions >= 4); + for table in [ + "mst2_chunk_map", + "mst2_chunk_map_leaf", + "mst2_chunk_map_node", + "mst2_chunk_map_source", + ] { + assert_eq!(count(&connection, table).await, 1); + } + fixture.counts.reset(); + let reconstructed = PostgresChunkMapRepository::new(connection.clone()) + .await + .unwrap(); + let source = ChunkMapSource::from_fact(fixture.fact().await, &fixture.oid).unwrap(); + let persisted = timeout( + Duration::from_secs(10), + reconstructed.read(&source, &fixture.state.storage.git_service.obj_storage), + ) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!( + format!("sha256:{}", hex_of(&persisted.map_id)), + map["map"]["map_id"] + ); + let page = reconstructed.selected_page(&persisted, 0).await.unwrap(); + page.verify_chunk(&persisted.map, 0, &fixture.raw[..CHUNK_SIZE as usize]) + .unwrap(); + fixture.counts.assert(0, 0); + assert_eq!(fixture.counts.receipt_reads.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn cached_inventory_is_coalesced_and_bound_to_the_actual_backend_arc() { + let fixture = Fixture::new().await; + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + let objects = &fixture.state.storage.git_service.obj_storage; + repository.maintain(objects, 8).await.unwrap(); + let first = fixture + .counts + .receipt_inventory_calls + .load(Ordering::SeqCst); + repository.maintain(objects, 8).await.unwrap(); + assert_eq!( + fixture + .counts + .receipt_inventory_calls + .load(Ordering::SeqCst), + first + ); + let counts = Arc::new(ReadCounts::default()); + counts + .receipt_retention_unsupported + .store(true, Ordering::SeqCst); + let other = MegaObjectStorageWrapper::new(Arc::new(CountingStorage { + inner: objects.clone(), + counts: counts.clone(), + })); + let error = repository.maintain(&other, 8).await.unwrap_err(); + assert_eq!(error.code, SnapshotErrorCode::TemporaryUnavailable); + assert_eq!(counts.receipt_inventory_calls.load(Ordering::SeqCst), 1); + fixture.counts.assert(0, 0); + assert_eq!( + count( + fixture.state.storage.mono_storage().get_connection(), + "mst2_chunk_receipt_generation" + ) + .await, + 0 + ); +} + +#[tokio::test] +async fn completion_during_real_inventory_keeps_backing_credit_after_pending_install_disappears() { + let mut fixture = Fixture::new().await; + let backing = mock_object_storage(); + backing + .inner + .put_stream( + &ObjectKey { + namespace: ObjectNamespace::Git, + key: fixture.oid.clone(), + }, + Box::pin(futures::stream::iter([Ok(Bytes::copy_from_slice( + &fixture.raw, + ))])), + ObjectMeta::default(), + ) + .await + .unwrap(); + fixture.state.storage.git_service = GitService { + obj_storage: MegaObjectStorageWrapper::new(Arc::new(CountingStorage { + inner: backing.clone(), + counts: fixture.counts.clone(), + })), + }; + fixture.app = router(&fixture.state); + let db = fixture.state.storage.mono_storage().get_connection(); + let create_entered = Arc::new(Notify::new()); + let create_release = Arc::new(Notify::new()); + *fixture.counts.receipt_late_create_holds.lock().unwrap() = + Some((create_entered.clone(), create_release.clone())); + let app = fixture.app.clone(); + let request = fixture.request("GET", "chunk-map?path=/file", Body::empty()); + let leader = tokio::spawn(async move { app.oneshot(request).await.unwrap() }); + timeout(Duration::from_secs(10), create_entered.notified()) + .await + .unwrap(); + *fixture.counts.receipt_late_create_holds.lock().unwrap() = None; + for index in 0..crate::orbit_api::object_storage::MAX_CHUNK_MAP_RECEIPTS - 1 { + backing + .inner + .put_metadata_atomic_create( + &ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: format!("{index:064x}"), + }, + Bytes::from_static(&[1; 129]), + ObjectMeta::default(), + ) + .await + .unwrap(); + } + // Synthetic occupants test physical capacity; they confer no source trust. + let actual = &fixture.state.storage.git_service.obj_storage; + let inventory_entered = Arc::new(Notify::new()); + let inventory_release = Arc::new(Notify::new()); + let observer_counts = Arc::new(ReadCounts::default()); + *observer_counts.receipt_inventory_holds.lock().unwrap() = + Some((inventory_entered.clone(), inventory_release.clone())); + let observer = PostgresChunkMapRepository::new(db.clone()).await.unwrap(); + let observed = MegaObjectStorageWrapper::new(Arc::new(CountingStorage { + inner: actual.clone(), + counts: observer_counts, + })); + let capacity = tokio::spawn(async move { observer.test_new_install_capacity(&observed).await }); + timeout(Duration::from_secs(30), inventory_entered.notified()) + .await + .unwrap(); + create_release.notify_one(); + success_json( + timeout(Duration::from_secs(30), leader) + .await + .unwrap() + .unwrap(), + ) + .await; + assert_eq!(count(db, "mst2_chunk_map_source").await, 1); + let pending:i64=db.query_one_raw(statement("SELECT count(*) AS count FROM mst2_chunk_receipt_generation WHERE state IN ('RESERVED','CREATING')",[])).await.unwrap().unwrap().try_get("","count").unwrap(); + assert_eq!(pending, 0); + inventory_release.notify_one(); + let error = timeout(Duration::from_secs(10), capacity) + .await + .unwrap() + .unwrap() + .unwrap_err(); + assert_eq!(error.code, SnapshotErrorCode::LimitExceeded); + fixture.counts.assert(1, fixture.raw.len()); + assert_eq!( + actual + .inner + .chunk_map_receipt_inventory() + .await + .unwrap() + .objects + .len(), + crate::orbit_api::object_storage::MAX_CHUNK_MAP_RECEIPTS + ); +} + +#[tokio::test] +async fn json_chunk_and_raw_last_transport_clones_block_actual_collection() { + use crate::ceres::snapshot::content_budget::{MemoryBudget, RANGE_WORK_BYTES}; + + for route in 0..3 { + let fixture = Fixture::new().await; + let map = fixture.map("/file").await; + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + let db = fixture.state.storage.mono_storage().get_connection(); + wait_count(db, "mst2_chunk_reader", 0).await; + fixture.counts.reset(); + let response_budget = MemoryBudget::new(CHUNK_SIZE as usize + 2048); + let scratch_budget = MemoryBudget::new(RANGE_WORK_BYTES); + let response = match route { + 0 => { + fixture + .send("GET", "chunk-map?path=/file", Body::empty()) + .await + } + 1 => budgeted_chunks_app(&fixture, &response_budget, &scratch_budget) + .oneshot( + fixture.request( + "POST", + "chunks", + Body::from( + json!({"items":[{ + "path":"/file", "expected_digest":fixture.digest_string(), + "map_id":map["map"]["map_id"], "chunk_index":"0" + }],"encoding":"identity"}) + .to_string(), + ), + ), + ) + .await + .unwrap(), + _ => super::raw_blob::budgeted_app(&fixture, &response_budget, &scratch_budget) + .oneshot(fixture.request("GET", "blob?path=/file", Body::empty())) + .await + .unwrap(), + }; + assert_eq!(response.status(), 200); + let mut body = response.into_body().into_data_stream(); + let mut wire = Vec::new(); + let mut transport = Vec::new(); + while let Some(frame) = body.next().await { + let bytes = frame.unwrap(); + wire.extend_from_slice(&bytes); + transport.push(bytes.clone()); + } + drop(body); + if route == 2 { + assert_eq!(wire, fixture.raw); + } + if route == 1 { + let frames = parse_stream(&wire).unwrap(); + assert!(frames.iter().any(|frame| matches!(frame, Frame::End(_)))); + } + assert!(!transport.is_empty()); + let last = transport.pop().unwrap(); + let last_transport = last.clone(); + drop(last); + drop(transport); + assert_eq!(scratch_budget.used(), 0); + if route == 1 { + assert_eq!(response_budget.used(), CHUNK_SIZE as usize + 2048); + } else if route == 2 { + assert_eq!( + response_budget.used(), + fixture.raw.len() - CHUNK_SIZE as usize + ); + } + assert_eq!(count(db, "mst2_chunk_reader").await, 1); + age_unowned_candidates(db).await; + repository + .maintain(&fixture.state.storage.git_service.obj_storage, 64) + .await + .unwrap(); + assert_eq!(count(db, "mst2_chunk_map_source").await, 1); + assert_eq!(count(db, "mst2_chunk_map").await, 1); + assert_eq!(fixture.counts.receipt_deletes.load(Ordering::SeqCst), 0); + drop(last_transport); + assert_eq!(response_budget.used(), 0); + wait_count(db, "mst2_chunk_reader", 0).await; + repository + .maintain(&fixture.state.storage.git_service.obj_storage, 64) + .await + .unwrap(); + for table in [ + "mst2_chunk_map_source", + "mst2_chunk_map", + "mst2_chunk_map_leaf", + "mst2_chunk_map_node", + "mst2_chunk_map_lifetime", + ] { + assert_eq!(count(db, table).await, 0); + } + assert_eq!(fixture.counts.receipt_deletes.load(Ordering::SeqCst), 1); + assert_eq!(count(db, "mst2_chunk_receipt_generation").await, 1); + assert_eq!(count(db, "mst2_chunk_map_gc").await, 1); + } +} + +#[tokio::test] +async fn expired_reader_arc_cannot_resurrect_or_authenticate_a_page() { + let fixture = Fixture::new().await; + fixture.map("/file").await; + let db = fixture.state.storage.mono_storage().get_connection(); + wait_count(db, "mst2_chunk_reader", 0).await; + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + let source = ChunkMapSource::from_fact(fixture.fact().await, &fixture.oid).unwrap(); + let map = repository + .read(&source, &fixture.state.storage.git_service.obj_storage) + .await + .unwrap() + .unwrap(); + let old_generation: i64 = db + .query_one_raw(statement( + "SELECT generation FROM mst2_chunk_map_lifetime WHERE map_id=$1", + [map.map_id.to_vec().into()], + )) + .await + .unwrap() + .unwrap() + .try_get("", "generation") + .unwrap(); + db.execute_unprepared("UPDATE mst2_chunk_reader SET deadline=pg_catalog.clock_timestamp()+interval '20 milliseconds'").await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + map.test_check_next_owner_operation().await; + assert_eq!( + map.ensure_live().await.unwrap_err().code, + SnapshotErrorCode::LeaseExpired + ); + assert_eq!( + map.record_progress().await.unwrap_err().code, + SnapshotErrorCode::LeaseExpired + ); + assert_eq!( + repository.selected_page(&map, 0).await.err().unwrap().code, + SnapshotErrorCode::LeaseExpired + ); + assert!(db.execute_unprepared("UPDATE mst2_chunk_reader SET deadline=pg_catalog.clock_timestamp()+interval '59 seconds'").await.is_err()); + age_unowned_candidates(db).await; + repository + .maintain(&fixture.state.storage.git_service.obj_storage, 64) + .await + .unwrap(); + assert_eq!(count(db, "mst2_chunk_map").await, 0); + assert_eq!( + map.ensure_live().await.unwrap_err().code, + SnapshotErrorCode::LeaseExpired + ); + fixture.counts.reset(); + assert_eq!( + fixture.map("/file").await["map"]["file_size"], + fixture.raw.len().to_string() + ); + fixture.counts.assert(1, fixture.raw.len()); + let new_generation: i64 = db + .query_one_raw(statement( + "SELECT generation FROM mst2_chunk_map_lifetime WHERE map_id=$1", + [map.map_id.to_vec().into()], + )) + .await + .unwrap() + .unwrap() + .try_get("", "generation") + .unwrap(); + assert!(new_generation > old_generation); + assert!( + db.execute_raw(statement( + "UPDATE mst2_chunk_map_lifetime SET state='DELETING' WHERE map_id=$1", + [map.map_id.to_vec().into()] + )) + .await + .is_err() + ); + assert!( + db.execute_raw(statement( + "DELETE FROM mst2_chunk_map_leaf WHERE map_id=$1", + [map.map_id.to_vec().into()] + )) + .await + .is_err() + ); + fixture.counts.reset(); + fixture.map("/alias").await; + fixture.counts.assert(0, 0); + assert_eq!( + map.ensure_live().await.unwrap_err().code, + SnapshotErrorCode::LeaseExpired + ); +} + +#[tokio::test] +async fn late_cancelled_create_is_reserved_and_old_key_replay_preserves_new_generation() { + let fixture = Fixture::new().await; + let db = fixture.state.storage.mono_storage().get_connection(); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + *fixture.counts.receipt_late_create_holds.lock().unwrap() = + Some((entered.clone(), release.clone())); + let app = fixture.app.clone(); + let request = fixture.request("GET", "chunk-map?path=/file", Body::empty()); + let cancelled = tokio::spawn(async move { app.oneshot(request).await }); + timeout(Duration::from_secs(10), entered.notified()) + .await + .unwrap(); + let old_key: String = db + .query_one_raw(statement( + "SELECT receipt_key FROM mst2_chunk_receipt_generation", + [], + )) + .await + .unwrap() + .unwrap() + .try_get("", "receipt_key") + .unwrap(); + assert!( + !fixture + .state + .storage + .git_service + .obj_storage + .inner + .exists(&ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: old_key.clone() + }) + .await + .unwrap() + ); + cancelled.abort(); + assert!(cancelled.await.err().unwrap().is_cancelled()); + *fixture.counts.receipt_late_create_holds.lock().unwrap() = None; + let current = fixture.map("/file").await; + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + wait_count(db, "mst2_chunk_reader", 0).await; + let history = db + .query_one_raw(statement( + "SELECT state,create_completed FROM mst2_chunk_receipt_generation WHERE receipt_key=$1", + [old_key.clone().into()], + )) + .await + .unwrap() + .unwrap(); + assert_eq!(history.try_get::("", "state").unwrap(), "APPLIED"); + assert!(!history.try_get::("", "create_completed").unwrap()); + let new_key: String = db + .query_one_raw(statement( + "SELECT receipt_key FROM mst2_chunk_map_source", + [], + )) + .await + .unwrap() + .unwrap() + .try_get("", "receipt_key") + .unwrap(); + assert_ne!(old_key, new_key); + release.notify_one(); + timeout(Duration::from_secs(10), async { + loop { + if fixture + .state + .storage + .git_service + .obj_storage + .inner + .exists(&ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: old_key.clone(), + }) + .await + .unwrap() + && fixture.counts.receipt_writes.load(Ordering::SeqCst) == 2 + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + repository + .maintain(&fixture.state.storage.git_service.obj_storage, 64) + .await + .unwrap(); + assert!( + !fixture + .state + .storage + .git_service + .obj_storage + .inner + .exists(&ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: old_key.clone() + }) + .await + .unwrap() + ); + assert!( + fixture + .state + .storage + .git_service + .obj_storage + .inner + .exists(&ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: new_key + }) + .await + .unwrap() + ); + fixture.counts.reset(); + assert_eq!(fixture.map("/file").await["map"], current["map"]); + fixture.counts.assert(0, 0); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 0); + let history = db + .query_one_raw(statement( + "SELECT create_completed FROM mst2_chunk_receipt_generation WHERE receipt_key=$1", + [old_key.into()], + )) + .await + .unwrap() + .unwrap(); + assert!(!history.try_get::("", "create_completed").unwrap()); +} + +#[tokio::test] +async fn actual_backing_quota_and_unsupported_capability_reject_before_source_open() { + for unsupported in [true, false] { + let fixture = Fixture::new().await; + let counts = Arc::new(ReadCounts::default()); + counts + .receipt_retention_unsupported + .store(unsupported, Ordering::SeqCst); + let backend = mock_object_storage(); + if !unsupported { + for index in 0..=crate::orbit_api::object_storage::MAX_CHUNK_MAP_RECEIPTS { + backend + .inner + .put_metadata_atomic_create( + &ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: format!("{index:064x}"), + }, + Bytes::from_static(&[1; 129]), + ObjectMeta::default(), + ) + .await + .unwrap(); + } + } + let mut state = fixture.state.clone(); + state.storage.git_service = GitService { + obj_storage: MegaObjectStorageWrapper::new(Arc::new(CountingStorage { + inner: backend, + counts: counts.clone(), + })), + }; + let response = router(&state) + .oneshot(fixture.request("GET", "chunk-map?path=/file", Body::empty())) + .await + .unwrap(); + if unsupported { + error(response, 503, "TEMPORARY_UNAVAILABLE", true).await; + } else { + error(response, 413, "LIMIT_EXCEEDED", false).await; + } + counts.assert(0, 0); + assert_eq!(counts.receipt_writes.load(Ordering::SeqCst), 0); + let db = fixture.state.storage.mono_storage().get_connection(); + assert_eq!(count(db, "mst2_chunk_receipt_generation").await, 0); + assert_eq!(count(db, "mst2_chunk_map").await, 0); + } +} + +#[tokio::test] +async fn empty_source_fragment_after_deadline_does_not_renew_install_or_publish() { + let fixture = Fixture::new().await; + let repository = fixture.state.storage.chunk_maps().await.unwrap(); + let source = ChunkMapSource::from_fact(fixture.fact().await, &fixture.oid).unwrap(); + let admission = Arc::new( + repository + .admit_install(&source, &fixture.state.storage.git_service.obj_storage) + .await + .unwrap(), + ); + let entered = Arc::new(Notify::new()); + let release = Arc::new(Notify::new()); + let drops = Arc::new(AtomicUsize::new(0)); + let tail_polls = Arc::new(AtomicUsize::new(0)); + *fixture.counts.object_fault.lock().unwrap() = Some(bounded_objects::StreamFault { + oid: fixture.oid.clone(), + kind: bounded_objects::FaultKind::HeldFragment { + prefix: Bytes::new(), + fragment: Some(Bytes::new()), + entered: entered.clone(), + release: release.clone(), + tail_polls: tail_polls.clone(), + drops: drops.clone(), + }, + }); + let handler = MonoApiService::from(&fixture.state); + let budget = crate::ceres::snapshot::content_budget::MemoryBudget::new(8 * 1024 * 1024); + let verifier_budget = budget.clone(); + let owner = admission.clone(); + let task = tokio::spawn(async move { + VerifiedSourceChunkMap::verify(&handler, source, &verifier_budget, &owner).await + }); + timeout(Duration::from_secs(10), entered.notified()) + .await + .unwrap(); + let db = fixture.state.storage.mono_storage().get_connection(); + db.execute_unprepared("UPDATE mst2_chunk_receipt_generation SET deadline=pg_catalog.clock_timestamp()+interval '20 milliseconds' WHERE state='RESERVED'").await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + admission.test_check_next_owner_operation().await; + release.notify_one(); + let error = timeout(Duration::from_secs(5), task) + .await + .unwrap() + .unwrap() + .err() + .unwrap(); + assert_eq!(error.code, SnapshotErrorCode::LeaseExpired); + assert_eq!(drops.load(Ordering::SeqCst), 1); + assert_eq!(tail_polls.load(Ordering::SeqCst), 0); + assert_eq!(fixture.counts.bytes.load(Ordering::SeqCst), 0); + assert_eq!(fixture.counts.receipt_writes.load(Ordering::SeqCst), 0); + assert_eq!(count(db, "mst2_chunk_map_source").await, 0); + assert_eq!(budget.used(), 0); + drop(admission); + repository + .maintain(&fixture.state.storage.git_service.obj_storage, 64) + .await + .unwrap(); + assert_eq!(count(db, "mst2_chunk_map").await, 0); +} diff --git a/src/api/router/snapshot_content.rs b/src/api/router/snapshot_content.rs index 275f7dd7..961d1b00 100644 --- a/src/api/router/snapshot_content.rs +++ b/src/api/router/snapshot_content.rs @@ -19,7 +19,7 @@ use super::{ }; use crate::ceres::snapshot::{ chunks::{ChunkMapSource, VerifiedSourceChunkMap, map_build_reservation_bytes}, - content_budget::{BudgetedFrame, MemoryLease, reserve_range_work, reserve_response}, + content_budget::{BudgetedFrame, MemoryLease, reserve_response}, error::{SnapshotError, SnapshotErrorCode}, pages::{MetadataWalkOutcome, base64_of, hex_of, resolve_abs_metadata}, resolver::FsKind, @@ -640,21 +640,29 @@ pub(super) async fn project_resolved, ) -> Result { let bytes = map_json_bytes(value, memory)?; let headers = super::REQUEST_HEADERS.try_with(Clone::clone).map_err(|_| { @@ -865,11 +874,17 @@ async fn guarded_map_json_response( ) })?; super::revalidate_access(state, context, &headers).await?; + map.ensure_live().await?; let state = state.clone(); let context = context.clone(); let stream = futures::stream::once(async move { super::revalidate_access(&state, &context, &headers).await?; - Ok::<_, SnapshotError>(bytes) + map.ensure_live().await?; + map.record_progress().await?; + Ok::<_, SnapshotError>(bytes::Bytes::from_owner(RetainedChunkFrame { + bytes, + _maps: std::sync::Arc::new(vec![map]), + })) }); Response::builder() .header("content-type", "application/json") @@ -879,6 +894,43 @@ async fn guarded_map_json_response( .map_err(|_| internal("chunk map JSON response build failed")) } +struct RetainedChunkFrame { + bytes: bytes::Bytes, + _maps: std::sync::Arc< + Vec>, + >, +} +impl AsRef<[u8]> for RetainedChunkFrame { + fn as_ref(&self) -> &[u8] { + &self.bytes + } +} + +fn retain_chunk_maps( + response: Response, + maps: Vec>, +) -> Response { + let maps = std::sync::Arc::new(maps); + let (parts, body) = response.into_parts(); + let stream = body.into_data_stream().then(move |frame| { + let maps = maps.clone(); + async move { + let bytes = frame.map_err(std::io::Error::other)?; + for map in maps.iter() { + map.ensure_live().await.map_err(std::io::Error::other)?; + if !bytes.is_empty() { + map.record_progress().await.map_err(std::io::Error::other)?; + } + } + Ok::<_, std::io::Error>(bytes::Bytes::from_owner(RetainedChunkFrame { + bytes, + _maps: maps, + })) + } + }); + Response::from_parts(parts, axum::body::Body::from_stream(stream)) +} + #[derive(Deserialize, Debug)] #[serde(deny_unknown_fields)] struct ChunksRequest { @@ -899,6 +951,20 @@ struct ChunkItem { const CHUNKS_MAX_ITEMS: usize = 128; const CHUNKS_TOTAL_MAX: u64 = 128 * 1024 * 1024; +pub(super) struct ChunksBudgets { + pub(super) response: std::sync::Arc, + pub(super) scratch: std::sync::Arc, +} + +impl Default for ChunksBudgets { + fn default() -> Self { + Self { + response: crate::ceres::snapshot::content_budget::response_budget().clone(), + scratch: crate::ceres::snapshot::content_budget::range_budget().clone(), + } + } +} + struct Planned { projection: std::sync::Arc, page: std::sync::Arc, @@ -932,6 +998,10 @@ async fn read_chunk_range( planned: &Planned, ) -> Result, Response> { let projection = &planned.projection; + projection + .ensure_live() + .await + .map_err(mst2_error_response)?; let len = projection.map.chunk_len(planned.index).map_err(|error| { mst2_error_response(internal(format!("invalid admitted chunk: {error}"))) })?; @@ -942,9 +1012,10 @@ async fn read_chunk_range( let end = start .checked_add(len) .ok_or_else(|| mst2_error_response(internal("chunk end overflow")))?; - let (mut input, meta) = handler - .get_raw_blob_range_stream_exact(&planned.oid, start, end) + let (mut input, meta) = projection + .await_backend(handler.get_raw_blob_range_stream_exact(&planned.oid, start, end)) .await + .map_err(mst2_error_response)? .map_err(|error| mst2_error_response(content_read_error(error)))? .ok_or_else(|| { mst2_error_response(SnapshotError::new( @@ -952,6 +1023,10 @@ async fn read_chunk_range( "fixed source does not support exact raw ranges", )) })?; + projection + .ensure_live() + .await + .map_err(mst2_error_response)?; if u64::try_from(meta.size).ok() != Some(projection.map.file_size) { return Err(mst2_error_response(SnapshotError::new( SnapshotErrorCode::IntegrityError, @@ -965,7 +1040,21 @@ async fn read_chunk_range( "range allocation could not be admitted", )) })?; - while let Some(part) = input.next().await { + let mut empty_parts = 0usize; + loop { + projection + .ensure_live() + .await + .map_err(mst2_error_response)?; + let part = projection + .await_backend(input.next()) + .await + .map_err(mst2_error_response)?; + projection + .ensure_live() + .await + .map_err(mst2_error_response)?; + let Some(part) = part else { break }; let bytes = part.map_err(|error| { tracing::warn!(error = %error, "fixed-view range stream failed"); mst2_error_response(SnapshotError::new( @@ -980,19 +1069,53 @@ async fn read_chunk_range( ))); } raw.extend_from_slice(&bytes); + if !bytes.is_empty() { + projection + .record_progress() + .await + .map_err(mst2_error_response)?; + } else { + empty_parts += 1; + if empty_parts == 32 { + tokio::task::yield_now().await; + projection + .ensure_live() + .await + .map_err(mst2_error_response)?; + empty_parts = 0; + } + } } planned .page .verify_chunk(&projection.map, planned.index, &raw) .map_err(mst2_error_response)?; + projection + .ensure_live() + .await + .map_err(mst2_error_response)?; + projection + .record_progress() + .await + .map_err(mst2_error_response)?; Ok(raw) } #[allow(clippy::result_large_err)] pub(super) async fn chunks( + state: State, + path: AxumPath, + body: Mst2Bytes, +) -> Result { + chunks_with_budgets(state, path, body, ChunksBudgets::default()).await +} + +#[allow(clippy::result_large_err)] +pub(super) async fn chunks_with_budgets( state: State, AxumPath(snapshot_id): AxumPath, Mst2Bytes(body): Mst2Bytes, + budgets: ChunksBudgets, ) -> Result { ensure(&state)?; let ctx = super::request_context(&state, &snapshot_id).map_err(mst2_error_response)?; @@ -1090,7 +1213,10 @@ pub(super) async fn chunks( .ok() .and_then(|bytes| bytes.checked_add(req.items.len() * 1024 + 1024)) .ok_or_else(|| mst2_error_response(internal("chunk response memory overflow")))?; - let response_memory = reserve_response(response_bytes).map_err(mst2_error_response)?; + let response_memory = budgets + .response + .reserve(response_bytes) + .map_err(mst2_error_response)?; let mut maps = std::collections::HashMap::new(); let mut pages = std::collections::HashMap::new(); for item in resolved { @@ -1141,7 +1267,10 @@ pub(super) async fn chunks( let mut stream = FrameStream::new(1, encoding); let mut out: Vec> = Vec::new(); for p in planned.iter() { - let _work_memory = reserve_range_work().map_err(mst2_error_response)?; + let _work_memory = budgets + .scratch + .reserve(crate::ceres::snapshot::content_budget::RANGE_WORK_BYTES) + .map_err(mst2_error_response)?; // The source is the current request's fixed OID, never a cached // handler/backend/credential from a different scope. let bytes = read_chunk_range(handler.as_ref(), p).await?; @@ -1163,7 +1292,7 @@ pub(super) async fn chunks( ); out.push(end); - guarded_treeframe_response_with_budget( + let response = guarded_treeframe_response_with_budget( &state, &ctx, &snapshot_id, @@ -1171,7 +1300,8 @@ pub(super) async fn chunks( out, Some(response_memory), ) - .map_err(mst2_error_response) + .map_err(mst2_error_response)?; + Ok(retain_chunk_maps(response, maps.into_values().collect())) } /// Strict decimal-string parse for unsigned counts (spec 04 §1: no leading diff --git a/src/api/router/snapshot_content_tests.rs b/src/api/router/snapshot_content_tests.rs index 2dc14015..052c8369 100644 --- a/src/api/router/snapshot_content_tests.rs +++ b/src/api/router/snapshot_content_tests.rs @@ -88,6 +88,9 @@ mod bounded_chunks; #[path = "snapshot_persisted_chunk_map_tests.rs"] mod persisted_chunk_maps; +#[path = "snapshot_chunk_map_retention_tests.rs"] +mod chunk_map_retention; + #[path = "snapshot_raw_blob_tests.rs"] mod raw_blob; @@ -107,6 +110,27 @@ mod generation_qualified_fixture; mod install_capability_fixture; type ReceiptWriteHold = (Arc, Arc); +type ReadOpenHold = ( + Arc, + Arc, + Arc, +); + +struct ReadOpenDrop(Arc); + +impl Drop for ReadOpenDrop { + fn drop(&mut self) { + self.0.fetch_add(1, Ordering::SeqCst); + } +} + +async fn await_read_open_hold(hold: Option) { + if let Some((entered, release, drops)) = hold { + let _owner = ReadOpenDrop(drops); + entered.notify_one(); + release.notified().await; + } +} #[path = "snapshot_rooted_metadata_tests.rs"] mod rooted_metadata; @@ -126,6 +150,15 @@ struct ReadCounts { receipt_read_meta_size: std::sync::atomic::AtomicI64, receipt_read_late_error: AtomicBool, receipt_write_holds: std::sync::Mutex>, + receipt_write_wait_drops: Arc, + receipt_late_create_holds: std::sync::Mutex>, + receipt_inventory_holds: std::sync::Mutex>, + receipt_inventory_calls: AtomicUsize, + receipt_retention_unsupported: AtomicBool, + receipt_deletes: AtomicUsize, + whole_open_holds: std::sync::Mutex>, + range_open_holds: std::sync::Mutex>, + receipt_open_holds: std::sync::Mutex>, } impl ReadCounts { @@ -151,12 +184,71 @@ struct CountingStorage { #[async_trait::async_trait] impl MegaObjectStorage for CountingStorage { + async fn chunk_map_receipt_inventory( + &self, + ) -> OrbitResult { + self.counts + .receipt_inventory_calls + .fetch_add(1, Ordering::SeqCst); + if self + .counts + .receipt_retention_unsupported + .load(Ordering::SeqCst) + { + return Err(crate::orbit_api::error::IoOrbitError::ChunkMapRetentionUnsupported); + } + let inventory = self.inner.inner.chunk_map_receipt_inventory().await?; + let holds = self.counts.receipt_inventory_holds.lock().unwrap().clone(); + if let Some((entered, release)) = holds { + entered.notify_one(); + release.notified().await; + } + Ok(inventory) + } + + async fn delete_chunk_map_receipt( + &self, + authority: &crate::orbit_api::object_storage::ChunkMapReceiptDeletion, + ) -> OrbitResult { + let deleted = self.inner.inner.delete_chunk_map_receipt(authority).await?; + if deleted { + self.counts.receipt_deletes.fetch_add(1, Ordering::SeqCst); + } + Ok(deleted) + } + async fn put_metadata_atomic_create( &self, key: &ObjectKey, bytes: Bytes, meta: ObjectMeta, ) -> OrbitResult<()> { + let late = if key.namespace == ObjectNamespace::ChunkMapReceipt { + self.counts + .receipt_late_create_holds + .lock() + .unwrap() + .clone() + } else { + None + }; + if let Some((entered, release)) = late { + let inner = self.inner.clone(); + let key = key.clone(); + let counts = self.counts.clone(); + return tokio::spawn(async move { + entered.notify_one(); + release.notified().await; + inner + .inner + .put_metadata_atomic_create(&key, bytes, meta) + .await?; + counts.receipt_writes.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + .await + .unwrap(); + } self.inner .inner .put_metadata_atomic_create(key, bytes, meta) @@ -165,6 +257,7 @@ impl MegaObjectStorage for CountingStorage { self.counts.receipt_writes.fetch_add(1, Ordering::SeqCst); let holds = self.counts.receipt_write_holds.lock().unwrap().clone(); if let Some((entered, release)) = holds { + let _hold = ReadOpenDrop(self.counts.receipt_write_wait_drops.clone()); entered.notify_one(); release.notified().await; } @@ -193,6 +286,8 @@ impl MegaObjectStorage for CountingStorage { async fn get_stream(&self, key: &ObjectKey) -> OrbitResult<(ObjectByteStream, ObjectMeta)> { if key.namespace == ObjectNamespace::ChunkMapReceipt { self.counts.receipt_reads.fetch_add(1, Ordering::SeqCst); + let holds = self.counts.receipt_open_holds.lock().unwrap().clone(); + await_read_open_hold(holds).await; if self.counts.receipt_read_failure.load(Ordering::SeqCst) { return Err( crate::orbit_api::error::IoOrbitError::object_store_not_found( @@ -225,6 +320,8 @@ impl MegaObjectStorage for CountingStorage { return self.inner.inner.get_stream(key).await; } self.counts.whole.fetch_add(1, Ordering::SeqCst); + let holds = self.counts.whole_open_holds.lock().unwrap().clone(); + await_read_open_hold(holds).await; let chunk_fault = self .counts .chunk_faults @@ -286,6 +383,8 @@ impl MegaObjectStorage for CountingStorage { end: u64, ) -> OrbitResult> { self.counts.range.fetch_add(1, Ordering::SeqCst); + let holds = self.counts.range_open_holds.lock().unwrap().clone(); + await_read_open_hold(holds).await; let fault = self .counts .chunk_faults @@ -312,7 +411,12 @@ impl MegaObjectStorage for CountingStorage { .inner .get_range_stream_exact(key, start, end) .await?; + let fault = self.counts.object_fault.lock().unwrap().clone(); Ok(result.map(|(stream, meta)| { + let stream = match fault { + Some(fault) if fault.oid == key.key => fault.stream(), + _ => stream, + }; let counts = self.counts.clone(); let stream = stream.map(move |part| { if let Ok(bytes) = &part { diff --git a/src/api/router/snapshot_persisted_chunk_map_tests.rs b/src/api/router/snapshot_persisted_chunk_map_tests.rs index 6b261115..23e1808b 100644 --- a/src/api/router/snapshot_persisted_chunk_map_tests.rs +++ b/src/api/router/snapshot_persisted_chunk_map_tests.rs @@ -832,7 +832,7 @@ pub(super) async fn assert_three_page_proofs_and_selected_sibling_faults( .await .unwrap(); } else { - db.execute_raw(statement("UPDATE mst2_chunk_map_node SET digest=$2 WHERE map_id=$1 AND first_page=2 AND page_count=1", [id.clone().into(), vec![9; 32].into()])).await.unwrap(); + db.execute_raw(statement("UPDATE mst2_chunk_map_node SET digest=$2 WHERE map_id=$1 AND first_page=2 AND page_count=1", [id.clone().into(), vec![9u8; 32].into()])).await.unwrap(); } db.execute_unprepared("ALTER TABLE mst2_chunk_map_node ENABLE TRIGGER USER") .await @@ -1160,7 +1160,7 @@ async fn ordinary_incomplete_source_dml_cannot_commit_an_admitted_partial_map() )) .await .unwrap(); - txn.execute_raw(statement("INSERT INTO mst2_chunk_map_source(storage_domain,git_oid,object_kind,fact_id,source_id,source_bytes,primary_scope,map_id,receipt_digest) VALUES('git',$1,'blob',$2,$3,$4,$5,$6,$7)", [fixture.oid.clone().into(), source.fact().id.into(), repository.source_identity(&source).unwrap().to_vec().into(), source.canonical_bytes().unwrap().into(), scope.to_vec().into(), map.map_id.to_vec().into(), vec![9;32].into()])).await.unwrap(); + txn.execute_raw(statement("INSERT INTO mst2_chunk_map_source(storage_domain,git_oid,object_kind,fact_id,source_id,source_bytes,primary_scope,map_id,receipt_digest) VALUES('git',$1,'blob',$2,$3,$4,$5,$6,$7)", [fixture.oid.clone().into(), source.fact().id.into(), repository.source_identity(&source).unwrap().to_vec().into(), source.canonical_bytes().unwrap().into(), scope.to_vec().into(), map.map_id.to_vec().into(), vec![9u8;32].into()])).await.unwrap(); assert!(txn.commit().await.is_err()); for table in [ "mst2_chunk_map", diff --git a/src/api/router/snapshot_raw_blob.rs b/src/api/router/snapshot_raw_blob.rs index ea438ec1..016310cd 100644 --- a/src/api/router/snapshot_raw_blob.rs +++ b/src/api/router/snapshot_raw_blob.rs @@ -133,10 +133,20 @@ pub(super) async fn blob_with_budgets( } else { None }; - let (input, meta) = handler - .get_raw_blob_stream_with_meta(&file.oid) - .await - .map_err(|error| mst2_error_response(content::content_read_error(error)))?; + if let Some(map) = map.as_ref() { + map.ensure_live().await.map_err(mst2_error_response)?; + } + let open = handler.get_raw_blob_stream_with_meta(&file.oid); + let opened = if let Some(map) = map.as_ref() { + map.await_backend(open).await.map_err(mst2_error_response)? + } else { + open.await + }; + let (input, meta) = + opened.map_err(|error| mst2_error_response(content::content_read_error(error)))?; + if let Some(map) = map.as_ref() { + map.ensure_live().await.map_err(mst2_error_response)?; + } if u64::try_from(meta.size).ok() != Some(file.size) { return Err(mst2_error_response(integrity( "raw source physical size disagrees with its fixed verified fact", @@ -231,7 +241,11 @@ impl AsRef<[u8]> for RawBlobChunk { impl RawBlobReader { async fn validate(&mut self) -> Result<(), SnapshotError> { - revalidate_access(&self.state, &self.context, &self.headers).await + revalidate_access(&self.state, &self.context, &self.headers).await?; + if let Some(map) = self.map.as_ref() { + map.ensure_live().await?; + } + Ok(()) } async fn next_chunk(&mut self) -> Result, SnapshotError> { @@ -327,6 +341,9 @@ impl RawBlobReader { } } self.validate().await?; + if let Some(map) = self.map.as_ref() { + map.record_progress().await?; + } if raw.capacity() > credit.bytes { return Err(integrity("raw chunk allocation exceeds its owned credit")); } @@ -342,19 +359,30 @@ impl RawBlobReader { } async fn poll_source(&mut self) -> Result, SnapshotError> { + self.validate().await?; let input = self .input .as_mut() .ok_or_else(|| integrity("raw source stream was already closed"))?; - let result = input.next().await; + let result = if let Some(map) = self.map.as_ref() { + map.await_backend(input.next()).await? + } else { + input.next().await + }; self.validate().await?; - result.transpose().map_err(|error| { + let part = result.transpose().map_err(|error| { tracing::warn!(%error, "fixed-view raw stream failed"); SnapshotError::new( SnapshotErrorCode::ObjectUnavailable, "fixed-view raw stream failed", ) - }) + })?; + if part.as_ref().is_some_and(|bytes| !bytes.is_empty()) { + if let Some(map) = self.map.as_ref() { + map.record_progress().await?; + } + } + Ok(part) } async fn require_eof(&mut self) -> Result<(), SnapshotError> { diff --git a/src/api/router/snapshot_raw_blob_tests.rs b/src/api/router/snapshot_raw_blob_tests.rs index 1c4e70f9..79a38c79 100644 --- a/src/api/router/snapshot_raw_blob_tests.rs +++ b/src/api/router/snapshot_raw_blob_tests.rs @@ -11,7 +11,7 @@ use crate::{ ceres::snapshot::content_budget::{MemoryBudget, RANGE_WORK_BYTES}, }; -fn budgeted_app( +pub(super) fn budgeted_app( fixture: &Fixture, response_budget: &Arc, scratch_budget: &Arc, diff --git a/src/ceres/snapshot/chunks.rs b/src/ceres/snapshot/chunks.rs index 8346cec2..8018c245 100644 --- a/src/ceres/snapshot/chunks.rs +++ b/src/ceres/snapshot/chunks.rs @@ -78,6 +78,7 @@ impl VerifiedSourceChunkMap { handler: &T, source: ChunkMapSource, budget: &Arc, + admission: &crate::jupiter::storage::native_chunk_map::retention::ChunkMapInstall, ) -> Result { let digest: [u8; 32] = source .fact @@ -106,6 +107,7 @@ impl VerifiedSourceChunkMap { }) }, budget, + admission, ) .await?; Ok(Self { source, projection }) diff --git a/src/ceres/snapshot/chunks_stream.rs b/src/ceres/snapshot/chunks_stream.rs index ab19e5b5..35e33e97 100644 --- a/src/ceres/snapshot/chunks_stream.rs +++ b/src/ceres/snapshot/chunks_stream.rs @@ -63,6 +63,7 @@ async fn build_stream( input, lease, size <= STAGED_CAP_BYTES as u64, + None, ) .await } @@ -72,6 +73,7 @@ pub(super) async fn build_source_stream( size: u64, open: F, budget: &Arc, + admission: &crate::jupiter::storage::native_chunk_map::retention::ChunkMapInstall, ) -> Result where F: FnOnce() -> Fut, @@ -84,6 +86,7 @@ where open, budget, BUILDERS.get_or_init(|| Arc::new(Semaphore::new(MAX_BUILDERS))), + Some(admission), ) .await } @@ -94,6 +97,7 @@ async fn build_source_with_resources( open: F, budget: &Arc, builders: &Arc, + admission: Option<&crate::jupiter::storage::native_chunk_map::retention::ChunkMapInstall>, ) -> Result where F: FnOnce() -> Fut, @@ -106,8 +110,15 @@ where ) })?; let lease = budget.reserve(source_reservation_bytes(size)?)?; - let input = open().await?; - build_stream_with_inline(content_id, size, input, lease, false).await + let input = if let Some(admission) = admission { + admission.ensure_live().await?; + let result = admission.await_backend(open()).await?; + admission.ensure_live().await?; + result? + } else { + open().await? + }; + build_stream_with_inline(content_id, size, input, lease, false, admission).await } pub(super) fn source_reservation_bytes(size: u64) -> Result { @@ -135,6 +146,7 @@ async fn build_stream_with_inline( mut input: ObjectByteStream, lease: MemoryLease, inline: bool, + admission: Option<&crate::jupiter::storage::native_chunk_map::retention::ChunkMapInstall>, ) -> Result { let chunk_count = size.div_ceil(CHUNK_SIZE as u64); let page_count = chunk_count.div_ceil(CHUNKS_PER_PAGE as u64); @@ -161,7 +173,17 @@ async fn build_stream_with_inline( let mut chunk_bytes = 0usize; let mut digests = 0u64; let mut since_yield = 0usize; - while let Some(part) = input.next().await { + let mut empty_parts = 0usize; + loop { + let next = if let Some(admission) = admission { + admission.ensure_live().await?; + let next = admission.await_backend(input.next()).await?; + admission.ensure_live().await?; + next + } else { + input.next().await + }; + let Some(part) = next else { break }; let bytes = part.map_err(|error| { tracing::warn!(error = %error, "fixed-view chunk projection stream failed"); SnapshotError::new( @@ -185,6 +207,18 @@ async fn build_stream_with_inline( raw.extend_from_slice(&bytes); } received += bytes.len() as u64; + if let Some(admission) = admission { + if !bytes.is_empty() { + admission.record_progress(received).await?; + } else { + empty_parts += 1; + if empty_parts == 32 { + tokio::task::yield_now().await; + admission.ensure_live().await?; + empty_parts = 0; + } + } + } let mut rest = bytes.as_ref(); while !rest.is_empty() { let len = rest.len().min(CHUNK_SIZE as usize - chunk_bytes); diff --git a/src/ceres/snapshot/chunks_stream_tests.rs b/src/ceres/snapshot/chunks_stream_tests.rs index 61de6453..d6f7e042 100644 --- a/src/ceres/snapshot/chunks_stream_tests.rs +++ b/src/ceres/snapshot/chunks_stream_tests.rs @@ -34,6 +34,7 @@ async fn exact_source_builder_admits_all_install_workspace_before_open_and_cance }, &budget, &builders, + None, ) .await .err() @@ -52,7 +53,8 @@ async fn exact_source_builder_admits_all_install_workspace_before_open_and_cance Ok(stream(vec![Ok(raw.clone())])) }, &budget, - &builders + &builders, + None ) .await .err() @@ -88,6 +90,7 @@ async fn exact_source_builder_admits_all_install_workspace_before_open_and_cance }, &task_budget, &task_builders, + None, ) .await }); @@ -110,6 +113,7 @@ async fn exact_source_builder_admits_all_install_workspace_before_open_and_cance }, &budget, &builders, + None, ) .await .unwrap(); diff --git a/src/ceres/snapshot/content_budget.rs b/src/ceres/snapshot/content_budget.rs index 6bb8d32b..18e16a51 100644 --- a/src/ceres/snapshot/content_budget.rs +++ b/src/ceres/snapshot/content_budget.rs @@ -89,10 +89,6 @@ pub(crate) fn response_budget() -> &'static Arc { BUDGET.get_or_init(|| MemoryBudget::new(RESPONSE_LIVE_BYTES)) } -pub(crate) fn reserve_range_work() -> Result { - range_budget().reserve(RANGE_WORK_BYTES) -} - pub(crate) fn range_budget() -> &'static Arc { static BUDGET: OnceLock> = OnceLock::new(); BUDGET.get_or_init(|| MemoryBudget::new(RANGE_SCRATCH_BYTES)) diff --git a/src/jupiter/migration/m20261008_000300_add_mst2_chunk_map_retention.rs b/src/jupiter/migration/m20261008_000300_add_mst2_chunk_map_retention.rs new file mode 100644 index 00000000..81e58229 --- /dev/null +++ b/src/jupiter/migration/m20261008_000300_add_mst2_chunk_map_retention.rs @@ -0,0 +1,26 @@ +use sea_orm::{ConnectionTrait, DbBackend}; +use sea_orm_migration::prelude::*; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + if manager.get_database_backend() != DbBackend::Postgres { + return Err(DbErr::Custom( + "chunk-map retention requires primary PostgreSQL".into(), + )); + } + manager + .get_connection() + .execute_unprepared(include_str!("m20261008_000300_chunk_map_retention.sql")) + .await?; + Ok(()) + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + // Never discard generation fences or late-create reconciliation history. + Ok(()) + } +} diff --git a/src/jupiter/migration/m20261008_000300_chunk_map_retention.sql b/src/jupiter/migration/m20261008_000300_chunk_map_retention.sql new file mode 100644 index 00000000..46a75aa3 --- /dev/null +++ b/src/jupiter/migration/m20261008_000300_chunk_map_retention.sql @@ -0,0 +1,262 @@ +-- Physical receipt generations are never reused. APPLIED history remains +-- bounded and is rechecked: a timed-out backend create can complete late. +CREATE SEQUENCE mst2_chunk_generation_sequence START 2; +ALTER TABLE mst2_chunk_map_source ADD COLUMN receipt_generation bigint NOT NULL DEFAULT 1 CHECK(receipt_generation>0); +ALTER TABLE mst2_chunk_map_source ADD COLUMN receipt_key text GENERATED ALWAYS AS + (CASE WHEN receipt_generation=1 THEN pg_catalog.encode(source_id,'hex') + ELSE pg_catalog.encode(source_id,'hex')||'-'||pg_catalog.lpad(pg_catalog.to_hex(receipt_generation),16,'0') END) STORED; +CREATE UNIQUE INDEX mst2_chunk_map_source_receipt_key ON mst2_chunk_map_source(receipt_key); + +CREATE TABLE mst2_chunk_receipt_generation ( + receipt_key text PRIMARY KEY CHECK(receipt_key ~ '^[0-9a-f]{64}(-[0-9a-f]{16})?$'), + source_id bytea NOT NULL CHECK(pg_catalog.octet_length(source_id)=32), + generation bigint NOT NULL CHECK(generation>0), + source_bytes bytea NOT NULL CHECK(pg_catalog.octet_length(source_bytes) BETWEEN 1 AND 2048), + primary_scope bytea NOT NULL CHECK(pg_catalog.octet_length(primary_scope) BETWEEN 1 AND 1024), + map_id bytea CHECK(map_id IS NULL OR pg_catalog.octet_length(map_id)=32), + map_generation bigint CHECK(map_generation IS NULL OR map_generation>0), + receipt_bytes bytea CHECK(receipt_bytes IS NULL OR pg_catalog.octet_length(receipt_bytes) BETWEEN 129 AND 4096), + create_completed boolean NOT NULL DEFAULT false, + create_completed_at timestamptz, + owner uuid, + state text NOT NULL CHECK(state IN ('RESERVED','CREATING','LIVE','DELETING','APPLIED')), + reserved_pages integer NOT NULL CHECK(reserved_pages BETWEEN 0 AND 32768), + reserved_nodes integer NOT NULL CHECK(reserved_nodes BETWEEN 0 AND 65535), + reserved_bytes bigint NOT NULL CHECK(reserved_bytes BETWEEN 0 AND 536870912), + deadline timestamptz, + created_at timestamptz NOT NULL DEFAULT pg_catalog.clock_timestamp(), + last_progress timestamptz NOT NULL DEFAULT pg_catalog.clock_timestamp(), + checked_at timestamptz NOT NULL DEFAULT '-infinity', + UNIQUE(source_id,generation), + CHECK(receipt_key=CASE WHEN generation=1 THEN pg_catalog.encode(source_id,'hex') ELSE pg_catalog.encode(source_id,'hex')||'-'||pg_catalog.lpad(pg_catalog.to_hex(generation),16,'0') END), + CHECK(state NOT IN ('CREATING','LIVE') OR (map_id IS NOT NULL AND receipt_bytes IS NOT NULL)), + CHECK(state NOT IN ('RESERVED','CREATING') OR (owner IS NOT NULL AND deadline IS NOT NULL)) +); +CREATE INDEX mst2_chunk_receipt_generation_maintenance ON mst2_chunk_receipt_generation(state,checked_at,receipt_key); +CREATE INDEX mst2_chunk_receipt_generation_created ON mst2_chunk_receipt_generation(created_at) WHERE receipt_bytes IS NOT NULL; +CREATE INDEX mst2_chunk_receipt_generation_completed ON mst2_chunk_receipt_generation(create_completed_at) WHERE receipt_bytes IS NOT NULL; +CREATE INDEX mst2_chunk_receipt_generation_install ON mst2_chunk_receipt_generation(map_id,map_generation,deadline) WHERE state='CREATING'; +CREATE TABLE mst2_chunk_map_lifetime ( + map_id bytea PRIMARY KEY REFERENCES mst2_chunk_map(map_id) DEFERRABLE INITIALLY DEFERRED, + generation bigint NOT NULL CHECK(generation>0), + state text NOT NULL CHECK(state IN ('LIVE','DELETING')), + last_used timestamptz NOT NULL DEFAULT pg_catalog.clock_timestamp(), + UNIQUE(map_id,generation) +); +CREATE TABLE mst2_chunk_reader ( + owner uuid PRIMARY KEY, + receipt_key text NOT NULL REFERENCES mst2_chunk_receipt_generation(receipt_key), + source_generation bigint NOT NULL CHECK(source_generation>0), + map_id bytea NOT NULL CHECK(pg_catalog.octet_length(map_id)=32), + map_generation bigint NOT NULL CHECK(map_generation>0), + deadline timestamptz NOT NULL, + last_progress timestamptz NOT NULL DEFAULT pg_catalog.clock_timestamp() +); +CREATE INDEX mst2_chunk_reader_receipt ON mst2_chunk_reader(receipt_key,source_generation,deadline); +CREATE INDEX mst2_chunk_reader_map ON mst2_chunk_reader(map_id,map_generation,deadline); +CREATE TABLE mst2_chunk_map_gc ( + map_id bytea NOT NULL CHECK(pg_catalog.octet_length(map_id)=32), + generation bigint NOT NULL CHECK(generation>0), + state text NOT NULL CHECK(state IN ('PENDING','APPLIED')), + created_at timestamptz NOT NULL DEFAULT pg_catalog.clock_timestamp(), + applied_at timestamptz, + PRIMARY KEY(map_id,generation) +); + +-- Bootstrap existing #77 data from actual bounded rows. SQL does not infer +-- backing receipt presence; repository admission separately inventories it. +DO $$ DECLARE maps bigint; sources bigint; leaves bigint; nodes bigint; bytes bigint; BEGIN + SELECT pg_catalog.count(*) INTO maps FROM mst2_chunk_map; + SELECT pg_catalog.count(*) INTO sources FROM mst2_chunk_map_source; + SELECT pg_catalog.count(*) INTO leaves FROM mst2_chunk_map_leaf; + SELECT pg_catalog.count(*) INTO nodes FROM mst2_chunk_map_node; + SELECT COALESCE(pg_catalog.sum(pg_catalog.pg_total_relation_size(c.oid)),0) INTO bytes + FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname=pg_catalog.current_schema() AND c.relkind='r' + AND c.relname IN ('mst2_chunk_map','mst2_chunk_map_source','mst2_chunk_map_leaf','mst2_chunk_map_node'); + IF maps>4096 OR sources>16384 OR leaves>65536 OR nodes>131072 OR bytes>536870912 THEN + RAISE EXCEPTION 'existing chunk-map indexes exceed bounded retention bootstrap'; + END IF; +END $$; + +INSERT INTO mst2_chunk_map_lifetime(map_id,generation,state) SELECT map_id,1,'LIVE' FROM mst2_chunk_map; +INSERT INTO mst2_chunk_receipt_generation(receipt_key,source_id,generation,source_bytes,primary_scope,map_id,map_generation,receipt_bytes,create_completed,create_completed_at,state,reserved_pages,reserved_nodes,reserved_bytes) + SELECT s.receipt_key,s.source_id,1,s.source_bytes,s.primary_scope,s.map_id,1, + pg_catalog.convert_to('MST2-CHUNK-MAP-RECEIPT','UTF8')||pg_catalog.decode('00','hex')|| + pg_catalog.int4send(pg_catalog.octet_length(s.primary_scope))||s.primary_scope|| + pg_catalog.int4send(pg_catalog.octet_length(s.source_bytes))||s.source_bytes||m.descriptor, + true,pg_catalog.clock_timestamp(),'LIVE',0,0,0 FROM mst2_chunk_map_source s JOIN mst2_chunk_map m ON m.map_id=s.map_id; + +CREATE FUNCTION mst2_chunk_retention_barrier() RETURNS void LANGUAGE plpgsql AS $$ +BEGIN + IF pg_catalog.pg_is_in_recovery() OR pg_catalog.current_setting('transaction_isolation')<>'read committed' THEN + RAISE EXCEPTION 'chunk retention requires primary READ COMMITTED'; + END IF; + PERFORM pg_catalog.pg_advisory_xact_lock(1296717363,pg_catalog.hashtext(pg_catalog.current_schema())); +END $$; + +CREATE FUNCTION mst2_chunk_retention_bytes() RETURNS bigint LANGUAGE sql AS $$ + SELECT COALESCE(pg_catalog.sum(pg_catalog.pg_total_relation_size(c.oid)),0)::bigint + FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname=pg_catalog.current_schema() AND c.relkind='r' + AND c.relname IN ('mst2_chunk_map','mst2_chunk_map_source','mst2_chunk_map_leaf','mst2_chunk_map_node', + 'mst2_chunk_receipt_generation','mst2_chunk_map_lifetime','mst2_chunk_reader','mst2_chunk_map_gc') +$$; +DO $$ BEGIN + IF mst2_chunk_retention_bytes()>536870912 THEN RAISE EXCEPTION 'bounded chunk retention bootstrap exceeds actual durable byte quota'; END IF; +END $$; + +CREATE OR REPLACE FUNCTION mst2_chunk_map_primary() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF pg_catalog.current_schema() IS DISTINCT FROM TG_TABLE_SCHEMA THEN + RAISE EXCEPTION 'chunk resident insertion requires its actual primary schema'; + END IF; + PERFORM mst2_chunk_retention_barrier(); + IF mst2_chunk_retention_bytes()+1048576>536870912 THEN + RAISE EXCEPTION 'chunk resident insertion lacks actual durable byte headroom'; + END IF; + RETURN NULL; +END $$; + +-- Preserve the original immutable/source-completeness oracles. Only exact +-- terminal generations with no live owners may lose their resident indexes. +CREATE OR REPLACE FUNCTION mst2_chunk_map_immutable() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE permitted boolean:=false; +BEGIN + IF TG_OP<>'DELETE' OR pg_catalog.current_schema() IS DISTINCT FROM TG_TABLE_SCHEMA THEN + RAISE EXCEPTION 'chunk map indexes and source receipts are immutable'; + END IF; + PERFORM mst2_chunk_retention_barrier(); + IF TG_TABLE_NAME='mst2_chunk_map_source' THEN + EXECUTE pg_catalog.format('SELECT EXISTS(SELECT 1 FROM %I.mst2_chunk_receipt_generation g WHERE g.receipt_key=$1 AND g.source_id=$2 AND g.generation=$3 AND g.map_id=$4 AND g.state=''APPLIED'') AND NOT EXISTS(SELECT 1 FROM %I.mst2_chunk_reader r WHERE r.receipt_key=$1 AND r.source_generation=$3 AND r.deadline>pg_catalog.clock_timestamp())',TG_TABLE_SCHEMA,TG_TABLE_SCHEMA) + INTO permitted USING OLD.receipt_key,OLD.source_id,OLD.receipt_generation,OLD.map_id; + ELSE + EXECUTE pg_catalog.format('SELECT EXISTS(SELECT 1 FROM %I.mst2_chunk_map_lifetime l JOIN %I.mst2_chunk_map_gc g ON g.map_id=l.map_id AND g.generation=l.generation WHERE l.map_id=$1 AND l.state=''DELETING'' AND g.state=''PENDING'') AND NOT EXISTS(SELECT 1 FROM %I.mst2_chunk_map_source s WHERE s.map_id=$1) AND NOT EXISTS(SELECT 1 FROM %I.mst2_chunk_reader r JOIN %I.mst2_chunk_map_lifetime l ON l.map_id=r.map_id AND l.generation=r.map_generation WHERE r.map_id=$1 AND r.deadline>pg_catalog.clock_timestamp())',TG_TABLE_SCHEMA,TG_TABLE_SCHEMA,TG_TABLE_SCHEMA,TG_TABLE_SCHEMA,TG_TABLE_SCHEMA) + INTO permitted USING OLD.map_id; + END IF; + IF NOT permitted THEN RAISE EXCEPTION 'chunk map deletion lacks an exact retired generation'; END IF; + RETURN OLD; +END $$; + +CREATE FUNCTION mst2_chunk_generation_guard() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE owned boolean; +BEGIN + IF pg_catalog.current_schema() IS DISTINCT FROM TG_TABLE_SCHEMA THEN + RAISE EXCEPTION 'chunk retention mutation requires its actual primary schema'; + END IF; + PERFORM mst2_chunk_retention_barrier(); + IF TG_OP='INSERT' THEN + EXECUTE pg_catalog.format('SELECT pg_catalog.count(*)>=131072 FROM %I.mst2_chunk_receipt_generation',TG_TABLE_SCHEMA) INTO owned; + IF owned THEN RAISE EXCEPTION 'chunk receipt history quota exhausted'; END IF; + END IF; + IF TG_OP='DELETE' OR TG_OP='TRUNCATE' THEN + RAISE EXCEPTION 'chunk receipt generations and collection history are retained'; + END IF; + IF mst2_chunk_retention_bytes()+CASE WHEN NEW.state IN ('DELETING','APPLIED') THEN 262144 ELSE 1048576 END>536870912 THEN + RAISE EXCEPTION 'chunk receipt mutation lacks actual durable byte headroom'; + END IF; + IF TG_OP='UPDATE' AND (NEW.receipt_key IS DISTINCT FROM OLD.receipt_key OR NEW.source_id IS DISTINCT FROM OLD.source_id + OR NEW.generation IS DISTINCT FROM OLD.generation OR NEW.source_bytes IS DISTINCT FROM OLD.source_bytes + OR NEW.primary_scope IS DISTINCT FROM OLD.primary_scope OR NEW.created_at IS DISTINCT FROM OLD.created_at + OR (OLD.receipt_bytes IS NOT NULL AND NEW.receipt_bytes IS DISTINCT FROM OLD.receipt_bytes) + OR (OLD.map_id IS NOT NULL AND NEW.map_id IS DISTINCT FROM OLD.map_id) + OR (OLD.map_generation IS NOT NULL AND NEW.map_generation IS DISTINCT FROM OLD.map_generation) + OR (OLD.create_completed AND NOT NEW.create_completed) + OR (OLD.create_completed_at IS NOT NULL AND NEW.create_completed_at IS DISTINCT FROM OLD.create_completed_at) + OR (OLD.state='APPLIED' AND NEW.state<>'APPLIED') + OR (OLD.state='DELETING' AND NEW.state NOT IN ('DELETING','APPLIED')) + OR (OLD.state='LIVE' AND NEW.state NOT IN ('LIVE','DELETING')) + OR (OLD.state='CREATING' AND NEW.state NOT IN ('CREATING','LIVE','DELETING')) + OR (OLD.state='RESERVED' AND NEW.state NOT IN ('RESERVED','CREATING','DELETING','APPLIED'))) THEN + RAISE EXCEPTION 'chunk receipt generation identity or transition changed'; + END IF; + IF NEW.state='DELETING' AND (TG_OP='INSERT' OR OLD.state<>'DELETING') THEN + EXECUTE pg_catalog.format('SELECT EXISTS(SELECT 1 FROM %I.mst2_chunk_reader WHERE receipt_key=$1 AND source_generation=$2 AND deadline>pg_catalog.clock_timestamp())',TG_TABLE_SCHEMA) + INTO owned USING NEW.receipt_key,NEW.generation; + IF owned THEN RAISE EXCEPTION 'receipt retirement cannot cross a live reader'; END IF; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_chunk_generation_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_chunk_receipt_generation + FOR EACH ROW EXECUTE FUNCTION mst2_chunk_generation_guard(); +CREATE TRIGGER mst2_chunk_generation_no_truncate BEFORE TRUNCATE ON mst2_chunk_receipt_generation + FOR EACH STATEMENT EXECUTE FUNCTION mst2_chunk_generation_guard(); + +CREATE FUNCTION mst2_chunk_gc_history_guard() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE full_history boolean; +BEGIN + IF pg_catalog.current_schema() IS DISTINCT FROM TG_TABLE_SCHEMA THEN RAISE EXCEPTION 'wrong chunk history schema'; END IF; + PERFORM mst2_chunk_retention_barrier(); + IF TG_OP='INSERT' THEN + EXECUTE pg_catalog.format('SELECT pg_catalog.count(*)>=131072 FROM %I.mst2_chunk_map_gc',TG_TABLE_SCHEMA) INTO full_history; + IF full_history THEN RAISE EXCEPTION 'chunk collection history quota exhausted'; END IF; + END IF; + IF TG_OP='DELETE' OR TG_OP='TRUNCATE' THEN RAISE EXCEPTION 'chunk collection history is retained'; END IF; + IF mst2_chunk_retention_bytes()+262144>536870912 THEN RAISE EXCEPTION 'chunk collection history lacks actual durable byte headroom'; END IF; + IF TG_OP='UPDATE' AND (NEW.map_id IS DISTINCT FROM OLD.map_id OR NEW.generation IS DISTINCT FROM OLD.generation + OR NEW.created_at IS DISTINCT FROM OLD.created_at OR OLD.state='APPLIED' OR NEW.state<>'APPLIED') THEN + RAISE EXCEPTION 'chunk collection history is immutable except terminal completion'; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_chunk_gc_history_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_chunk_map_gc + FOR EACH ROW EXECUTE FUNCTION mst2_chunk_gc_history_guard(); +CREATE TRIGGER mst2_chunk_gc_no_truncate BEFORE TRUNCATE ON mst2_chunk_map_gc + FOR EACH STATEMENT EXECUTE FUNCTION mst2_chunk_gc_history_guard(); + +CREATE FUNCTION mst2_chunk_reader_guard() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE live boolean; +BEGIN + IF pg_catalog.current_schema() IS DISTINCT FROM TG_TABLE_SCHEMA THEN RAISE EXCEPTION 'wrong chunk reader schema'; END IF; + PERFORM mst2_chunk_retention_barrier(); + IF TG_OP='TRUNCATE' THEN RAISE EXCEPTION 'chunk readers cannot be truncated'; END IF; + IF TG_OP='DELETE' THEN RETURN OLD; END IF; + IF mst2_chunk_retention_bytes()+1048576>536870912 THEN RAISE EXCEPTION 'chunk reader mutation lacks actual durable byte headroom'; END IF; + IF TG_OP='INSERT' THEN + EXECUTE pg_catalog.format('SELECT pg_catalog.count(*)>=4096 FROM %I.mst2_chunk_reader',TG_TABLE_SCHEMA) INTO live; + IF live THEN RAISE EXCEPTION 'chunk reader quota exhausted'; END IF; + END IF; + IF NEW.deadline>pg_catalog.clock_timestamp()+interval '60 seconds' OR NEW.deadline<=pg_catalog.clock_timestamp() + OR (TG_OP='UPDATE' AND (OLD.deadline<=pg_catalog.clock_timestamp() OR NEW.owner IS DISTINCT FROM OLD.owner + OR NEW.receipt_key IS DISTINCT FROM OLD.receipt_key OR NEW.source_generation IS DISTINCT FROM OLD.source_generation + OR NEW.map_id IS DISTINCT FROM OLD.map_id OR NEW.map_generation IS DISTINCT FROM OLD.map_generation)) THEN + RAISE EXCEPTION 'chunk reader cannot renew an expired or different owner'; + END IF; + EXECUTE pg_catalog.format('SELECT EXISTS(SELECT 1 FROM %I.mst2_chunk_receipt_generation g JOIN %I.mst2_chunk_map_lifetime l ON l.map_id=g.map_id AND l.generation=g.map_generation JOIN %I.mst2_chunk_map_source s ON s.receipt_key=g.receipt_key AND s.receipt_generation=g.generation WHERE g.receipt_key=$1 AND g.generation=$2 AND g.map_id=$3 AND g.map_generation=$4 AND g.state=''LIVE'' AND l.state=''LIVE'')',TG_TABLE_SCHEMA,TG_TABLE_SCHEMA,TG_TABLE_SCHEMA) + INTO live USING NEW.receipt_key,NEW.source_generation,NEW.map_id,NEW.map_generation; + IF NOT live THEN RAISE EXCEPTION 'chunk reader lacks exact live source and map generations'; END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_chunk_reader_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_chunk_reader + FOR EACH ROW EXECUTE FUNCTION mst2_chunk_reader_guard(); +CREATE TRIGGER mst2_chunk_reader_no_truncate BEFORE TRUNCATE ON mst2_chunk_reader + FOR EACH STATEMENT EXECUTE FUNCTION mst2_chunk_reader_guard(); + +CREATE FUNCTION mst2_chunk_map_lifetime_guard() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE permitted boolean; +BEGIN + IF pg_catalog.current_schema() IS DISTINCT FROM TG_TABLE_SCHEMA THEN RAISE EXCEPTION 'wrong chunk map lifetime schema'; END IF; + PERFORM mst2_chunk_retention_barrier(); + IF TG_OP='TRUNCATE' THEN RAISE EXCEPTION 'chunk map lifetimes cannot be truncated'; END IF; + IF TG_OP='DELETE' THEN + EXECUTE pg_catalog.format('SELECT $2=''DELETING'' AND EXISTS(SELECT 1 FROM %I.mst2_chunk_map_gc g WHERE g.map_id=$1 AND g.generation=$3 AND g.state=''PENDING'') AND NOT EXISTS(SELECT 1 FROM %I.mst2_chunk_map_source s WHERE s.map_id=$1) AND NOT EXISTS(SELECT 1 FROM %I.mst2_chunk_reader r WHERE r.map_id=$1 AND r.map_generation=$3 AND r.deadline>pg_catalog.clock_timestamp())',TG_TABLE_SCHEMA,TG_TABLE_SCHEMA,TG_TABLE_SCHEMA) + INTO permitted USING OLD.map_id,OLD.state,OLD.generation; + IF NOT permitted THEN RAISE EXCEPTION 'map lifetime deletion lacks exact reader-free collection claim'; END IF; + RETURN OLD; + END IF; + IF mst2_chunk_retention_bytes()+CASE WHEN NEW.state='DELETING' THEN 262144 ELSE 1048576 END>536870912 THEN + RAISE EXCEPTION 'chunk map lifetime mutation lacks actual durable byte headroom'; + END IF; + IF TG_OP='UPDATE' AND (NEW.map_id IS DISTINCT FROM OLD.map_id OR NEW.generation IS DISTINCT FROM OLD.generation + OR (OLD.state='DELETING' AND NEW.state<>'DELETING')) THEN RAISE EXCEPTION 'map lifetime cannot reincarnate in place'; END IF; + IF TG_OP='UPDATE' AND NEW.state='DELETING' AND OLD.state='LIVE' THEN + EXECUTE pg_catalog.format('SELECT EXISTS(SELECT 1 FROM %I.mst2_chunk_map_gc g WHERE g.map_id=$1 AND g.generation=$2 AND g.state=''PENDING'') AND NOT EXISTS(SELECT 1 FROM %I.mst2_chunk_map_source WHERE map_id=$1) AND NOT EXISTS(SELECT 1 FROM %I.mst2_chunk_reader WHERE map_id=$1 AND map_generation=$2 AND deadline>pg_catalog.clock_timestamp()) AND NOT EXISTS(SELECT 1 FROM %I.mst2_chunk_receipt_generation WHERE map_id=$1 AND map_generation=$2 AND state=''CREATING'' AND deadline>pg_catalog.clock_timestamp())',TG_TABLE_SCHEMA,TG_TABLE_SCHEMA,TG_TABLE_SCHEMA,TG_TABLE_SCHEMA) + INTO permitted USING OLD.map_id,OLD.generation; + IF NOT permitted THEN RAISE EXCEPTION 'map collection cannot cross a reader or active partial installation'; END IF; + END IF; + RETURN NEW; +END $$; +CREATE TRIGGER mst2_chunk_map_lifetime_guard BEFORE INSERT OR UPDATE OR DELETE ON mst2_chunk_map_lifetime + FOR EACH ROW EXECUTE FUNCTION mst2_chunk_map_lifetime_guard(); +CREATE TRIGGER mst2_chunk_map_lifetime_no_truncate BEFORE TRUNCATE ON mst2_chunk_map_lifetime + FOR EACH STATEMENT EXECUTE FUNCTION mst2_chunk_map_lifetime_guard(); diff --git a/src/jupiter/migration/mod.rs b/src/jupiter/migration/mod.rs index 2fbc20ce..018716bc 100644 --- a/src/jupiter/migration/mod.rs +++ b/src/jupiter/migration/mod.rs @@ -153,6 +153,7 @@ mod m20261007_000500_add_mst2_install_capability; mod m20261007_000600_add_mst2_storage_routes; mod m20261008_000100_add_mst2_chunk_maps; mod m20261008_000200_add_mst2_rooted_qualified_family; +mod m20261008_000300_add_mst2_chunk_map_retention; mod runner; pub use m20260905_000100_add_push_queue::ensure_queue_control_seed; pub use runner::apply_migrations; @@ -294,6 +295,7 @@ impl MigratorTrait for Migrator { Box::new(m20261007_000600_add_mst2_storage_routes::Migration), Box::new(m20261008_000100_add_mst2_chunk_maps::Migration), Box::new(m20261008_000200_add_mst2_rooted_qualified_family::Migration), + Box::new(m20261008_000300_add_mst2_chunk_map_retention::Migration), ] } } @@ -1204,7 +1206,7 @@ mod tests { async fn import_repo_alias_rows_canonicalized() { let names = migration_names(); assert_eq!( - &names[names.len() - 16..names.len() - 7], + &names[names.len() - 17..names.len() - 8], &[ "m20260923_000200_canonicalize_import_repo_paths".to_string(), "m20260925_000100_media_paging".to_string(), @@ -1219,34 +1221,38 @@ mod tests { "native retention, metadata installation and view tables follow media paging" ); assert_eq!( - &names[names.len() - 7], + &names[names.len() - 8], "m20261007_000200_add_mst2_metadata_generations" ); assert_eq!( - &names[names.len() - 6], + &names[names.len() - 7], "m20261007_000300_add_mst2_metadata_lifetime_history" ); assert_eq!( - &names[names.len() - 5], + &names[names.len() - 6], "m20261007_000400_add_mst2_qualified_metadata_gc" ); assert_eq!( - &names[names.len() - 4], + &names[names.len() - 5], "m20261007_000500_add_mst2_install_capability" ); assert_eq!( - &names[names.len() - 3], + &names[names.len() - 4], "m20261007_000600_add_mst2_storage_routes" ); assert_eq!( - &names[names.len() - 2], + &names[names.len() - 3], "m20261008_000100_add_mst2_chunk_maps" ); assert_eq!( - names.last().unwrap(), + &names[names.len() - 2], "m20261008_000200_add_mst2_rooted_qualified_family" ); + assert_eq!( + names.last().unwrap(), + "m20261008_000300_add_mst2_chunk_map_retention" + ); let db = alias_db().await; insert_repo(&db, 1, "/third-party//a").await; insert_repo(&db, 2, "/third-party/b/").await; diff --git a/src/jupiter/storage/native_chunk_map.rs b/src/jupiter/storage/native_chunk_map.rs index 5ce849a8..1d5e5666 100644 --- a/src/jupiter/storage/native_chunk_map.rs +++ b/src/jupiter/storage/native_chunk_map.rs @@ -2,8 +2,8 @@ //! //! The object writer is trusted like the verified-object writer. A database //! row, its checksum or a map's presence is never a full-source proof. Only -//! the opaque full-stream verifier can install a source receipt. This initial -//! store is append-only; bounded retention and collection remain separate work. +//! the opaque full-stream verifier can install a source receipt. Persistent +//! owners, reservations and physical generations bound its retention. use std::sync::Arc; @@ -33,11 +33,17 @@ const DESCRIPTOR_CREDIT: usize = 32 * 1024; const PAGE_CREDIT: usize = 64 * 1024; const RECEIPT_DOMAIN: &[u8] = b"MST2-CHUNK-MAP-RECEIPT\0"; +pub(crate) mod retention; + +#[derive(Clone)] pub(crate) struct PostgresChunkMapRepository { connection: DatabaseConnection, primary_scope: Vec, budget: Arc, schema: String, + receipt_observation: Arc>>>, + cancelled_installs: Arc>>, + admission_maintenance: Arc>>, } pub(crate) struct PersistedChunkMap { @@ -45,6 +51,7 @@ pub(crate) struct PersistedChunkMap { pub map_id: [u8; 32], source: ChunkMapSource, source_id: [u8; 32], + reader: retention::ChunkMapReader, _memory: MemoryLease, } @@ -52,6 +59,26 @@ impl PersistedChunkMap { pub(crate) fn source_id(&self) -> [u8; 32] { self.source_id } + + pub(crate) async fn ensure_live(&self) -> Result<(), SnapshotError> { + self.reader.ensure_live().await + } + + pub(crate) async fn record_progress(&self) -> Result<(), SnapshotError> { + self.reader.record_progress().await + } + + pub(crate) async fn await_backend( + &self, + future: F, + ) -> Result { + self.reader.await_backend(future).await + } + + #[cfg(test)] + pub(crate) async fn test_check_next_owner_operation(&self) { + self.reader.test_check_next_owner_operation().await; + } } pub(crate) struct AuthenticatedChunkPage { @@ -100,6 +127,9 @@ impl PostgresChunkMapRepository { primary_scope, budget: projection_budget().clone(), schema, + receipt_observation: Arc::default(), + cancelled_installs: Arc::default(), + admission_maintenance: Arc::default(), }) } @@ -108,27 +138,78 @@ impl PostgresChunkMapRepository { source: &ChunkMapSource, objects: &MegaObjectStorageWrapper, ) -> Result>, SnapshotError> { + for pass in 0..2 { + let (map, pending) = self.read_current(source, objects).await?; + if !pending { + return Ok(map); + } + if pass == 0 { + self.maintain(objects, 8).await?; + } + } + Err(SnapshotError::new( + SnapshotErrorCode::TemporaryUnavailable, + "previous source receipt deletion is still pending", + )) + } + + async fn read_current( + &self, + source: &ChunkMapSource, + objects: &MegaObjectStorageWrapper, + ) -> Result<(Option>, bool), SnapshotError> { let memory = self.budget.reserve(DESCRIPTOR_CREDIT)?; let txn = self.transaction().await?; let result = async { self.require_scope(&txn).await?; require_current_source(&txn, source, &self.schema).await?; let Some(row) = source_row(&txn, source, &self.schema).await? else { - return Ok(None); + let pending = self.require_missing_source_retired(&txn, source).await?; + return Ok((None, pending)); }; let (map, source_id) = self.validate_source_row(source, &row)?; + let generation_state: Option = + row.try_get("", "generation_state").map_err(db_error)?; + if generation_state.as_deref() == Some("DELETING") { + // A legitimate retirement can still retain its source row + // until the physical delete completes. Replay it just like + // a missing retired row, after checking every source fact; + // damaged LIVE rows never enter this cold-rebuild path. + return Ok((None, true)); + } let bytes = receipt(&self.primary_scope, source, &map)?; - read_receipt(objects, &receipt_key(source_id), &bytes).await?; - Ok(Some(Arc::new(PersistedChunkMap { - map_id: map.map_id(), - map, - source: source.clone(), - source_id, - _memory: memory, - }))) + let reader = self.admit_reader(&txn, source, &row, &map).await?; + let key = ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: row.try_get("", "receipt_key").map_err(db_error)?, + }; + Ok(( + Some(( + Arc::new(PersistedChunkMap { + map_id: map.map_id(), + map, + source: source.clone(), + source_id, + reader, + _memory: memory, + }), + key, + bytes, + )), + false, + )) } .await; - finish(txn, result).await + let (admitted, pending) = finish(txn, result).await?; + let Some((map, key, bytes)) = admitted else { + return Ok((None, pending)); + }; + // Admission is durable before I/O. No completion barrier or fact row + // lock is held while the independent backend receipt is consumed. + map.await_backend(read_receipt(objects, &key, &bytes)) + .await??; + map.reader.validate_after_receipt().await?; + Ok((Some(map), false)) } /// Publication accepts no caller-provided map or "verified" flag. @@ -136,75 +217,96 @@ impl PostgresChunkMapRepository { &self, verified: VerifiedSourceChunkMap, objects: &MegaObjectStorageWrapper, + admission: &retention::ChunkMapInstall, + ) -> Result<(), SnapshotError> { + let result = self.install_reserved(verified, objects, admission).await; + if result.is_err() { + if let Err(error) = admission.retire().await { + tracing::warn!( + ?error, + "failed chunk-map installation will retire by database deadline" + ); + } else if let Err(error) = self.collect_maps(64).await { + tracing::warn!( + ?error, + "retired partial chunk-map indexes await bounded collection replay" + ); + } + } + result + } + + async fn install_reserved( + &self, + verified: VerifiedSourceChunkMap, + objects: &MegaObjectStorageWrapper, + admission: &retention::ChunkMapInstall, ) -> Result<(), SnapshotError> { let source = verified.source(); let map = verified.map(); - // The verifier reserved index and bounded SQL parameter workspace - // before opening the body. It owns those credits through publication. let nodes = indexed_nodes(verified.leaf_hashes())?; - if nodes.last().map(|n| n.digest) != Some(map.pages_root) { + if nodes.last().map(|node| node.digest) != Some(map.pages_root) { return Err(integrity( "verified chunk map index disagrees with canonical root", )); } - let source_bytes = source.canonical_bytes()?; - let source_id = source_id(&self.primary_scope, &source_bytes); - let bytes = receipt(&self.primary_scope, source, map)?; - let key = receipt_key(source_id); + let bytes = admission.prepare(&verified).await?; + let key = admission.key(); + admission + .await_backend(objects.inner.put_metadata_atomic_create( + key, + Bytes::copy_from_slice(&bytes), + ObjectMeta { + size: bytes.len() as i64, + ..Default::default() + }, + )) + .await? + .map_err(storage_error)?; + admission.acknowledge_create().await?; + admission + .await_backend(read_receipt(objects, key, &bytes)) + .await??; + let observation = retention::inventory(self, objects).await?; + let map_generation = self + .stage_map_start(&verified, admission, &observation) + .await?; + for batch in verified.leaves().chunks(64) { + self.stage_map_batch(&verified, admission, map_generation, Some(batch), None) + .await?; + } + for batch in nodes.chunks(1024) { + self.stage_map_batch(&verified, admission, map_generation, None, Some(batch)) + .await?; + } + self.compare_staged_map(&verified, &nodes, admission, map_generation) + .await?; let txn = self.transaction().await?; - let result = async { + let result=async { self.require_scope(&txn).await?; - require_current_source(&txn, source, &self.schema).await?; - if let Some(row) = source_row(&txn, source, &self.schema).await? { - let (stored, _) = self.validate_source_row(source, &row)?; - if stored != *map { return Err(integrity("immutable source map conflicts with reverified content")); } - read_receipt(objects, &key, &bytes).await?; - compare_complete_map(&txn, &verified, &nodes, &self.schema).await?; - return Ok(()); - } - // The complete source pass happened before this atomic create. - // A lost DB commit leaves an orphan trusted receipt, never an - // admitted partial map; exact re-verification can replay it. - objects.inner.put_metadata_atomic_create(&key, Bytes::copy_from_slice(&bytes), ObjectMeta { - size: bytes.len() as i64, ..Default::default() - }).await.map_err(storage_error)?; - read_receipt(objects, &key, &bytes).await?; - txn.execute_raw(stmt(&self.schema, "INSERT INTO mst2_chunk_map(map_id,descriptor,page_count,pages_root) VALUES($1,$2,$3,$4) ON CONFLICT(map_id) DO NOTHING", - [map.map_id().to_vec().into(), map.encode().into(), (map.page_count as i32).into(), map.pages_root.to_vec().into()])).await.map_err(db_error)?; - // Existing indexes must be compared, never repaired or overwritten. - let present = txn.query_one_raw(stmt(&self.schema, "SELECT EXISTS(SELECT 1 FROM mst2_chunk_map_source WHERE map_id=$1) AS sealed", [map.map_id().to_vec().into()])) - .await.map_err(db_error)?.ok_or_else(|| integrity("chunk map seal observation is missing"))? - .try_get::("", "sealed").map_err(db_error)?; - if !present { - for batch in verified.leaves().chunks(64) { - let encoded = encode_leaves(batch)?; - txn.execute_raw(stmt(&self.schema, "INSERT INTO mst2_chunk_map_leaf(map_id,page_index,payload) SELECT $1,p.page_index,pg_catalog.decode(p.payload,'hex') FROM pg_catalog.jsonb_to_recordset($2::jsonb) AS p(page_index integer,payload text) ON CONFLICT(map_id,page_index) DO NOTHING", - [map.map_id().to_vec().into(), encoded.into()])).await.map_err(db_error)?; - } - for batch in nodes.chunks(1024) { - let encoded = encode_nodes(batch)?; - txn.execute_raw(stmt(&self.schema, "INSERT INTO mst2_chunk_map_node(map_id,first_page,page_count,digest) SELECT $1,p.first_page,p.page_count,pg_catalog.decode(p.digest,'hex') FROM pg_catalog.jsonb_to_recordset($2::jsonb) AS p(first_page integer,page_count integer,digest text) ON CONFLICT(map_id,first_page,page_count) DO NOTHING", - [map.map_id().to_vec().into(), encoded.into()])).await.map_err(db_error)?; - } - } - compare_complete_map(&txn, &verified, &nodes, &self.schema).await?; - let fact = source.fact(); - txn.execute_raw(stmt(&self.schema, "INSERT INTO mst2_chunk_map_source(storage_domain,git_oid,object_kind,fact_id,source_id,source_bytes,primary_scope,map_id,receipt_digest) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9) ON CONFLICT(storage_domain,git_oid,object_kind) DO NOTHING", - [fact.storage_domain.clone().into(), fact.git_oid.clone().into(), fact.object_kind.clone().into(), fact.id.into(), - source_id.to_vec().into(), source_bytes.into(), self.primary_scope.clone().into(), map.map_id().to_vec().into(), Sha256::digest(&bytes).to_vec().into()])).await.map_err(db_error)?; - let row = source_row(&txn, source, &self.schema).await?.ok_or_else(|| integrity("installed source receipt is missing"))?; - let (stored, _) = self.validate_source_row(source, &row)?; - if stored != *map { return Err(integrity("concurrent source installation conflicts")); } + require_current_source(&txn,source,&self.schema).await?; + retention::map_barrier(&txn,map.map_id()).await?; + retention::barrier(&txn,&self.schema).await?; + retention::ensure_quotas(&txn,&self.schema,&observation,0,0,0,Some(admission.owner())).await?; + let admitted=txn.query_one_raw(stmt(&self.schema,"UPDATE mst2_chunk_receipt_generation SET state='LIVE',owner=NULL,deadline=NULL,reserved_pages=0,reserved_nodes=0,reserved_bytes=0,last_progress=pg_catalog.clock_timestamp() WHERE receipt_key=$1 AND generation=$2 AND owner=$3::uuid AND state='CREATING' AND deadline>pg_catalog.clock_timestamp() AND create_completed AND map_id=$4 AND map_generation=$5 AND receipt_bytes=$6 AND EXISTS(SELECT 1 FROM mst2_chunk_map_lifetime WHERE map_id=$4 AND generation=$5 AND state='LIVE') RETURNING generation",[key.key.clone().into(),admission.generation().into(),admission.owner().to_string().into(),map.map_id().to_vec().into(),map_generation.into(),bytes.clone().into()])).await.map_err(db_error)?; + if admitted.is_none() {return Err(SnapshotError::new(SnapshotErrorCode::LeaseExpired,"chunk-map publication lost its exact live install reservation"));} + let fact=source.fact(); + let source_bytes=source.canonical_bytes()?; + let source_id=source_id(&self.primary_scope,&source_bytes); + txn.execute_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_map_source(storage_domain,git_oid,object_kind,fact_id,source_id,source_bytes,primary_scope,map_id,receipt_digest,receipt_generation) VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10) ON CONFLICT(storage_domain,git_oid,object_kind) DO NOTHING",[fact.storage_domain.clone().into(),fact.git_oid.clone().into(),fact.object_kind.clone().into(),fact.id.into(),source_id.to_vec().into(),source_bytes.into(),self.primary_scope.clone().into(),map.map_id().to_vec().into(),Sha256::digest(&bytes).to_vec().into(),admission.generation().into()])).await.map_err(db_error)?; + let row=source_row(&txn,source,&self.schema).await?.ok_or_else(||integrity("installed source receipt is missing"))?; + let (stored,_)=self.validate_source_row(source,&row)?; + if stored!=*map || row.try_get::("","receipt_key").map_err(db_error)?!=key.key {return Err(integrity("concurrent source installation conflicts"));} Ok(()) }.await; finish(txn, result).await } - pub(crate) async fn selected_page( &self, map: &PersistedChunkMap, page_index: u64, ) -> Result, SnapshotError> { + map.ensure_live().await?; let intervals = proof_intervals(map.map.page_count, page_index)?; let memory = self.budget.reserve(PAGE_CREDIT)?; let txn = self.transaction().await?; @@ -240,7 +342,10 @@ impl PostgresChunkMapRepository { tracing::debug!(target:"mst2::chunk_map", selected_leaf_bytes = bytes.len(), selected_sibling_rows = nodes.len(), page_count = map.map.page_count, "authenticated persisted chunk map selected page"); Ok(Arc::new(AuthenticatedChunkPage { leaf, proof, _memory: memory })) }.await; - finish(txn, result).await + let page = finish(txn, result).await?; + map.ensure_live().await?; + map.record_progress().await?; + Ok(page) } async fn transaction(&self) -> Result { @@ -412,7 +517,7 @@ async fn source_row( source: &ChunkMapSource, schema: &str, ) -> Result, SnapshotError> { - connection.query_one_raw(stmt(schema, "SELECT s.fact_id,CASE WHEN pg_catalog.octet_length(s.source_id)=32 THEN s.source_id ELSE NULL END AS source_id,CASE WHEN pg_catalog.octet_length(s.source_bytes)<=2048 THEN s.source_bytes ELSE NULL END AS source_bytes,CASE WHEN pg_catalog.octet_length(s.primary_scope)<=1024 THEN s.primary_scope ELSE NULL END AS primary_scope,CASE WHEN pg_catalog.octet_length(s.map_id)=32 THEN s.map_id ELSE NULL END AS map_id,CASE WHEN pg_catalog.octet_length(s.receipt_digest)=32 THEN s.receipt_digest ELSE NULL END AS receipt_digest,CASE WHEN pg_catalog.octet_length(m.descriptor)=100 THEN m.descriptor ELSE NULL END AS descriptor,m.page_count,CASE WHEN pg_catalog.octet_length(m.pages_root)=32 THEN m.pages_root ELSE NULL END AS pages_root FROM mst2_chunk_map_source s LEFT JOIN mst2_chunk_map m ON m.map_id=s.map_id WHERE s.storage_domain=$1 AND s.git_oid=$2 AND s.object_kind=$3", + connection.query_one_raw(stmt(schema, "SELECT s.fact_id,s.receipt_key,s.receipt_generation,g.state AS generation_state,l.state AS map_state,l.generation AS map_generation,CASE WHEN pg_catalog.octet_length(g.receipt_bytes)<=4096 THEN g.receipt_bytes ELSE NULL END AS generation_receipt_bytes,CASE WHEN pg_catalog.octet_length(s.source_id)=32 THEN s.source_id ELSE NULL END AS source_id,CASE WHEN pg_catalog.octet_length(s.source_bytes)<=2048 THEN s.source_bytes ELSE NULL END AS source_bytes,CASE WHEN pg_catalog.octet_length(s.primary_scope)<=1024 THEN s.primary_scope ELSE NULL END AS primary_scope,CASE WHEN pg_catalog.octet_length(s.map_id)=32 THEN s.map_id ELSE NULL END AS map_id,CASE WHEN pg_catalog.octet_length(s.receipt_digest)=32 THEN s.receipt_digest ELSE NULL END AS receipt_digest,CASE WHEN pg_catalog.octet_length(m.descriptor)=100 THEN m.descriptor ELSE NULL END AS descriptor,m.page_count,CASE WHEN pg_catalog.octet_length(m.pages_root)=32 THEN m.pages_root ELSE NULL END AS pages_root FROM mst2_chunk_map_source s LEFT JOIN mst2_chunk_map m ON m.map_id=s.map_id LEFT JOIN mst2_chunk_receipt_generation g ON g.receipt_key=s.receipt_key AND g.generation=s.receipt_generation AND g.map_id=s.map_id LEFT JOIN mst2_chunk_map_lifetime l ON l.map_id=s.map_id AND l.generation=g.map_generation WHERE s.storage_domain=$1 AND s.git_oid=$2 AND s.object_kind=$3", [source.fact().storage_domain.clone().into(), source.fact().git_oid.clone().into(), source.fact().object_kind.clone().into()])).await.map_err(db_error) } @@ -447,13 +552,6 @@ fn receipt( Ok(bytes) } -fn receipt_key(source_id: [u8; 32]) -> ObjectKey { - ObjectKey { - namespace: ObjectNamespace::ChunkMapReceipt, - key: hex::encode(source_id), - } -} - async fn read_receipt( objects: &MegaObjectStorageWrapper, key: &ObjectKey, @@ -505,46 +603,6 @@ fn encode_nodes(nodes: &[ChunkMapNode]) -> Result { serde_json::to_string(&nodes.iter().map(|n| json!({"first_page":n.start,"page_count":n.pages,"digest":hex::encode(n.digest)})).collect::>()).map_err(db_error) } -async fn compare_complete_map( - txn: &DatabaseTransaction, - verified: &VerifiedSourceChunkMap, - nodes: &[ChunkMapNode], - schema: &str, -) -> Result<(), SnapshotError> { - let map = verified.map(); - let row = txn.query_one_raw(stmt(schema, "SELECT CASE WHEN pg_catalog.octet_length(descriptor)=100 THEN descriptor ELSE NULL END AS descriptor,page_count,CASE WHEN pg_catalog.octet_length(pages_root)=32 THEN pages_root ELSE NULL END AS pages_root,(SELECT pg_catalog.count(*) FROM mst2_chunk_map_leaf WHERE map_id=$1) AS leaves,(SELECT pg_catalog.count(*) FROM mst2_chunk_map_node WHERE map_id=$1) AS nodes FROM mst2_chunk_map WHERE map_id=$1", [map.map_id().to_vec().into()])) - .await.map_err(db_error)?.ok_or_else(|| integrity("chunk map installation descriptor is missing"))?; - if bounded_bytes(&row, "descriptor")? != map.encode() - || row.try_get::("", "page_count").map_err(db_error)? as u64 != map.page_count - || bounded_bytes(&row, "pages_root")? != map.pages_root - || row.try_get::("", "leaves").map_err(db_error)? as u64 != map.page_count - || row.try_get::("", "nodes").map_err(db_error)? as usize != nodes.len() - { - return Err(integrity( - "chunk map installation has conflicting descriptor or incomplete coverage", - )); - } - for batch in verified.leaves().chunks(64) { - let bad = txn.query_one_raw(stmt(schema, "SELECT p.page_index FROM pg_catalog.jsonb_to_recordset($2::jsonb) AS p(page_index integer,payload text) LEFT JOIN mst2_chunk_map_leaf l ON l.map_id=$1 AND l.page_index=p.page_index WHERE l.payload IS DISTINCT FROM pg_catalog.decode(p.payload,'hex') LIMIT 1", - [map.map_id().to_vec().into(), encode_leaves(batch)?.into()])).await.map_err(db_error)?; - if bad.is_some() { - return Err(integrity( - "immutable stored chunk map leaf conflicts with verified source", - )); - } - } - for batch in nodes.chunks(1024) { - let bad = txn.query_one_raw(stmt(schema, "SELECT p.first_page FROM pg_catalog.jsonb_to_recordset($2::jsonb) AS p(first_page integer,page_count integer,digest text) LEFT JOIN mst2_chunk_map_node n ON n.map_id=$1 AND n.first_page=p.first_page AND n.page_count=p.page_count WHERE n.digest IS DISTINCT FROM pg_catalog.decode(p.digest,'hex') LIMIT 1", - [map.map_id().to_vec().into(), encode_nodes(batch)?.into()])).await.map_err(db_error)?; - if bad.is_some() { - return Err(integrity( - "immutable stored chunk map node conflicts with verified source", - )); - } - } - Ok(()) -} - fn quoted(schema: &str) -> String { format!("\"{}\"", schema.replace('"', "\"\"")) } @@ -591,6 +649,11 @@ fn qualify_relations(schema: &str, sql: &str) -> String { "mst2_chunk_map_source", "mst2_chunk_map_leaf", "mst2_chunk_map_node", + "mst2_chunk_receipt_generation", + "mst2_chunk_map_lifetime", + "mst2_chunk_reader", + "mst2_chunk_map_gc", + "mst2_chunk_retention_barrier", ] .contains(&&sql[start..cursor]) { diff --git a/src/jupiter/storage/native_chunk_map/retention.rs b/src/jupiter/storage/native_chunk_map/retention.rs new file mode 100644 index 00000000..82a74da2 --- /dev/null +++ b/src/jupiter/storage/native_chunk_map/retention.rs @@ -0,0 +1,1220 @@ +//! Bounded primary owners and exact physical-generation reconciliation. + +use std::{ + sync::OnceLock, + time::{Duration, Instant}, +}; + +use tokio::sync::Mutex; +use uuid::Uuid; + +use super::*; +use crate::orbit_api::{ + error::IoOrbitError, + object_storage::{ChunkMapReceiptDeletion, ChunkMapReceiptInventory}, +}; + +const RENEW_INTERVAL: Duration = Duration::from_secs(10); +const HISTORY_CAP: i64 = 131_072; +const MAINTENANCE_MAX: usize = 64; +const OWNER_TTL: Duration = Duration::from_secs(59); + +struct OwnerDeadline(std::sync::Mutex); + +impl OwnerDeadline { + fn from_sql_start(started: Instant) -> Self { + // The SQL INSERT/renew sets clock_timestamp()+59s after this sample. + // Counting SQL and commit latency against the TTL is conservative. + Self(std::sync::Mutex::new(started + OWNER_TTL)) + } + + fn get(&self) -> Result { + let deadline = *self + .0 + .lock() + .map_err(|_| integrity("owner deadline is poisoned"))?; + if deadline <= Instant::now() { + return Err(expired("chunk-map ownership deadline elapsed")); + } + Ok(deadline) + } + + fn observe(&self, deadline: Instant, renewed: bool) -> Result<(), SnapshotError> { + let mut previous = self + .0 + .lock() + .map_err(|_| integrity("owner deadline is poisoned"))?; + *previous = if renewed { + deadline + } else { + (*previous).min(deadline) + }; + if *previous <= Instant::now() { + return Err(expired("chunk-map ownership deadline elapsed")); + } + Ok(()) + } + + async fn bound(&self, future: F) -> Result { + let deadline = self.get()?; + let result = tokio::time::timeout_at(tokio::time::Instant::from_std(deadline), future) + .await + .map_err(|_| expired("chunk-map ownership deadline elapsed"))?; + self.get()?; + Ok(result) + } +} + +fn remaining_deadline(started: Instant, row: &QueryResult) -> Result { + let remaining = row.try_get::("", "remaining_us").map_err(db_error)?; + let remaining = u64::try_from(remaining) + .ok() + .filter(|value| *value > 0) + .ok_or_else(|| expired("chunk-map database ownership deadline elapsed"))?; + started + .checked_add(Duration::from_micros(remaining)) + .ok_or_else(|| integrity("chunk-map ownership deadline is not representable")) +} + +fn retention_budget() -> &'static Arc { + // Inventory survives requests. Its separate process-wide credits bound + // independent repositories without retaining source or response credit. + static BUDGET: OnceLock> = OnceLock::new(); + BUDGET.get_or_init(|| MemoryBudget::new(128 * 1024 * 1024)) +} + +pub(super) struct BackingObservation { + inventory: ChunkMapReceiptInventory, + started_at: String, + captured: Instant, + backing: MegaObjectStorageWrapper, + _memory: MemoryLease, +} + +pub(crate) struct ChunkMapReader { + repository: PostgresChunkMapRepository, + source: ChunkMapSource, + owner: Uuid, + receipt_key: String, + generation: i64, + map_id: [u8; 32], + map_generation: i64, + confirmed: Mutex<(Instant, Instant)>, + deadline: OwnerDeadline, +} + +impl ChunkMapReader { + #[cfg(test)] + pub(crate) async fn test_check_next_owner_operation(&self) { + let before = Instant::now() - Duration::from_secs(11); + *self.confirmed.lock().await = (before, before); + } + pub(crate) async fn ensure_live(&self) -> Result<(), SnapshotError> { + self.deadline.get()?; + let mut confirmed = self.confirmed.lock().await; + self.deadline.get()?; + if confirmed.0.elapsed() < RENEW_INTERVAL { + return Ok(()); + } + let deadline = self.deadline.bound(self.check(false)).await??; + self.deadline.observe(deadline, false)?; + confirmed.0 = Instant::now(); + Ok(()) + } + + pub(super) async fn record_progress(&self) -> Result<(), SnapshotError> { + self.deadline.get()?; + let mut confirmed = self.confirmed.lock().await; + self.deadline.get()?; + if confirmed.1.elapsed() < RENEW_INTERVAL { + return Ok(()); + } + let deadline = self.deadline.bound(self.check(true)).await??; + self.deadline.observe(deadline, true)?; + *confirmed = (Instant::now(), Instant::now()); + Ok(()) + } + + pub(super) async fn validate_after_receipt(&self) -> Result<(), SnapshotError> { + let mut confirmed = self.confirmed.lock().await; + self.deadline.get()?; + let deadline = self.deadline.bound(self.check(true)).await??; + self.deadline.observe(deadline, true)?; + *confirmed = (Instant::now(), Instant::now()); + Ok(()) + } + + pub(crate) async fn await_backend( + &self, + future: F, + ) -> Result { + self.ensure_live().await?; + tokio::pin!(future); + loop { + let deadline = self.deadline.get()?; + let check_at = self.confirmed.lock().await.0 + RENEW_INTERVAL; + tokio::select! { + biased; + _ = tokio::time::sleep_until(tokio::time::Instant::from_std(deadline)) => { + return Err(expired("chunk-map reader backend wait outlived its ownership")); + } + _ = tokio::time::sleep_until(tokio::time::Instant::from_std(check_at)) => { + self.ensure_live().await?; + } + value = &mut future => { + self.deadline.get()?; + return Ok(value); + } + } + } + } + + async fn check(&self, renew: bool) -> Result { + let started = Instant::now(); + let txn = self.repository.transaction().await?; + let result = async { + self.repository.require_scope(&txn).await?; + require_current_source(&txn, &self.source, &self.repository.schema).await?; + barrier(&txn, &self.repository.schema).await?; + let row = if renew {txn.query_one_raw(stmt(&self.repository.schema, + "UPDATE mst2_chunk_reader SET deadline=pg_catalog.clock_timestamp()+interval '59 seconds',last_progress=pg_catalog.clock_timestamp() WHERE owner=$1::uuid AND receipt_key=$2 AND source_generation=$3 AND map_id=$4 AND map_generation=$5 AND deadline>pg_catalog.clock_timestamp() RETURNING owner", + [self.owner.to_string().into(),self.receipt_key.clone().into(),self.generation.into(),self.map_id.to_vec().into(),self.map_generation.into()])).await.map_err(db_error)?} else { + txn.query_one_raw(stmt(&self.repository.schema,"SELECT r.owner FROM mst2_chunk_reader r JOIN mst2_chunk_receipt_generation g ON g.receipt_key=r.receipt_key AND g.generation=r.source_generation JOIN mst2_chunk_map_lifetime l ON l.map_id=r.map_id AND l.generation=r.map_generation WHERE r.owner=$1::uuid AND r.receipt_key=$2 AND r.source_generation=$3 AND r.map_id=$4 AND r.map_generation=$5 AND r.deadline>pg_catalog.clock_timestamp() AND g.state='LIVE' AND l.state='LIVE'",[self.owner.to_string().into(),self.receipt_key.clone().into(),self.generation.into(),self.map_id.to_vec().into(),self.map_generation.into()])).await.map_err(db_error)? + }; + if row.is_none() { return Err(expired("chunk-map reader expired before progress resumed")); } + if renew {txn.execute_raw(stmt(&self.repository.schema, + "UPDATE mst2_chunk_map_lifetime SET last_used=pg_catalog.clock_timestamp() WHERE map_id=$1 AND generation=$2 AND state='LIVE'", + [self.map_id.to_vec().into(),self.map_generation.into()])).await.map_err(db_error)?; + txn.execute_raw(stmt(&self.repository.schema,"UPDATE mst2_chunk_receipt_generation SET last_progress=pg_catalog.clock_timestamp() WHERE receipt_key=$1 AND generation=$2 AND state='LIVE'",[self.receipt_key.clone().into(),self.generation.into()])).await.map_err(db_error)?; + } + let remaining = txn.query_one_raw(stmt(&self.repository.schema, + "SELECT pg_catalog.floor(EXTRACT(EPOCH FROM (deadline-pg_catalog.clock_timestamp()))*1000000)::bigint AS remaining_us FROM mst2_chunk_reader WHERE owner=$1::uuid", + [self.owner.to_string().into()])).await.map_err(db_error)? + .ok_or_else(|| expired("chunk-map reader deadline disappeared"))?; + remaining_deadline(started, &remaining) + }.await; + finish(txn, result).await + } +} + +impl Drop for ChunkMapReader { + fn drop(&mut self) { + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + return; + }; + let repository = self.repository.clone(); + let owner = self.owner; + runtime.spawn(async move { + let result = async { + let txn = repository.transaction().await?; + let result = async { + repository.require_scope(&txn).await?; + barrier(&txn, &repository.schema).await?; + txn.execute_raw(stmt( + &repository.schema, + "DELETE FROM mst2_chunk_reader WHERE owner=$1::uuid", + [owner.to_string().into()], + )) + .await + .map_err(db_error)?; + Ok(()) + } + .await; + finish(txn, result).await + } + .await; + if let Err(error) = result { + tracing::warn!( + ?error, + "chunk-map reader release will expire by database deadline" + ); + } + }); + } +} + +/// A capacity reservation exists before the verifier opens the source body. +/// It cannot publish source trust; installation still consumes the opaque +/// complete-stream verifier and independently checks the backend receipt. +pub(crate) struct ChunkMapInstall { + repository: PostgresChunkMapRepository, + source: ChunkMapSource, + owner: Uuid, + key: ObjectKey, + generation: i64, + progress: Mutex<(Instant, Instant, u64)>, + deadline: OwnerDeadline, +} + +#[derive(Clone)] +pub(super) struct CancelledInstall { + owner: Uuid, + key: String, + generation: i64, +} + +impl ChunkMapInstall { + #[cfg(test)] + pub(crate) async fn test_check_next_owner_operation(&self) { + let mut progress = self.progress.lock().await; + let before = Instant::now() - Duration::from_secs(11); + progress.0 = before; + } + pub(crate) async fn ensure_live(&self) -> Result<(), SnapshotError> { + self.renew(false, 0).await + } + + pub(crate) async fn record_progress(&self, completed: u64) -> Result<(), SnapshotError> { + self.renew(true, completed).await + } + + pub(crate) async fn await_backend( + &self, + future: F, + ) -> Result { + self.ensure_live().await?; + tokio::pin!(future); + loop { + let deadline = self.deadline.get()?; + let check_at = self.progress.lock().await.0 + RENEW_INTERVAL; + tokio::select! { + biased; + _ = tokio::time::sleep_until(tokio::time::Instant::from_std(deadline)) => { + return Err(expired("chunk-map install backend wait outlived its ownership")); + } + _ = tokio::time::sleep_until(tokio::time::Instant::from_std(check_at)) => { + self.ensure_live().await?; + } + value = &mut future => { + self.deadline.get()?; + return Ok(value); + } + } + } + } + + async fn renew(&self, progress: bool, completed: u64) -> Result<(), SnapshotError> { + self.deadline.get()?; + let mut confirmed = self.progress.lock().await; + self.deadline.get()?; + if progress && completed <= confirmed.2 { + return Ok(()); + } + if (if progress { + confirmed.1.elapsed() + } else { + confirmed.0.elapsed() + }) < RENEW_INTERVAL + { + if progress { + confirmed.2 = completed; + } + return Ok(()); + } + // Calling ensure_live after a long stalled await checks the old + // deadline without extending it. Only new source progress renews it. + let started = Instant::now(); + let deadline = self.deadline.bound(async { + let txn = self.repository.transaction().await?; + let result=async { + self.repository.require_scope(&txn).await?; + require_current_source(&txn,&self.source,&self.repository.schema).await?; + barrier(&txn,&self.repository.schema).await?; + let row=if progress { + txn.query_one_raw(stmt(&self.repository.schema,"UPDATE mst2_chunk_receipt_generation SET deadline=pg_catalog.clock_timestamp()+interval '59 seconds',last_progress=pg_catalog.clock_timestamp() WHERE receipt_key=$1 AND generation=$2 AND owner=$3::uuid AND state IN ('RESERVED','CREATING') AND deadline>pg_catalog.clock_timestamp() RETURNING generation",[self.key.key.clone().into(),self.generation.into(),self.owner.to_string().into()])).await.map_err(db_error)? + } else { + txn.query_one_raw(stmt(&self.repository.schema,"SELECT generation FROM mst2_chunk_receipt_generation WHERE receipt_key=$1 AND generation=$2 AND owner=$3::uuid AND state IN ('RESERVED','CREATING') AND deadline>pg_catalog.clock_timestamp()",[self.key.key.clone().into(),self.generation.into(),self.owner.to_string().into()])).await.map_err(db_error)? + }; + if row.is_none() { return Err(expired("chunk-map install reservation expired or was retired")); } + let remaining = txn.query_one_raw(stmt(&self.repository.schema, + "SELECT pg_catalog.floor(EXTRACT(EPOCH FROM (deadline-pg_catalog.clock_timestamp()))*1000000)::bigint AS remaining_us FROM mst2_chunk_receipt_generation WHERE receipt_key=$1 AND generation=$2 AND owner=$3::uuid AND state IN ('RESERVED','CREATING')", + [self.key.key.clone().into(),self.generation.into(),self.owner.to_string().into()])).await.map_err(db_error)? + .ok_or_else(|| expired("chunk-map install deadline disappeared"))?; + remaining_deadline(started, &remaining) + }.await; + finish(txn, result).await + }).await??; + self.deadline.observe(deadline, progress)?; + if progress { + *confirmed = (Instant::now(), Instant::now(), completed); + } else { + confirmed.0 = Instant::now(); + } + Ok(()) + } + + pub(super) async fn prepare( + &self, + verified: &VerifiedSourceChunkMap, + ) -> Result, SnapshotError> { + if verified.source() != &self.source { + return Err(integrity( + "verified source differs from install reservation", + )); + } + let bytes = receipt(&self.repository.primary_scope, &self.source, verified.map())?; + let started = Instant::now(); + self.deadline.bound(async { + let txn = self.repository.transaction().await?; + let result=async { + self.repository.require_scope(&txn).await?; + require_current_source(&txn,&self.source,&self.repository.schema).await?; + barrier(&txn,&self.repository.schema).await?; + let row=txn.query_one_raw(stmt(&self.repository.schema,"UPDATE mst2_chunk_receipt_generation SET state='CREATING',map_id=$4,receipt_bytes=$5,deadline=pg_catalog.clock_timestamp()+interval '59 seconds',last_progress=pg_catalog.clock_timestamp() WHERE receipt_key=$1 AND generation=$2 AND owner=$3::uuid AND state='RESERVED' AND deadline>pg_catalog.clock_timestamp() RETURNING generation",[self.key.key.clone().into(),self.generation.into(),self.owner.to_string().into(),verified.map().map_id().to_vec().into(),bytes.clone().into()])).await.map_err(db_error)?; + if row.is_none() { return Err(expired("chunk-map install lost its exact reservation before receipt creation")); } + Ok(()) + }.await; + finish(txn, result).await?; + Ok::<_, SnapshotError>(()) + }).await??; + self.deadline.observe(started + OWNER_TTL, true)?; + Ok(bytes) + } + + pub(super) fn key(&self) -> &ObjectKey { + &self.key + } + pub(super) fn generation(&self) -> i64 { + self.generation + } + pub(super) fn owner(&self) -> Uuid { + self.owner + } + + pub(super) async fn acknowledge_create(&self) -> Result<(), SnapshotError> { + // This method is called only after the actual atomic create future + // returns success. Absence, a checksum or inventory cannot call it. + let started = Instant::now(); + let renewed = self.deadline.bound(async { + let txn = self.repository.transaction().await?; + let result=async { + self.repository.require_scope(&txn).await?; + barrier(&txn,&self.repository.schema).await?; + let row=txn.query_one_raw(stmt(&self.repository.schema,"UPDATE mst2_chunk_receipt_generation SET create_completed=true,create_completed_at=COALESCE(create_completed_at,pg_catalog.clock_timestamp()),deadline=CASE WHEN state='CREATING' AND owner=$3::uuid AND deadline>pg_catalog.clock_timestamp() THEN pg_catalog.clock_timestamp()+interval '59 seconds' ELSE deadline END WHERE receipt_key=$1 AND generation=$2 AND receipt_bytes IS NOT NULL RETURNING generation,CASE WHEN state='CREATING' AND owner=$3::uuid AND deadline>pg_catalog.clock_timestamp() THEN true ELSE false END AS deadline_renewed",[self.key.key.clone().into(),self.generation.into(),self.owner.to_string().into()])).await.map_err(db_error)? + .ok_or_else(|| integrity("completed create lost its persistent generation fence"))?; + row.try_get::("", "deadline_renewed").map_err(db_error) + }.await; + finish(txn, result).await + }).await??; + if renewed { + self.deadline.observe(started + OWNER_TTL, true)?; + } + Ok(()) + } + + pub(super) async fn retire(&self) -> Result<(), SnapshotError> { + retire_install(&self.repository, self.owner, &self.key.key, self.generation).await + } +} + +impl Drop for ChunkMapInstall { + fn drop(&mut self) { + if let Ok(mut cancelled) = self.repository.cancelled_installs.lock() { + if cancelled.len() < 64 { + cancelled.push(CancelledInstall { + owner: self.owner, + key: self.key.key.clone(), + generation: self.generation, + }); + } + } + let Ok(runtime) = tokio::runtime::Handle::try_current() else { + return; + }; + let repository = self.repository.clone(); + let owner = self.owner; + let key = self.key.key.clone(); + let generation = self.generation; + runtime.spawn(async move { + let result = retire_install(&repository, owner, &key, generation).await; + if result.is_ok() { + forget_cancelled_install(&repository, owner); + } + if let Err(error) = result { + tracing::warn!( + ?error, + "chunk-map install cancellation will reconcile by database deadline" + ); + } + }); + } +} + +fn forget_cancelled_install(repository: &PostgresChunkMapRepository, owner: Uuid) { + if let Ok(mut cancelled) = repository.cancelled_installs.lock() { + cancelled.retain(|install| install.owner != owner); + } +} + +async fn retire_install( + repository: &PostgresChunkMapRepository, + owner: Uuid, + key: &str, + generation: i64, +) -> Result<(), SnapshotError> { + let txn = repository.transaction().await?; + let result = async { + repository.require_scope(&txn).await?; + barrier(&txn, &repository.schema).await?; + txn.execute_raw(stmt(&repository.schema,"UPDATE mst2_chunk_receipt_generation SET state=CASE WHEN receipt_bytes IS NULL THEN 'APPLIED' ELSE 'DELETING' END,owner=NULL,deadline=NULL,reserved_pages=0,reserved_nodes=0,reserved_bytes=0 WHERE receipt_key=$1 AND generation=$2 AND owner=$3::uuid AND state IN ('RESERVED','CREATING')",[key.into(),generation.into(),owner.to_string().into()])).await.map_err(db_error)?; + Ok(()) + }.await; + finish(txn, result).await +} + +/// Constructor stays in the collector module. Its private fields carry a +/// real primary claim and the independently read exact backend body. +pub(crate) struct VerifiedReceiptDeletionClaim { + key: ObjectKey, + expected: Vec, + _generation: i64, +} +impl VerifiedReceiptDeletionClaim { + pub(crate) fn key(&self) -> &ObjectKey { + &self.key + } + pub(crate) fn expected_bytes(&self) -> &[u8] { + &self.expected + } +} + +impl PostgresChunkMapRepository { + async fn replay_cancelled_installs(&self) -> Result<(), SnapshotError> { + let cancelled = self + .cancelled_installs + .lock() + .map_err(|_| integrity("cancelled install replay ownership is poisoned"))? + .clone(); + for install in cancelled { + retire_install(self, install.owner, &install.key, install.generation).await?; + forget_cancelled_install(self, install.owner); + } + Ok(()) + } + + async fn maintain_before_admission( + &self, + objects: &MegaObjectStorageWrapper, + ) -> Result<(), SnapshotError> { + let mut completed = self.admission_maintenance.lock().await; + self.replay_cancelled_installs().await?; + if completed + .as_ref() + .is_some_and(|last| last.elapsed() < RENEW_INTERVAL) + { + let txn = self.transaction().await?; + self.require_scope(&txn).await?; + let pending=txn.query_one_raw(stmt(&self.schema,"SELECT EXISTS(SELECT 1 FROM mst2_chunk_receipt_generation WHERE state='DELETING' OR (state IN ('RESERVED','CREATING') AND deadline<=pg_catalog.clock_timestamp())) AS pending",[])).await.map_err(db_error)?.ok_or_else(||integrity("pending chunk-map admission replay observation missing"))?.try_get::("","pending").map_err(db_error)?; + txn.commit().await.map_err(db_error)?; + if !pending { + return Ok(()); + } + } + self.maintain(objects, 8).await?; + *completed = Some(Instant::now()); + Ok(()) + } + #[cfg(test)] + pub(crate) async fn test_new_install_capacity( + &self, + objects: &MegaObjectStorageWrapper, + ) -> Result<(), SnapshotError> { + let observation = inventory(self, objects).await?; + let txn = self.transaction().await?; + let result = async { + self.require_scope(&txn).await?; + barrier(&txn, &self.schema).await?; + ensure_quotas(&txn, &self.schema, &observation, 1, 1, 1024 * 1024, None).await + } + .await; + finish(txn, result).await + } + pub(super) async fn stage_map_start( + &self, + verified: &VerifiedSourceChunkMap, + admission: &ChunkMapInstall, + observation: &BackingObservation, + ) -> Result { + let map = verified.map(); + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + require_current_source(&txn,verified.source(),&self.schema).await?; + map_barrier(&txn,map.map_id()).await?; + barrier(&txn,&self.schema).await?; + let inserted=txn.query_one_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_map(map_id,descriptor,page_count,pages_root) VALUES($1,$2,$3,$4) ON CONFLICT(map_id) DO NOTHING RETURNING map_id",[map.map_id().to_vec().into(),map.encode().into(),(map.page_count as i32).into(),map.pages_root.to_vec().into()])).await.map_err(db_error)?.is_some(); + let lifetime=txn.query_one_raw(stmt(&self.schema,"SELECT generation,state FROM mst2_chunk_map_lifetime WHERE map_id=$1",[map.map_id().to_vec().into()])).await.map_err(db_error)?; + let generation=if let Some(row)=lifetime { + if row.try_get::("","state").map_err(db_error)?!="LIVE" {return Err(unavailable("shared chunk map collection is pending"));} + row.try_get::("","generation").map_err(db_error)? + } else if inserted { + let generation=next_generation(&txn,&self.schema).await?; + txn.execute_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_map_lifetime(map_id,generation,state) VALUES($1,$2,'LIVE')",[map.map_id().to_vec().into(),generation.into()])).await.map_err(db_error)?; + generation + } else {return Err(integrity("existing chunk map has no exact lifetime; never repair it"));}; + let row=txn.query_one_raw(stmt(&self.schema,"SELECT CASE WHEN pg_catalog.octet_length(descriptor)=100 THEN descriptor ELSE NULL END AS descriptor,page_count,CASE WHEN pg_catalog.octet_length(pages_root)=32 THEN pages_root ELSE NULL END AS pages_root FROM mst2_chunk_map WHERE map_id=$1",[map.map_id().to_vec().into()])).await.map_err(db_error)?.ok_or_else(||integrity("staged map descriptor missing"))?; + if bounded_bytes(&row,"descriptor")?!=map.encode() || row.try_get::("","page_count").map_err(db_error)? as u64!=map.page_count || bounded_bytes(&row,"pages_root")?!=map.pages_root {return Err(integrity("immutable staged map descriptor conflicts with verified source"));} + let assigned=txn.query_one_raw(stmt(&self.schema,"UPDATE mst2_chunk_receipt_generation SET map_generation=$4,deadline=pg_catalog.clock_timestamp()+interval '59 seconds',last_progress=pg_catalog.clock_timestamp() WHERE receipt_key=$1 AND generation=$2 AND owner=$3::uuid AND state='CREATING' AND deadline>pg_catalog.clock_timestamp() AND map_id=$5 AND create_completed RETURNING generation",[admission.key.key.clone().into(),admission.generation.into(),admission.owner.to_string().into(),generation.into(),map.map_id().to_vec().into()])).await.map_err(db_error)?; + if assigned.is_none() {return Err(expired("map staging lost the exact completed-create reservation"));} + ensure_quotas(&txn,&self.schema,observation,0,0,0,Some(admission.owner)).await?; + Ok(generation) + }.await; + finish(txn, result).await + } + + pub(super) async fn stage_map_batch( + &self, + verified: &VerifiedSourceChunkMap, + admission: &ChunkMapInstall, + generation: i64, + leaves: Option<&[ChunkLeaf]>, + nodes: Option<&[ChunkMapNode]>, + ) -> Result<(), SnapshotError> { + let map_id = verified.map().map_id(); + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + require_current_source(&txn,verified.source(),&self.schema).await?; + map_barrier(&txn,map_id).await?; + let sealed=txn.query_one_raw(stmt(&self.schema,"SELECT EXISTS(SELECT 1 FROM mst2_chunk_map_source WHERE map_id=$1) AS sealed",[map_id.to_vec().into()])).await.map_err(db_error)?.ok_or_else(||integrity("staged map seal observation missing"))?.try_get::("","sealed").map_err(db_error)?; + if !sealed { + if let Some(batch)=leaves { + txn.execute_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_map_leaf(map_id,page_index,payload) SELECT $1,p.page_index,pg_catalog.decode(p.payload,'hex') FROM pg_catalog.jsonb_to_recordset($2::jsonb) AS p(page_index integer,payload text) ON CONFLICT(map_id,page_index) DO NOTHING",[map_id.to_vec().into(),encode_leaves(batch)?.into()])).await.map_err(db_error)?; + } + if let Some(batch)=nodes { + txn.execute_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_map_node(map_id,first_page,page_count,digest) SELECT $1,p.first_page,p.page_count,pg_catalog.decode(p.digest,'hex') FROM pg_catalog.jsonb_to_recordset($2::jsonb) AS p(first_page integer,page_count integer,digest text) ON CONFLICT(map_id,first_page,page_count) DO NOTHING",[map_id.to_vec().into(),encode_nodes(batch)?.into()])).await.map_err(db_error)?; + } + } + self.checkpoint_staging(&txn,admission,map_id,generation).await + }.await; + finish(txn, result).await + } + + async fn checkpoint_staging( + &self, + txn: &DatabaseTransaction, + admission: &ChunkMapInstall, + map_id: [u8; 32], + generation: i64, + ) -> Result<(), SnapshotError> { + barrier(txn, &self.schema).await?; + let progress=txn.query_one_raw(stmt(&self.schema,"UPDATE mst2_chunk_receipt_generation SET deadline=pg_catalog.clock_timestamp()+interval '59 seconds',last_progress=pg_catalog.clock_timestamp() WHERE receipt_key=$1 AND generation=$2 AND owner=$3::uuid AND state='CREATING' AND deadline>pg_catalog.clock_timestamp() AND map_id=$4 AND map_generation=$5 AND EXISTS(SELECT 1 FROM mst2_chunk_map_lifetime WHERE map_id=$4 AND generation=$5 AND state='LIVE') RETURNING generation",[admission.key.key.clone().into(),admission.generation.into(),admission.owner.to_string().into(),map_id.to_vec().into(),generation.into()])).await.map_err(db_error)?; + if progress.is_none() { + return Err(expired( + "bounded map publication batch lost its exact install generation", + )); + } + Ok(()) + } + + pub(super) async fn compare_staged_map( + &self, + verified: &VerifiedSourceChunkMap, + nodes: &[ChunkMapNode], + admission: &ChunkMapInstall, + generation: i64, + ) -> Result<(), SnapshotError> { + let map = verified.map(); + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + require_current_source(&txn,verified.source(),&self.schema).await?; + map_barrier(&txn,map.map_id()).await?; + let row=txn.query_one_raw(stmt(&self.schema,"SELECT CASE WHEN pg_catalog.octet_length(descriptor)=100 THEN descriptor ELSE NULL END AS descriptor,page_count,CASE WHEN pg_catalog.octet_length(pages_root)=32 THEN pages_root ELSE NULL END AS pages_root,(SELECT pg_catalog.count(*) FROM mst2_chunk_map_leaf WHERE map_id=$1) AS leaves,(SELECT pg_catalog.count(*) FROM mst2_chunk_map_node WHERE map_id=$1) AS nodes FROM mst2_chunk_map WHERE map_id=$1",[map.map_id().to_vec().into()])).await.map_err(db_error)?.ok_or_else(||integrity("staged map descriptor missing"))?; + if bounded_bytes(&row,"descriptor")?!=map.encode() || row.try_get::("","page_count").map_err(db_error)? as u64!=map.page_count || bounded_bytes(&row,"pages_root")?!=map.pages_root || row.try_get::("","leaves").map_err(db_error)? as u64!=map.page_count || row.try_get::("","nodes").map_err(db_error)? as usize!=nodes.len() {return Err(integrity("staged map has conflicting descriptor or incomplete coverage"));} + self.checkpoint_staging(&txn,admission,map.map_id(),generation).await + }.await; + finish(txn, result).await?; + for batch in verified.leaves().chunks(64) { + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + require_current_source(&txn,verified.source(),&self.schema).await?; + map_barrier(&txn,map.map_id()).await?; + let bad=txn.query_one_raw(stmt(&self.schema,"SELECT p.page_index FROM pg_catalog.jsonb_to_recordset($2::jsonb) AS p(page_index integer,payload text) LEFT JOIN mst2_chunk_map_leaf l ON l.map_id=$1 AND l.page_index=p.page_index WHERE l.payload IS DISTINCT FROM pg_catalog.decode(p.payload,'hex') LIMIT 1",[map.map_id().to_vec().into(),encode_leaves(batch)?.into()])).await.map_err(db_error)?; + if bad.is_some() {return Err(integrity("immutable stored chunk map leaf conflicts with verified source"));} + self.checkpoint_staging(&txn,admission,map.map_id(),generation).await + }.await; + finish(txn, result).await?; + } + for batch in nodes.chunks(1024) { + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + require_current_source(&txn,verified.source(),&self.schema).await?; + map_barrier(&txn,map.map_id()).await?; + let bad=txn.query_one_raw(stmt(&self.schema,"SELECT p.first_page FROM pg_catalog.jsonb_to_recordset($2::jsonb) AS p(first_page integer,page_count integer,digest text) LEFT JOIN mst2_chunk_map_node n ON n.map_id=$1 AND n.first_page=p.first_page AND n.page_count=p.page_count WHERE n.digest IS DISTINCT FROM pg_catalog.decode(p.digest,'hex') LIMIT 1",[map.map_id().to_vec().into(),encode_nodes(batch)?.into()])).await.map_err(db_error)?; + if bad.is_some() {return Err(integrity("immutable stored chunk map node conflicts with verified source"));} + self.checkpoint_staging(&txn,admission,map.map_id(),generation).await + }.await; + finish(txn, result).await?; + } + Ok(()) + } + + pub(crate) async fn maintain( + &self, + objects: &MegaObjectStorageWrapper, + limit: usize, + ) -> Result<(), SnapshotError> { + if !(1..=MAINTENANCE_MAX).contains(&limit) { + return Err(SnapshotError::new( + SnapshotErrorCode::InvalidRequest, + "chunk-map maintenance limit must be 1..64", + )); + } + // Unsupported enumeration must fail before changing any resident + // generation. Overquota stores may still replay known deletions. + match inventory(self, objects).await { + Ok(_) => {} + Err(error) if error.code == SnapshotErrorCode::LimitExceeded => {} + Err(error) => return Err(error), + } + self.replay_cancelled_installs().await?; + let _memory = retention_budget().reserve(8 * 1024 * 1024)?; + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + barrier(&txn,&self.schema).await?; + txn.execute_raw(stmt(&self.schema,"DELETE FROM mst2_chunk_reader WHERE owner IN (SELECT owner FROM mst2_chunk_reader WHERE deadline<=pg_catalog.clock_timestamp() ORDER BY deadline,owner LIMIT $1)",[(limit as i64).into()])).await.map_err(db_error)?; + txn.execute_raw(stmt(&self.schema,"UPDATE mst2_chunk_receipt_generation SET state=CASE WHEN receipt_bytes IS NULL THEN 'APPLIED' ELSE 'DELETING' END,owner=NULL,deadline=NULL,reserved_pages=0,reserved_nodes=0,reserved_bytes=0 WHERE receipt_key IN (SELECT receipt_key FROM mst2_chunk_receipt_generation WHERE state IN ('RESERVED','CREATING') AND deadline<=pg_catalog.clock_timestamp() ORDER BY deadline,receipt_key LIMIT $1)",[(limit as i64).into()])).await.map_err(db_error)?; + // Replay unfinished receipt claims before admitting new victims. + let pending=txn.query_one_raw(stmt(&self.schema,"SELECT EXISTS(SELECT 1 FROM mst2_chunk_receipt_generation WHERE state='DELETING') AS pending",[])).await.map_err(db_error)?.ok_or_else(||integrity("chunk receipt replay observation missing"))?.try_get::("","pending").map_err(db_error)?; + if !pending { + txn.execute_raw(stmt(&self.schema,"UPDATE mst2_chunk_receipt_generation SET state='DELETING' WHERE receipt_key IN (SELECT g.receipt_key FROM mst2_chunk_receipt_generation g WHERE g.state='LIVE' AND g.last_progress<=pg_catalog.clock_timestamp()-interval '1 hour' AND NOT EXISTS(SELECT 1 FROM mst2_chunk_reader r WHERE r.receipt_key=g.receipt_key AND r.source_generation=g.generation AND r.deadline>pg_catalog.clock_timestamp()) ORDER BY g.last_progress,g.receipt_key LIMIT $1)",[(limit as i64).into()])).await.map_err(db_error)?; + } + // Rotate APPLIED claims too: absence is never a promise that a + // cancelled remote create cannot restore this retired key later. + txn.query_all_raw(stmt(&self.schema,"SELECT receipt_key,generation,CASE WHEN pg_catalog.octet_length(receipt_bytes)<=4096 THEN receipt_bytes ELSE NULL END AS receipt_bytes,CASE WHEN pg_catalog.octet_length(primary_scope)<=1024 THEN primary_scope ELSE NULL END AS primary_scope,map_id,map_generation,state FROM mst2_chunk_receipt_generation WHERE state IN ('DELETING','APPLIED') ORDER BY (state='DELETING') DESC,checked_at,receipt_key LIMIT $1",[(limit as i64).into()])).await.map_err(db_error) + }.await; + let claims = finish(txn, result).await?; + for row in claims { + self.reconcile_receipt(objects, row).await?; + } + self.collect_maps(limit).await?; + let observation = inventory(self, objects).await?; + let requested = serde_json::to_string( + &observation + .inventory + .objects + .iter() + .map(|(key, _)| &key.key) + .collect::>(), + ) + .map_err(db_error)?; + let txn = self.transaction().await?; + self.require_scope(&txn).await?; + let unknown=txn.query_all_raw(stmt(&self.schema,"SELECT p.key FROM pg_catalog.jsonb_array_elements_text($1::jsonb) AS p(key) LEFT JOIN mst2_chunk_receipt_generation g ON g.receipt_key=p.key WHERE g.receipt_key IS NULL ORDER BY p.key LIMIT $2",[requested.into(),(limit as i64).into()])).await.map_err(db_error)?; + txn.commit().await.map_err(db_error)?; + for row in unknown { + let key = ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: row.try_get("", "key").map_err(db_error)?, + }; + self.claim_orphan(objects, &key).await?; + } + tracing::debug!(target:"mst2::chunk_map",maintenance_limit=limit,actual_backing_receipts=observation.inventory.objects.len(),actual_backing_bytes=observation.inventory.bytes,"bounded chunk-map retention tick"); + Ok(()) + } + + async fn reconcile_receipt( + &self, + objects: &MegaObjectStorageWrapper, + row: QueryResult, + ) -> Result<(), SnapshotError> { + if bounded_bytes(&row, "primary_scope")? != self.primary_scope { + return Err(integrity( + "retired receipt belongs to a different primary scope", + )); + } + let key = ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: row.try_get("", "receipt_key").map_err(db_error)?, + }; + let generation: i64 = row.try_get("", "generation").map_err(db_error)?; + let expected: Option> = row.try_get("", "receipt_bytes").map_err(db_error)?; + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + barrier(&txn,&self.schema).await?; + txn.query_one_raw(stmt(&self.schema,"SELECT generation FROM mst2_chunk_receipt_generation WHERE receipt_key=$1 AND generation=$2 AND receipt_bytes IS NOT DISTINCT FROM $3::bytea AND primary_scope=$4 AND state IN ('DELETING','APPLIED') AND NOT EXISTS(SELECT 1 FROM mst2_chunk_reader WHERE receipt_key=$1 AND source_generation=$2 AND deadline>pg_catalog.clock_timestamp())",[key.key.clone().into(),generation.into(),expected.clone().into(),self.primary_scope.clone().into()])).await.map_err(db_error) + }.await; + if finish(txn, result).await?.is_none() { + return Ok(()); + } + if let Some(bytes) = expected.as_ref() { + let (scope, _, map, physical_generation) = decode_orphan(&key, bytes)?; + let stored_map: Option> = row.try_get("", "map_id").map_err(db_error)?; + if scope != self.primary_scope + || physical_generation != generation + || stored_map.as_deref() != Some(map.map_id().as_slice()) + { + return Err(integrity( + "receipt deletion claim disagrees with the independent physical generation", + )); + } + } + if let Some(expected) = expected { + let present = tokio::time::timeout(Duration::from_secs(5), objects.inner.exists(&key)) + .await + .map_err(|_| unavailable("retired receipt existence check timed out"))? + .map_err(storage_error)?; + if present { + read_receipt(objects, &key, &expected).await?; + let authority = ChunkMapReceiptDeletion::from_claim(VerifiedReceiptDeletionClaim { + key: key.clone(), + expected, + _generation: generation, + }); + tokio::time::timeout( + Duration::from_secs(5), + objects.inner.delete_chunk_map_receipt(&authority), + ) + .await + .map_err(|_| unavailable("retired receipt deletion timed out"))? + .map_err(storage_error)?; + } + } else if tokio::time::timeout(Duration::from_secs(5), objects.inner.exists(&key)) + .await + .map_err(|_| unavailable("unused receipt check timed out"))? + .map_err(storage_error)? + { + return Err(integrity( + "receipt exists for a generation that never prepared a create", + )); + } + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + barrier(&txn,&self.schema).await?; + let applied=txn.query_one_raw(stmt(&self.schema,"UPDATE mst2_chunk_receipt_generation SET state='APPLIED',owner=NULL,deadline=NULL,reserved_pages=0,reserved_nodes=0,reserved_bytes=0,checked_at=pg_catalog.clock_timestamp() WHERE receipt_key=$1 AND generation=$2 AND state IN ('DELETING','APPLIED') AND NOT EXISTS(SELECT 1 FROM mst2_chunk_reader r WHERE r.receipt_key=$1 AND r.source_generation=$2 AND r.deadline>pg_catalog.clock_timestamp()) RETURNING generation",[key.key.clone().into(),generation.into()])).await.map_err(db_error)?; + if applied.is_none() { return Err(integrity("retired receipt lost its exact reader-free completion claim")); } + txn.execute_raw(stmt(&self.schema,"DELETE FROM mst2_chunk_map_source WHERE receipt_key=$1 AND receipt_generation=$2",[key.key.into(),generation.into()])).await.map_err(db_error)?; + Ok(()) + }.await; + finish(txn, result).await + } + + async fn claim_orphan( + &self, + objects: &MegaObjectStorageWrapper, + key: &ObjectKey, + ) -> Result<(), SnapshotError> { + let actual = read_bounded_orphan(objects, key).await?; + let Some(actual) = actual else { return Ok(()) }; + let (scope, source_bytes, map, generation) = decode_orphan(key, &actual)?; + if scope != self.primary_scope { + return Err(integrity( + "orphan receipt belongs to a different captured primary", + )); + } + let identity = source_id(&scope, &source_bytes); + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + barrier(&txn,&self.schema).await?; + let count=txn.query_one_raw(stmt(&self.schema,"SELECT pg_catalog.count(*) AS history FROM mst2_chunk_receipt_generation",[])).await.map_err(db_error)?.ok_or_else(||integrity("orphan receipt history count missing"))?.try_get::("","history").map_err(db_error)?; + if count>=HISTORY_CAP { return Err(SnapshotError::new(SnapshotErrorCode::LimitExceeded,"chunk receipt reconciliation history quota exhausted")); } + // An orphan never admits a source or map. Its actual body only + // identifies a retired physical key for collection. + txn.execute_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_receipt_generation(receipt_key,source_id,generation,source_bytes,primary_scope,map_id,receipt_bytes,state,reserved_pages,reserved_nodes,reserved_bytes) VALUES($1,$2,$3,$4,$5,$6,$7,'DELETING',0,0,0) ON CONFLICT(receipt_key) DO NOTHING",[key.key.clone().into(),identity.to_vec().into(),generation.into(),source_bytes.into(),scope.into(),map.map_id().to_vec().into(),actual.into()])).await.map_err(db_error)?; + Ok(()) + }.await; + finish(txn, result).await + } + + pub(super) async fn collect_maps(&self, limit: usize) -> Result<(), SnapshotError> { + let txn = self.transaction().await?; + self.require_scope(&txn).await?; + let rows=txn.query_all_raw(stmt(&self.schema,"SELECT l.map_id,l.generation,l.state FROM mst2_chunk_map_lifetime l WHERE l.state='DELETING' OR (NOT EXISTS(SELECT 1 FROM mst2_chunk_map_lifetime WHERE state='DELETING') AND (l.last_used<=pg_catalog.clock_timestamp()-interval '1 hour' OR EXISTS(SELECT 1 FROM mst2_chunk_receipt_generation g WHERE g.map_id=l.map_id AND g.map_generation=l.generation AND g.state IN ('DELETING','APPLIED'))) AND NOT EXISTS(SELECT 1 FROM mst2_chunk_map_source s WHERE s.map_id=l.map_id) AND NOT EXISTS(SELECT 1 FROM mst2_chunk_receipt_generation g WHERE g.map_id=l.map_id AND g.map_generation=l.generation AND g.state='CREATING' AND g.deadline>pg_catalog.clock_timestamp())) ORDER BY (l.state='DELETING') DESC,l.last_used,l.map_id LIMIT $1",[(limit as i64).into()])).await.map_err(db_error)?; + txn.commit().await.map_err(db_error)?; + for row in rows { + let map_id: Vec = row.try_get("", "map_id").map_err(db_error)?; + let generation: i64 = row.try_get("", "generation").map_err(db_error)?; + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + let acquired=txn.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres,"SELECT pg_catalog.pg_try_advisory_xact_lock(1296717364,pg_catalog.hashtext($1)) AS acquired",[hex::encode(&map_id).into()])).await.map_err(db_error)?.ok_or_else(||integrity("map collection lock observation missing"))?.try_get::("","acquired").map_err(db_error)?; + if !acquired { return Ok(()); } + barrier(&txn,&self.schema).await?; + let eligible=txn.query_one_raw(stmt(&self.schema,"SELECT state FROM mst2_chunk_map_lifetime l WHERE map_id=$1 AND generation=$2 AND (state='DELETING' OR last_used<=pg_catalog.clock_timestamp()-interval '1 hour' OR EXISTS(SELECT 1 FROM mst2_chunk_receipt_generation g WHERE g.map_id=l.map_id AND g.map_generation=l.generation AND g.state IN ('DELETING','APPLIED'))) AND NOT EXISTS(SELECT 1 FROM mst2_chunk_map_source s WHERE s.map_id=$1) AND NOT EXISTS(SELECT 1 FROM mst2_chunk_reader r WHERE r.map_id=$1 AND r.map_generation=$2 AND r.deadline>pg_catalog.clock_timestamp()) AND NOT EXISTS(SELECT 1 FROM mst2_chunk_receipt_generation g WHERE g.map_id=$1 AND g.map_generation=$2 AND g.state='CREATING' AND g.deadline>pg_catalog.clock_timestamp())",[map_id.clone().into(),generation.into()])).await.map_err(db_error)?; + let Some(eligible)=eligible else { return Ok(()) }; + if eligible.try_get::("","state").map_err(db_error)?=="LIVE" { + let count=txn.query_one_raw(stmt(&self.schema,"SELECT pg_catalog.count(*) AS history FROM mst2_chunk_map_gc",[])).await.map_err(db_error)?.ok_or_else(||integrity("map collection history count missing"))?.try_get::("","history").map_err(db_error)?; + if count>=HISTORY_CAP { return Err(SnapshotError::new(SnapshotErrorCode::LimitExceeded,"chunk map collection history quota exhausted")); } + txn.execute_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_map_gc(map_id,generation,state) VALUES($1,$2,'PENDING') ON CONFLICT(map_id,generation) DO NOTHING",[map_id.clone().into(),generation.into()])).await.map_err(db_error)?; + txn.execute_raw(stmt(&self.schema,"UPDATE mst2_chunk_map_lifetime SET state='DELETING' WHERE map_id=$1 AND generation=$2 AND state='LIVE'",[map_id.clone().into(),generation.into()])).await.map_err(db_error)?; + } + // One bounded resident batch per exact map generation. Large + // maps stay PENDING and are replayed before new map victims. + txn.execute_raw(stmt(&self.schema,"DELETE FROM mst2_chunk_map_leaf WHERE map_id=$1 AND page_index IN (SELECT page_index FROM mst2_chunk_map_leaf WHERE map_id=$1 ORDER BY page_index LIMIT 1024)",[map_id.clone().into()])).await.map_err(db_error)?; + txn.execute_raw(stmt(&self.schema,"DELETE FROM mst2_chunk_map_node WHERE (map_id,first_page,page_count) IN (SELECT map_id,first_page,page_count FROM mst2_chunk_map_node WHERE map_id=$1 ORDER BY first_page,page_count LIMIT 1024)",[map_id.clone().into()])).await.map_err(db_error)?; + let empty=txn.query_one_raw(stmt(&self.schema,"SELECT NOT EXISTS(SELECT 1 FROM mst2_chunk_map_leaf WHERE map_id=$1) AND NOT EXISTS(SELECT 1 FROM mst2_chunk_map_node WHERE map_id=$1) AS empty",[map_id.clone().into()])).await.map_err(db_error)?.ok_or_else(||integrity("map collection completion observation missing"))?.try_get::("","empty").map_err(db_error)?; + if empty { + // Deferred lifetime FK permits map deletion while its + // exact PENDING generation is still available to guards. + txn.execute_raw(stmt(&self.schema,"DELETE FROM mst2_chunk_map WHERE map_id=$1",[map_id.clone().into()])).await.map_err(db_error)?; + txn.execute_raw(stmt(&self.schema,"DELETE FROM mst2_chunk_map_lifetime WHERE map_id=$1 AND generation=$2 AND state='DELETING'",[map_id.clone().into(),generation.into()])).await.map_err(db_error)?; + txn.execute_raw(stmt(&self.schema,"UPDATE mst2_chunk_map_gc SET state='APPLIED',applied_at=pg_catalog.clock_timestamp() WHERE map_id=$1 AND generation=$2 AND state='PENDING'",[map_id.clone().into(),generation.into()])).await.map_err(db_error)?; + } + Ok(()) + }.await; + finish(txn, result).await?; + } + Ok(()) + } + + pub(crate) async fn admit_install( + &self, + source: &ChunkMapSource, + objects: &MegaObjectStorageWrapper, + ) -> Result { + // Replay first. Every backend await happens outside the primary + // completion barrier; active reservations count concurrent creates. + self.maintain_before_admission(objects).await?; + let inventory = inventory(self, objects).await?; + let source_bytes = source.canonical_bytes()?; + let identity = source_id(&self.primary_scope, &source_bytes); + let pages = (source.fact().size as u64) + .div_ceil(mst2_codec::chunkmap::CHUNK_SIZE as u64) + .div_ceil(CHUNKS_PER_PAGE as u64) as i32; + let nodes = pages * 2 - 1; + let reserved_bytes = pages as i64 * 12288 + nodes as i64 * 512 + 1024 * 1024; + let owner = Uuid::new_v4(); + let txn = self.transaction().await?; + let result=async { + self.require_scope(&txn).await?; + require_current_source(&txn,source,&self.schema).await?; + barrier(&txn,&self.schema).await?; + ensure_quotas(&txn,&self.schema,&inventory,pages,nodes,reserved_bytes,None).await?; + let row=txn.query_one_raw(stmt(&self.schema,"SELECT EXISTS(SELECT 1 FROM mst2_chunk_receipt_generation WHERE source_id=$1) AS used,EXISTS(SELECT 1 FROM mst2_chunk_receipt_generation WHERE source_id=$1 AND state IN ('RESERVED','CREATING','LIVE','DELETING')) AS occupied",[identity.to_vec().into()])).await.map_err(db_error)?.ok_or_else(|| integrity("chunk-map reservation observation missing"))?; + if row.try_get::("","occupied").map_err(db_error)? { return Err(unavailable("exact source already has a live or pending chunk-map generation")); } + let generation=if row.try_get::("","used").map_err(db_error)? { next_generation(&txn,&self.schema).await? } else { 1 }; + let key=physical_key(identity,generation); + let owner_started=Instant::now(); + txn.execute_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_receipt_generation(receipt_key,source_id,generation,source_bytes,primary_scope,owner,state,reserved_pages,reserved_nodes,reserved_bytes,deadline) VALUES($1,$2,$3,$4,$5,$6::uuid,'RESERVED',$7,$8,$9,pg_catalog.clock_timestamp()+interval '59 seconds')",[key.key.clone().into(),identity.to_vec().into(),generation.into(),source_bytes.clone().into(),self.primary_scope.clone().into(),owner.to_string().into(),pages.into(),nodes.into(),reserved_bytes.into()])).await.map_err(db_error)?; + Ok(ChunkMapInstall{repository:self.clone(),source:source.clone(),owner,key,generation,progress:Mutex::new((Instant::now(),Instant::now(),0)),deadline:OwnerDeadline::from_sql_start(owner_started)}) + }.await; + // An uncertain commit is an error. Its UUID is never reused to mint + // another reservation; the durable deadline resolves any lost reply. + finish(txn, result).await + } + + pub(super) async fn admit_reader( + &self, + txn: &DatabaseTransaction, + source: &ChunkMapSource, + row: &QueryResult, + map: &ChunkMap, + ) -> Result { + barrier(txn, &self.schema).await?; + let key: String = row.try_get("", "receipt_key").map_err(db_error)?; + let generation: i64 = row.try_get("", "receipt_generation").map_err(db_error)?; + let map_generation: i64 = row + .try_get::>("", "map_generation") + .map_err(db_error)? + .ok_or_else(|| integrity("admitted chunk map lifetime is missing"))?; + let state: Option = row.try_get("", "generation_state").map_err(db_error)?; + let map_state: Option = row.try_get("", "map_state").map_err(db_error)?; + if state.as_deref() == Some("DELETING") { + return Err(unavailable("admitted chunk map generation is retiring")); + } + if state.as_deref() != Some("LIVE") || map_state.as_deref() != Some("LIVE") { + return Err(integrity( + "admitted source has a missing or terminal map lifetime", + )); + } + let identity = self.source_identity(source)?; + if key != physical_key(identity, generation).key + || bounded_bytes(row, "generation_receipt_bytes")? + != receipt(&self.primary_scope, source, map)? + { + return Err(integrity( + "chunk map lifetime and independent receipt identity disagree", + )); + } + txn.execute_raw(stmt(&self.schema,"DELETE FROM mst2_chunk_reader WHERE owner IN (SELECT owner FROM mst2_chunk_reader WHERE deadline<=pg_catalog.clock_timestamp() ORDER BY deadline,owner LIMIT 64)",[])).await.map_err(db_error)?; + let count = txn + .query_one_raw(stmt( + &self.schema, + "SELECT pg_catalog.count(*) AS readers FROM mst2_chunk_reader", + [], + )) + .await + .map_err(db_error)? + .ok_or_else(|| integrity("chunk reader count missing"))? + .try_get::("", "readers") + .map_err(db_error)?; + if count >= 4096 { + return Err(unavailable("chunk-map reader quota is occupied")); + } + let owner = Uuid::new_v4(); + let owner_started = Instant::now(); + txn.execute_raw(stmt(&self.schema,"INSERT INTO mst2_chunk_reader(owner,receipt_key,source_generation,map_id,map_generation,deadline) VALUES($1::uuid,$2,$3,$4,$5,pg_catalog.clock_timestamp()+interval '59 seconds')",[owner.to_string().into(),key.clone().into(),generation.into(),map.map_id().to_vec().into(),map_generation.into()])).await.map_err(db_error)?; + txn.execute_raw(stmt(&self.schema,"UPDATE mst2_chunk_map_lifetime SET last_used=pg_catalog.clock_timestamp() WHERE map_id=$1 AND generation=$2 AND state='LIVE'",[map.map_id().to_vec().into(),map_generation.into()])).await.map_err(db_error)?; + txn.execute_raw(stmt(&self.schema,"UPDATE mst2_chunk_receipt_generation SET last_progress=pg_catalog.clock_timestamp() WHERE receipt_key=$1 AND generation=$2 AND state='LIVE'",[key.clone().into(),generation.into()])).await.map_err(db_error)?; + Ok(ChunkMapReader { + repository: self.clone(), + source: source.clone(), + owner, + receipt_key: key, + generation, + map_id: map.map_id(), + map_generation, + confirmed: Mutex::new((Instant::now(), Instant::now())), + deadline: OwnerDeadline::from_sql_start(owner_started), + }) + } + + pub(super) async fn require_missing_source_retired( + &self, + txn: &DatabaseTransaction, + source: &ChunkMapSource, + ) -> Result { + let identity = self.source_identity(source)?; + let row=txn.query_one_raw(stmt(&self.schema,"SELECT state FROM mst2_chunk_receipt_generation WHERE source_id=$1 AND state IN ('LIVE','DELETING') ORDER BY generation DESC LIMIT 1",[identity.to_vec().into()])).await.map_err(db_error)?; + if let Some(row) = row { + if row.try_get::("", "state").map_err(db_error)? == "LIVE" { + return Err(integrity( + "live chunk-map source row is missing; never fall back to the cold body", + )); + } + return Ok(true); + } + Ok(false) + } +} + +pub(super) async fn barrier(txn: &DatabaseTransaction, schema: &str) -> Result<(), SnapshotError> { + txn.query_one_raw(stmt(schema, "SELECT mst2_chunk_retention_barrier()", [])) + .await + .map_err(db_error)?; + Ok(()) +} + +pub(super) async fn map_barrier( + txn: &DatabaseTransaction, + map_id: [u8; 32], +) -> Result<(), SnapshotError> { + // Per-map lock precedes the short global completion barrier. Collection + // never holds the global barrier while waiting for a map writer. + txn.query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT pg_catalog.pg_advisory_xact_lock(1296717364,pg_catalog.hashtext($1))", + [hex::encode(map_id).into()], + )) + .await + .map_err(db_error)?; + Ok(()) +} + +pub(super) async fn next_generation( + txn: &DatabaseTransaction, + schema: &str, +) -> Result { + txn.query_one_raw(Statement::from_sql_and_values( + DbBackend::Postgres, + "SELECT pg_catalog.nextval($1::regclass) AS generation", + [format!("{}.mst2_chunk_generation_sequence", quoted(schema)).into()], + )) + .await + .map_err(db_error)? + .ok_or_else(|| integrity("chunk generation sequence missing"))? + .try_get("", "generation") + .map_err(db_error) +} + +fn physical_key(source: [u8; 32], generation: i64) -> ObjectKey { + ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: if generation == 1 { + hex::encode(source) + } else { + format!("{}-{generation:016x}", hex::encode(source)) + }, + } +} + +pub(super) async fn inventory( + repository: &PostgresChunkMapRepository, + objects: &MegaObjectStorageWrapper, +) -> Result, SnapshotError> { + let mut cached = repository.receipt_observation.lock().await; + if let Some(observation) = cached.as_ref().filter(|observation| { + Arc::ptr_eq(&observation.backing.inner, &objects.inner) + && observation.captured.elapsed() < RENEW_INTERVAL + }) { + return Ok(observation.clone()); + } + let memory = retention_budget().reserve(4 * 1024 * 1024)?; + let txn = repository.transaction().await?; + repository.require_scope(&txn).await?; + let started_at = txn + .query_one_raw(Statement::from_string( + DbBackend::Postgres, + "SELECT pg_catalog.clock_timestamp()::text AS started_at", + )) + .await + .map_err(db_error)? + .ok_or_else(|| integrity("backing inventory database clock missing"))? + .try_get("", "started_at") + .map_err(db_error)?; + txn.commit().await.map_err(db_error)?; + let inventory = tokio::time::timeout( + Duration::from_secs(30), + objects.inner.chunk_map_receipt_inventory(), + ) + .await + .map_err(|_| unavailable("bounded chunk receipt inventory timed out"))? + .map_err(|error| match error { + IoOrbitError::ChunkMapRetentionUnsupported => SnapshotError::new( + SnapshotErrorCode::TemporaryUnavailable, + "backing store does not support bounded chunk-map retention", + ), + IoOrbitError::ChunkMapRetentionCapacityExceeded => SnapshotError::new( + SnapshotErrorCode::LimitExceeded, + "actual backing chunk-map receipts exceed retention quota", + ), + error => storage_error(error), + })?; + let observation = Arc::new(BackingObservation { + inventory, + started_at, + captured: Instant::now(), + backing: objects.clone(), + _memory: memory, + }); + *cached = Some(observation.clone()); + Ok(observation) +} + +pub(super) async fn ensure_quotas( + txn: &DatabaseTransaction, + schema: &str, + observation: &BackingObservation, + pages: i32, + nodes: i32, + bytes: i64, + exclude: Option, +) -> Result<(), SnapshotError> { + let inventory = &observation.inventory; + let row=txn.query_one_raw(stmt(schema,"SELECT (SELECT pg_catalog.count(*) FROM mst2_chunk_map) AS maps,(SELECT pg_catalog.count(*) FROM mst2_chunk_map_source) AS sources,(SELECT pg_catalog.count(*) FROM mst2_chunk_map_leaf) AS leaves,(SELECT pg_catalog.count(*) FROM mst2_chunk_map_node) AS nodes,(SELECT pg_catalog.count(*) FROM mst2_chunk_receipt_generation) AS history,(SELECT pg_catalog.count(*) FROM mst2_chunk_map_gc) AS map_history,(SELECT pg_catalog.count(*) FROM mst2_chunk_receipt_generation WHERE receipt_bytes IS NOT NULL AND NOT create_completed) AS uncertain,(SELECT pg_catalog.count(*) FROM mst2_chunk_receipt_generation WHERE receipt_bytes IS NOT NULL AND (created_at>=$2::timestamptz OR create_completed_at>=$2::timestamptz)) AS recent,pg_catalog.count(*) AS pending,COALESCE(pg_catalog.sum(reserved_pages),0)::bigint AS pending_pages,COALESCE(pg_catalog.sum(reserved_nodes),0)::bigint AS pending_nodes,COALESCE(pg_catalog.sum(reserved_bytes),0)::bigint AS pending_bytes FROM mst2_chunk_receipt_generation WHERE state IN ('RESERVED','CREATING') AND ($1::uuid IS NULL OR owner IS DISTINCT FROM $1::uuid)",[exclude.map(|id|id.to_string()).into(),observation.started_at.clone().into()])).await.map_err(db_error)?.ok_or_else(|| integrity("chunk-map actual quota observation missing"))?; + let get = |name: &str| row.try_get::("", name).map_err(db_error); + let pending = get("pending")?; + let relation_bytes=txn.query_one_raw(Statement::from_sql_and_values(DbBackend::Postgres,"SELECT COALESCE(pg_catalog.sum(pg_catalog.pg_total_relation_size(c.oid)),0)::bigint AS bytes FROM pg_catalog.pg_class c JOIN pg_catalog.pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname=$1 AND c.relkind='r' AND c.relname IN ('mst2_chunk_map','mst2_chunk_map_source','mst2_chunk_map_leaf','mst2_chunk_map_node','mst2_chunk_receipt_generation','mst2_chunk_map_lifetime','mst2_chunk_reader','mst2_chunk_map_gc')",[schema.into()])).await.map_err(db_error)?.ok_or_else(||integrity("chunk relation byte observation missing"))?.try_get::("","bytes").map_err(db_error)?; + let extra = if exclude.is_none() { 1 } else { 0 }; + if get("maps")? + pending + extra > 4096 + || get("sources")? + pending + extra > 16384 + || get("leaves")? + get("pending_pages")? + pages as i64 > 65536 + || get("nodes")? + get("pending_nodes")? + nodes as i64 > 131072 + || relation_bytes + get("pending_bytes")? + bytes + 1024 * 1024 > 536870912 + || get("history")? + extra > HISTORY_CAP + || get("map_history")? > HISTORY_CAP + || pending + extra > 64 + || inventory.objects.len() as i64 + pending + get("uncertain")? + get("recent")? + extra + > 16384 + || inventory.bytes + ((pending + get("uncertain")? + get("recent")? + extra) as u64) * 4096 + > 64 * 1024 * 1024 + { + return Err(SnapshotError::new( + SnapshotErrorCode::LimitExceeded, + "actual chunk-map rows, history, backing or reserved capacity exceed retention quota", + )); + } + Ok(()) +} + +fn unavailable(message: &str) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::TemporaryUnavailable, message) +} +fn expired(message: &str) -> SnapshotError { + SnapshotError::new(SnapshotErrorCode::LeaseExpired, message) +} + +async fn read_bounded_orphan( + objects: &MegaObjectStorageWrapper, + key: &ObjectKey, +) -> Result>, SnapshotError> { + tokio::time::timeout(Duration::from_secs(5), async { + let (mut stream, meta) = match objects.inner.get_stream(key).await { + Ok(value) => value, + Err(error) if error.is_not_found() => return Ok(None), + Err(error) => return Err(storage_error(error)), + }; + if !(129..=4096).contains(&meta.size) { + return Err(integrity("orphan receipt exceeds its byte profile")); + } + let mut bytes = Vec::with_capacity(meta.size as usize); + while let Some(part) = stream.next().await { + let part = part.map_err(storage_error)?; + if part.len() > meta.size as usize - bytes.len() { + return Err(integrity( + "orphan receipt exceeds its advertised exact size", + )); + } + bytes.extend_from_slice(&part); + } + if bytes.len() != meta.size as usize { + return Err(integrity("orphan receipt is truncated")); + } + Ok(Some(bytes)) + }) + .await + .map_err(|_| unavailable("orphan receipt read timed out"))? +} + +fn decode_orphan( + key: &ObjectKey, + bytes: &[u8], +) -> Result<(Vec, Vec, ChunkMap, i64), SnapshotError> { + if !bytes.starts_with(RECEIPT_DOMAIN) { + return Err(integrity("orphan is not a canonical trusted receipt")); + } + let mut offset = RECEIPT_DOMAIN.len(); + let take = |offset: &mut usize, max: usize| -> Result, SnapshotError> { + let len_bytes = bytes + .get(*offset..*offset + 4) + .ok_or_else(|| integrity("orphan receipt length is truncated"))?; + let len = u32::from_be_bytes( + len_bytes + .try_into() + .map_err(|_| integrity("invalid orphan receipt length"))?, + ) as usize; + *offset += 4; + if len == 0 || len > max { + return Err(integrity("orphan receipt field exceeds its profile")); + } + let field = bytes + .get(*offset..*offset + len) + .ok_or_else(|| integrity("orphan receipt field is truncated"))? + .to_vec(); + *offset += len; + Ok(field) + }; + let scope = take(&mut offset, 1024)?; + let source = take(&mut offset, 2048)?; + let descriptor = bytes + .get(offset..) + .ok_or_else(|| integrity("orphan map descriptor is missing"))?; + let map = ChunkMap::decode(descriptor) + .map_err(|_| integrity("orphan map descriptor is not canonical MCM2"))?; + if map.encode() != descriptor { + return Err(integrity("orphan receipt has noncanonical map bytes")); + } + let generation = if key.key.len() == 64 { + 1 + } else if key.key.len() == 81 && key.key.as_bytes()[64] == b'-' { + i64::from_str_radix(&key.key[65..], 16) + .map_err(|_| integrity("orphan physical generation is invalid"))? + } else { + return Err(integrity( + "orphan receipt key has invalid generation profile", + )); + }; + if generation <= 0 || physical_key(source_id(&scope, &source), generation) != *key { + return Err(integrity( + "orphan physical receipt key disagrees with its exact body", + )); + } + Ok((scope, source, map, generation)) +} diff --git a/src/jupiter/storage/object_storage.rs b/src/jupiter/storage/object_storage.rs index f65a33ab..3fbe45d8 100644 --- a/src/jupiter/storage/object_storage.rs +++ b/src/jupiter/storage/object_storage.rs @@ -278,6 +278,56 @@ impl MegaObjectStorage for InMemoryObjectStorage { Ok((Self::stream_bytes(bytes), meta)) } + async fn chunk_map_receipt_inventory( + &self, + ) -> OrbitResult { + use crate::orbit_api::object_storage::{ + ChunkMapReceiptInventory, MAX_CHUNK_MAP_RECEIPT_BYTES, MAX_CHUNK_MAP_RECEIPTS, + ObjectNamespace, + }; + let objects = self + .objects + .lock() + .map_err(|_| IoOrbitError::Other("object storage lock poisoned".into()))?; + let mut inventory = ChunkMapReceiptInventory { + objects: Vec::new(), + bytes: 0, + }; + for (key, (bytes, _)) in objects + .iter() + .filter(|(key, _)| key.namespace == ObjectNamespace::ChunkMapReceipt) + { + if inventory.objects.len() == MAX_CHUNK_MAP_RECEIPTS { + return Err(IoOrbitError::ChunkMapRetentionCapacityExceeded); + } + inventory.bytes = inventory + .bytes + .checked_add(bytes.len() as u64) + .filter(|bytes| *bytes <= MAX_CHUNK_MAP_RECEIPT_BYTES) + .ok_or(IoOrbitError::ChunkMapRetentionCapacityExceeded)?; + inventory.objects.push((key.clone(), bytes.len() as u64)); + } + Ok(inventory) + } + + async fn delete_chunk_map_receipt( + &self, + authority: &crate::orbit_api::object_storage::ChunkMapReceiptDeletion, + ) -> OrbitResult { + let mut objects = self + .objects + .lock() + .map_err(|_| IoOrbitError::Other("object storage lock poisoned".into()))?; + let Some((bytes, _)) = objects.get(authority.key()) else { + return Ok(false); + }; + if bytes.as_ref() != authority.expected_bytes() { + return Err(IoOrbitError::Other("retired receipt body changed".into())); + } + objects.remove(authority.key()); + Ok(true) + } + async fn get_range_stream( &self, key: &ObjectKey, diff --git a/src/orbit/adapter/object.rs b/src/orbit/adapter/object.rs index 87327a39..4ae923b0 100644 --- a/src/orbit/adapter/object.rs +++ b/src/orbit/adapter/object.rs @@ -104,6 +104,111 @@ impl MegaObjectStorage for ObjectStoreAdapter { Ok((Box::pin(stream), meta)) } + async fn chunk_map_receipt_inventory( + &self, + ) -> OrbitResult { + use crate::orbit_api::object_storage::{ + ChunkMapReceiptInventory, MAX_CHUNK_MAP_RECEIPT_BYTES, MAX_CHUNK_MAP_RECEIPTS, + }; + // object_store 0.14 cannot inventory/delete retained cloud versions. + // A current-object listing is not a physical backing-byte quota. + if !matches!(&self.store, BackendStore::Local(_)) { + return Err(IoOrbitError::ChunkMapRetentionUnsupported); + } + let prefix = object_store::path::Path::from("chunk-map-receipt"); + let mut listing = self.to_store().list(Some(&prefix)); + let mut inventory = ChunkMapReceiptInventory { + objects: Vec::new(), + bytes: 0, + }; + while let Some(entry) = listing.next().await { + let meta = entry.map_err(IoOrbitError::from)?; + if inventory.objects.len() == MAX_CHUNK_MAP_RECEIPTS { + return Err(IoOrbitError::ChunkMapRetentionCapacityExceeded); + } + inventory.bytes = inventory + .bytes + .checked_add(meta.size) + .filter(|bytes| *bytes <= MAX_CHUNK_MAP_RECEIPT_BYTES) + .ok_or(IoOrbitError::ChunkMapRetentionCapacityExceeded)?; + let location = meta.location.as_ref(); + let parts: Vec<_> = location.split('/').collect(); + if parts.len() != 5 + || parts[0] != "chunk-map-receipt" + || parts[1..4].iter().any(|part| part.len() != 2) + || parts[4].len() > 128 + { + return Err(IoOrbitError::Other( + "invalid physical chunk-map receipt path".into(), + )); + } + let key = ObjectKey { + namespace: ObjectNamespace::ChunkMapReceipt, + key: parts[1..].concat(), + }; + key.validate()?; + if key.default_sharding() != location { + return Err(IoOrbitError::Other( + "noncanonical physical chunk-map receipt path".into(), + )); + } + inventory.objects.push((key, meta.size)); + } + Ok(inventory) + } + + async fn delete_chunk_map_receipt( + &self, + authority: &crate::orbit_api::object_storage::ChunkMapReceiptDeletion, + ) -> OrbitResult { + if !matches!(&self.store, BackendStore::Local(_)) { + return Err(IoOrbitError::ChunkMapRetentionUnsupported); + } + let key = authority.key(); + if key.namespace != ObjectNamespace::ChunkMapReceipt { + return Err(IoOrbitError::Other( + "invalid sealed receipt namespace".into(), + )); + } + let (mut stream, meta) = match self.get_stream(key).await { + Ok(object) => object, + Err(error) if error.is_not_found() => return Ok(false), + Err(error) => return Err(error), + }; + let expected = authority.expected_bytes(); + if meta.size != expected.len() as i64 { + return Err(IoOrbitError::Other( + "retired receipt body size changed".into(), + )); + } + let mut offset: usize = 0; + while let Some(part) = stream.next().await { + let bytes = part?; + let end = offset + .checked_add(bytes.len()) + .filter(|end| *end <= expected.len()) + .ok_or_else(|| { + IoOrbitError::Other("retired receipt body exceeds its exact profile".into()) + })?; + if expected[offset..end] != bytes[..] { + return Err(IoOrbitError::Other("retired receipt body changed".into())); + } + offset = end; + } + if offset != expected.len() { + return Err(IoOrbitError::Other( + "retired receipt body is truncated".into(), + )); + } + // Physical generation keys are never reused. A late create can only + // restore this same retired body; persistent history will reconcile it. + match self.to_store().delete(&Self::checked_path(key)?).await { + Ok(()) => Ok(true), + Err(object_store::Error::NotFound { .. }) => Ok(false), + Err(error) => Err(IoOrbitError::from(error)), + } + } + async fn get_range_stream( &self, key: &ObjectKey, diff --git a/src/orbit_api/error.rs b/src/orbit_api/error.rs index 365d2e4d..9de51a0c 100644 --- a/src/orbit_api/error.rs +++ b/src/orbit_api/error.rs @@ -17,6 +17,12 @@ pub enum IoOrbitError { #[error("write manifest precondition failed")] WriteManifestPreconditionFailed, + #[error("chunk-map retention is not supported by this storage backend")] + ChunkMapRetentionUnsupported, + + #[error("chunk-map backing receipts exceed the fixed retention quota")] + ChunkMapRetentionCapacityExceeded, + #[error("other error: {0}")] Other(String), } diff --git a/src/orbit_api/object_storage.rs b/src/orbit_api/object_storage.rs index 90539315..9264d944 100644 --- a/src/orbit_api/object_storage.rs +++ b/src/orbit_api/object_storage.rs @@ -171,6 +171,39 @@ pub type ObjectByteStream = Pin, + pub bytes: u64, +} + +/// Sealed by the primary retention repository after claiming one exact +/// retired generation and independently reading its immutable backend body. +/// Ordinary object callers cannot construct a receipt deletion authority. +pub struct ChunkMapReceiptDeletion { + claim: crate::jupiter::storage::native_chunk_map::retention::VerifiedReceiptDeletionClaim, +} + +impl ChunkMapReceiptDeletion { + pub(crate) fn from_claim( + claim: crate::jupiter::storage::native_chunk_map::retention::VerifiedReceiptDeletionClaim, + ) -> Self { + Self { claim } + } + + pub(crate) fn key(&self) -> &ObjectKey { + self.claim.key() + } + + pub(crate) fn expected_bytes(&self) -> &[u8] { + self.claim.expected_bytes() + } +} + /// A streaming source of multiple objects. /// /// Each item yields: @@ -278,6 +311,22 @@ pub trait MegaObjectStorage: Send + Sync { )) } + /// Enumerate the real receipt namespace with fixed count/byte bounds. + /// Unsupported stores fail before a cold source body can be opened. + async fn chunk_map_receipt_inventory(&self) -> OrbitResult { + Err(IoOrbitError::ChunkMapRetentionUnsupported) + } + + /// Delete only the independently checked body of a sealed retired + /// physical generation. The key is never reused by a later installation. + /// Missing is idempotent; transport/authentication errors remain errors. + async fn delete_chunk_map_receipt( + &self, + _authority: &ChunkMapReceiptDeletion, + ) -> OrbitResult { + Err(IoOrbitError::ChunkMapRetentionUnsupported) + } + /// Retrieve a single object from the storage backend. /// /// # Returns diff --git a/src/server/http_server.rs b/src/server/http_server.rs index 04be7f77..777e732a 100644 --- a/src/server/http_server.rs +++ b/src/server/http_server.rs @@ -380,6 +380,35 @@ fn spawn_view_worker_task( spawn_view_worker_with_round(storage, token, production_round()) } +fn spawn_chunk_map_retention_task( + storage: Storage, + token: CancellationToken, + available: bool, +) -> Option> { + if !available { + return None; + } + Some(tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(30)); + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { + tokio::select! { + _=token.cancelled()=>break, + _=interval.tick()=>{ + // Cancellation drops the real maintenance future. Claims + // are durable, so interrupted backend operations replay. + tokio::select! { + _=token.cancelled()=>break, + result=async { + storage.chunk_maps().await?.maintain(&storage.git_service.obj_storage,64).await + }=>if let Err(error)=result {tracing::warn!(?error,"bounded chunk-map retention tick failed");} + } + } + } + } + })) +} + /// Returns a future that completes when the cancellation token is triggered. async fn shutdown_signal(token: CancellationToken) { token.cancelled().await; @@ -450,6 +479,46 @@ pub async fn start_http(ctx: AppContext, options: CommonHttpOptions) -> MegaResu // TP-15 / 4.1 ②③⑥: open CLs, last_policy vs non-terminal rows, watermark reset. ctx.storage.prepare_push_policy_startup().await?; ctx.storage.check_view_reserved_startup().await?; + let chunk_map_retention_available = if ctx.storage.config().mst2.enabled { + // Detect the actual backend capability. Unsupported CHUNK retention + // must not disable other snapshot objects or generic HTTP serving. + match tokio::time::timeout( + std::time::Duration::from_secs(30), + ctx.storage + .git_service + .obj_storage + .inner + .chunk_map_receipt_inventory(), + ) + .await + { + Ok(Err(crate::orbit_api::error::IoOrbitError::ChunkMapRetentionUnsupported)) => { + tracing::info!( + "CHUNK cold admission and maintenance unavailable for this backing store; other snapshot routes remain available" + ); + false + } + _ => { + let result = async { + ctx.storage + .chunk_maps() + .await? + .maintain(&ctx.storage.git_service.obj_storage, 64) + .await + } + .await; + if let Err(error) = result { + tracing::warn!( + ?error, + "initial CHUNK retention check failed; cold admission remains fail closed" + ); + } + true + } + } + } else { + false + }; // First-build the shared authorization snapshot before the listener binds // (UN-02). `off` is a no-op; a failed first build fails startup. @@ -479,6 +548,11 @@ pub async fn start_http(ctx: AppContext, options: CommonHttpOptions) -> MegaResu spawn_artifact_gc_task(shutdown_context.clone(), shutdown_token.clone())?; let view_worker_handle = spawn_view_worker_task(shutdown_context.storage.clone(), shutdown_token.clone())?; + let chunk_map_retention_handle = spawn_chunk_map_retention_task( + shutdown_context.storage.clone(), + shutdown_token.clone(), + chunk_map_retention_available, + ); let notification_shutdown = shutdown_context.notification_shutdown.clone(); let server_token = shutdown_token.clone(); @@ -541,7 +615,7 @@ pub async fn start_http(ctx: AppContext, options: CommonHttpOptions) -> MegaResu }, }; - let (cleanup_result, artifact_gc_result, view_worker_result) = tokio::join!( + let (cleanup_result, artifact_gc_result, view_worker_result, chunk_map_retention_result) = tokio::join!( async { if let Some(handle) = cleanup_handle { match tokio::time::timeout(std::time::Duration::from_secs(30), handle).await { @@ -612,11 +686,34 @@ pub async fn start_http(ctx: AppContext, options: CommonHttpOptions) -> MegaResu } else { Ok(()) } + }, + async { + if let Some(mut handle) = chunk_map_retention_handle { + match tokio::time::timeout(std::time::Duration::from_secs(30), &mut handle).await { + Ok(Ok(())) => Ok(()), + Ok(Err(error)) => { + tracing::error!(%error,"chunk-map retention task panicked"); + Err(()) + } + Err(_) => { + handle.abort(); + let _ = handle.await; + tracing::error!( + "chunk-map retention task exceeded shutdown deadline and was aborted" + ); + Err(()) + } + } + } else { + Ok(()) + } } ); - let shutdown_failed = - cleanup_result.is_err() || artifact_gc_result.is_err() || view_worker_result.is_err(); + let shutdown_failed = cleanup_result.is_err() + || artifact_gc_result.is_err() + || view_worker_result.is_err() + || chunk_map_retention_result.is_err(); match (shutdown_failed, &server_result) { (false, Ok(())) => { tracing::info!("Graceful shutdown completed successfully");