diff --git a/node/src/aggregates/graph_jobs.rs b/node/src/aggregates/graph_jobs.rs index 9af924e8..ea1d97f6 100644 --- a/node/src/aggregates/graph_jobs.rs +++ b/node/src/aggregates/graph_jobs.rs @@ -216,7 +216,7 @@ async fn run_step( JOB_POST_GRAPH_DATA => post_graph_data(deps, graph_id, params).await, JOB_FETCH_GRAPH => fetch_graph(deps, graph_id, params, attempts).await, pegout if PEGOUT_JOBS.contains(&pegout) => { - run_pegout_step(deps, pegout, graph_id, params, attempts).await + run_pegout_step(deps, pegout, graph_id, params).await } other => bail!("unknown graph job {other}"), } @@ -615,9 +615,12 @@ async fn ingest_soldering_proof(deps: &JobDeps, graph_id: Uuid, params: &[u8]) - let Some(frozen) = operator_state.candidate_verifier_pubkeys.clone() else { return Step::retry("operator verifier membership is not frozen", 30); }; - let verifier_pubkey = *frozen.get(candidate_index).ok_or_else(|| { - anyhow!("SolderingProofReady candidate index {candidate_index} out of range") - })?; + // The membership is frozen, so an index outside it can never become valid. + let Some(&verifier_pubkey) = frozen.get(candidate_index) else { + return invalid(format!( + "SolderingProofReady candidate index {candidate_index} out of range" + )); + }; let candidate = operator_state .candidates .iter() @@ -1561,6 +1564,8 @@ async fn verifier_validate_graph(deps: &JobDeps, graph_id: Uuid, params: &[u8]) else { return Step::ignore("local verifier is not in this graph"); }; + // Only a graph this verifier ran the setup for is worth the RPC checks and the + // graph rebuild below. let Some(verifier_state) = load_babe_setup_state(&deps.local_db, instance_id, graph_id)? .and_then(|state| state.verifier) else { @@ -1682,6 +1687,11 @@ async fn committee_ingest_graph(deps: &JobDeps, graph_id: Uuid, params: &[u8]) - if local_member(deps, instance_id).await?.is_none() { return Step::ignore("not in this instance's committee"); } + // Checked first: validation retries a lagging Bitcoin backend without limit, + // and the gate ends those retries at T0 or when the instance fails. + if let Some(reason) = gate_closed(deps, instance_id).await? { + return Step::ignore(format!("signing gate closed: {reason}")); + } // Structure, policy, prekickoff continuity, instance parameters against // both chains, watchtower set and operator stake; verifier endorsements // may arrive later. @@ -1794,6 +1804,11 @@ async fn committee_nonces(deps: &JobDeps, graph_id: Uuid, _params: &[u8]) -> Res graph.parameters.gc_data.len() )); } + // Checked first: validation retries a lagging Bitcoin backend without limit, + // and the gate ends those retries at T0 or when the instance fails. + if let Some(reason) = gate_closed(deps, instance_id).await? { + return Step::ignore(format!("signing gate closed: {reason}")); + } let validation = validate_init_graph( &deps.local_db, &deps.btc_client, @@ -1804,9 +1819,6 @@ async fn committee_nonces(deps: &JobDeps, graph_id: Uuid, _params: &[u8]) -> Res .await; deps.metrics_state.record_graph_validation(validation.is_ok()); validation?; - if let Some(reason) = gate_closed(deps, instance_id).await? { - return Step::ignore(format!("signing gate closed: {reason}")); - } let Some(instance_keypair) = local_member(deps, instance_id).await? else { return Step::ignore("not in this instance's committee"); }; @@ -2363,11 +2375,22 @@ const FETCH_GRAPH_FAST_ATTEMPTS: i64 = 5; const FETCH_GRAPH_SLOW_RETRY_SECS: i64 = 600; const FETCH_GRAPH_MAX_ATTEMPTS: i64 = FETCH_GRAPH_FAST_ATTEMPTS + super::common::PARKED_MESSAGE_TTL_SECS / FETCH_GRAPH_SLOW_RETRY_SECS; +/// Slow retries between two alerts for a posted graph nobody serves (about a day). +const FETCH_GRAPH_POSTED_ALERT_EVERY: i64 = 24 * 60 * 60 / FETCH_GRAPH_SLOW_RETRY_SECS; async fn fetch_graph(deps: &JobDeps, graph_id: Uuid, params: &[u8], attempts: i64) -> Result { let lookup: GraphLookup = bincode::deserialize(params).context("decode graph lookup")?; + // A graph posted on Goat exists, and only the event watcher asks for it, once: + // that pull never gives up. It keeps retrying slowly and alerts about once a day. + let posted_on_chain = matches!(lookup, GraphLookup::Id { finalized: true, .. }); let give_up = |reason: String| { - if attempts + 1 >= FETCH_GRAPH_MAX_ATTEMPTS { + if posted_on_chain && attempts + 1 >= FETCH_GRAPH_FAST_ATTEMPTS { + if (attempts + 1 - FETCH_GRAPH_FAST_ATTEMPTS) % FETCH_GRAPH_POSTED_ALERT_EVERY == 0 { + warn!(event = "graph_fetch", graph_id = %graph_id, ?lookup, attempts, reason, "posted graph still not served by any peer"); + deps.metrics_state.record_instance_alert("posted_graph_unavailable"); + } + Step::retry(reason, FETCH_GRAPH_SLOW_RETRY_SECS) + } else if attempts + 1 >= FETCH_GRAPH_MAX_ATTEMPTS { warn!(event = "graph_fetch", graph_id = %graph_id, ?lookup, attempts, reason, "giving up on the pull"); Step::ignore(format!("gave up pulling {lookup:?}: {reason}")) } else if attempts + 1 >= FETCH_GRAPH_FAST_ATTEMPTS { diff --git a/node/src/aggregates/graph_pegout_jobs.rs b/node/src/aggregates/graph_pegout_jobs.rs index 68e63d27..1330d780 100644 --- a/node/src/aggregates/graph_pegout_jobs.rs +++ b/node/src/aggregates/graph_pegout_jobs.rs @@ -58,7 +58,6 @@ pub(super) async fn run_pegout_step( action: &str, graph_id: Uuid, params: &[u8], - attempts: i64, ) -> Result { match action { JOB_KICKOFF => kickoff(deps, graph_id, params).await, @@ -68,7 +67,7 @@ pub(super) async fn run_pegout_step( let Some((row, graph)) = load_graph(&deps.local_db, graph_id).await? else { return Step::ignore("graph not stored locally"); }; - settlement_step(deps, action, &row, graph, signal, attempts).await + settlement_step(deps, action, &row, graph, signal).await } } } @@ -79,7 +78,6 @@ async fn settlement_step( row: &Graph, graph: BitvmGcGraph, signal: PegoutSignal, - attempts: i64, ) -> Result { use PegoutSignal::*; // A closed graph's branches are over; it may still owe the relayer's Goat @@ -97,7 +95,7 @@ async fn settlement_step( } match (action, signal) { (JOB_ENFORCE_PREVIOUS_SLOT, _) => enforce_previous_slot(deps, &graph).await, - (JOB_PROCEED_WITHDRAW, _) => proceed_withdraw(deps, row, &graph, attempts).await, + (JOB_PROCEED_WITHDRAW, _) => proceed_withdraw(deps, row, &graph).await, (JOB_CHALLENGE_KICKOFF, _) => challenge_kickoff(deps, &graph).await, (JOB_WATCHTOWER_CHALLENGE_INIT, ChallengeSent { challenge_txid }) => { watchtower_challenge_init(deps, row, graph, challenge_txid).await @@ -139,9 +137,9 @@ async fn settlement_step( .await } (JOB_TAKE1, _) => take1(deps, row, graph).await, - (JOB_FINISH_HAPPY, _) => finish_happy(deps, &graph, attempts).await, + (JOB_FINISH_HAPPY, _) => finish_happy(deps, &graph).await, (JOB_TAKE2, _) => take2(deps, row, graph).await, - (JOB_FINISH_UNHAPPY, _) => finish_unhappy(deps, &graph, attempts).await, + (JOB_FINISH_UNHAPPY, _) => finish_unhappy(deps, &graph).await, (action, signal) => bail!("{action} does not act on {signal:?}"), } } @@ -372,7 +370,14 @@ async fn settle_predecessors(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> R else { bail!("slot {slot} definition disappeared"); }; - operator_skip_graph(&deps.btc_client, &mut skipped).await?; + if !operator_skip_graph(&deps.btc_client, &mut skipped).await? { + let reason = format!( + "slot {slot}: Kickoff connector cannot pay the Skip-Kickoff fee at the current rate" + ); + warn!(event = "graph_skip", graph_id = %graph_id, skipped_graph_id = %local.graph_id, slot, reason = %reason, "earlier slot cannot be skipped yet"); + alert("pegout_predecessor_stuck"); + return after_blocks(deps, reason, 10); + } info!( event = "graph_skip", graph_id = %graph_id, @@ -1720,23 +1725,40 @@ async fn watchtower_proof_timeout(row: &Graph, graph: &BitvmGcGraph) -> Result Result> { + let Some(height) = deps.btc_client.get_tx_status(txid).await?.block_height else { + return Ok(None); + }; + let tip = deps.btc_client.get_height().await?; + Ok(Some(u64::from(tip.saturating_add(1).saturating_sub(height)))) +} /// For proceed/happy/unhappy jobs, wait for the withdraw to become visible. /// Disproved settlement must not use this gate: it also handles rogue kickoffs /// without initWithdraw. These jobs run once per graph, so an early /// `ignore` would leave the Goat side behind for good. -fn withdraw_missing(deps: &JobDeps, graph_id: Uuid, call: &str, attempts: i64) -> Result { - if attempts < WITHDRAW_LAG_RETRIES { +fn withdraw_missing( + deps: &JobDeps, + graph_id: Uuid, + call: &str, + depth: Option, +) -> Result { + if depth.is_none_or(|depth| depth < WITHDRAW_LAG_BLOCKS) { return after_blocks(deps, format!("withdraw is not visible on Goat yet; {call} waits"), 1); } - warn!(event = "graph_withdraw", graph_id = %graph_id, call, attempts, "withdraw never appeared on Goat; giving up"); + warn!(event = "graph_withdraw", graph_id = %graph_id, call, ?depth, "withdraw never appeared on Goat; giving up"); alert("pegout_withdraw_missing"); - Step::ignore(format!("withdraw not on Goat after {attempts} attempts; {call} skipped")) + Step::ignore(format!("withdraw not on Goat {depth:?} blocks after settlement; {call} skipped")) } /// `proceedWithdraw` has not landed: wait for it, but not forever. @@ -1744,14 +1766,16 @@ fn withdraw_not_proceeded( deps: &JobDeps, graph_id: Uuid, call: &str, - attempts: i64, + depth: Option, ) -> Result { - if attempts < WITHDRAW_PROCEED_RETRIES { + if depth.is_none_or(|depth| depth < WITHDRAW_PROCEED_BLOCKS) { return after_blocks(deps, "withdraw is initialized but the kickoff is not posted", 6); } - warn!(event = "graph_withdraw", graph_id = %graph_id, call, attempts, "proceedWithdraw never landed; giving up"); + warn!(event = "graph_withdraw", graph_id = %graph_id, call, ?depth, "proceedWithdraw never landed; giving up"); alert("pegout_withdraw_stuck"); - Step::ignore(format!("withdraw still Initialized after {attempts} attempts; {call} skipped")) + Step::ignore(format!( + "withdraw still Initialized {depth:?} blocks after settlement; {call} skipped" + )) } /// Goat shows a status the chain contradicts: report it, nothing to send. @@ -1766,12 +1790,7 @@ fn withdraw_inconsistent( Step::ignore(format!("withdraw is {status:?} although {settled_by} confirmed; {call} skipped")) } -async fn proceed_withdraw( - deps: &JobDeps, - row: &Graph, - graph: &BitvmGcGraph, - attempts: i64, -) -> Result { +async fn proceed_withdraw(deps: &JobDeps, row: &Graph, graph: &BitvmGcGraph) -> Result { if !deps.relayer { return Step::ignore("not a relayer"); } @@ -1784,7 +1803,8 @@ async fn proceed_withdraw( return Step::ignore("graph closed without a withdraw on Goat"); } WithdrawStatus::None => { - return withdraw_missing(deps, graph_id, "proceedWithdraw", attempts); + let depth = confirmations(deps, &graph.kickoff.tx().compute_txid()).await?; + return withdraw_missing(deps, graph_id, "proceedWithdraw", depth); } WithdrawStatus::Canceled => { // Cancelling needs the committee; a cancelled withdraw is handled by hand. @@ -1833,7 +1853,7 @@ async fn spv_covered(deps: &JobDeps, txid: &Txid, what: &str) -> Result Result { +async fn finish_happy(deps: &JobDeps, graph: &BitvmGcGraph) -> Result { if !deps.relayer { return Step::ignore("not a relayer"); } @@ -1846,10 +1866,12 @@ async fn finish_happy(deps: &JobDeps, graph: &BitvmGcGraph, attempts: i64) -> Re match withdraw_status { WithdrawStatus::Processing => {} WithdrawStatus::Initialized => { - return withdraw_not_proceeded(deps, graph_id, "finishWithdrawHappyPath", attempts); + let depth = confirmations(deps, &take1_txid).await?; + return withdraw_not_proceeded(deps, graph_id, "finishWithdrawHappyPath", depth); } WithdrawStatus::None => { - return withdraw_missing(deps, graph_id, "finishWithdrawHappyPath", attempts); + let depth = confirmations(deps, &take1_txid).await?; + return withdraw_missing(deps, graph_id, "finishWithdrawHappyPath", depth); } WithdrawStatus::Complete => return Step::ignore("withdraw already complete on Goat"), other => { @@ -1867,7 +1889,7 @@ async fn finish_happy(deps: &JobDeps, graph: &BitvmGcGraph, attempts: i64) -> Re } /// Take-2 confirmed: `finishWithdrawUnhappyPath`. -async fn finish_unhappy(deps: &JobDeps, graph: &BitvmGcGraph, attempts: i64) -> Result { +async fn finish_unhappy(deps: &JobDeps, graph: &BitvmGcGraph) -> Result { if !deps.relayer { return Step::ignore("not a relayer"); } @@ -1880,10 +1902,12 @@ async fn finish_unhappy(deps: &JobDeps, graph: &BitvmGcGraph, attempts: i64) -> match withdraw_status { WithdrawStatus::Processing => {} WithdrawStatus::Initialized => { - return withdraw_not_proceeded(deps, graph_id, "finishWithdrawUnhappyPath", attempts); + let depth = confirmations(deps, &take2_txid).await?; + return withdraw_not_proceeded(deps, graph_id, "finishWithdrawUnhappyPath", depth); } WithdrawStatus::None => { - return withdraw_missing(deps, graph_id, "finishWithdrawUnhappyPath", attempts); + let depth = confirmations(deps, &take2_txid).await?; + return withdraw_missing(deps, graph_id, "finishWithdrawUnhappyPath", depth); } WithdrawStatus::Complete => return Step::ignore("withdraw already complete on Goat"), other => { diff --git a/node/src/aggregates/graph_scan.rs b/node/src/aggregates/graph_scan.rs index 96792475..6d6b8547 100644 --- a/node/src/aggregates/graph_scan.rs +++ b/node/src/aggregates/graph_scan.rs @@ -30,8 +30,9 @@ use goat::constants::TimelockConfig; use goat::transactions::base::output_topology; use goat::transactions::pre_signed::PreSignedTransaction; use serde::{Deserialize, Serialize}; -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::str::FromStr; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{LazyLock, Mutex}; use store::localdb::{GraphQuery, LocalDB, StorageProcessor}; use store::{Graph, GraphBtcTxVoutMonitor, GraphStatus, InstanceBridgeInStatus, SerializableTxid}; @@ -94,11 +95,19 @@ fn resume_index(slots: &[(String, i64)], cursor: Option<&(String, i64)>) -> usiz /// Lowest slot per operator whose Pre-Kickoff can never confirm. The later /// slots extend it, so their own scans report it too and keep them Obsoleted -/// instead of following their own unspent outpoints. Graphs are scanned -/// lowest slot first, which rebuilds the map after a restart. +/// instead of following their own unspent outpoints. The map lives in memory: +/// after a restart [`seed_dead_slots`] rebuilds it before any Obsoleted graph is +/// scanned, since a later slot scanned first would otherwise look usable again. static DEAD_SLOTS: LazyLock>> = LazyLock::new(|| Mutex::new(HashMap::new())); +/// Operators whose Obsoleted slots were checked for a dead slot since the +/// start, and whether every operator has been. Kept across ticks, so a seed +/// cut short by the maintenance timeout resumes instead of starting over. +static SEEDED_OPERATORS: LazyLock>> = + LazyLock::new(|| Mutex::new(HashSet::new())); +static DEAD_SLOTS_SEEDED: AtomicBool = AtomicBool::new(false); + fn dead_slot_below(operator_pubkey: &str, slot: u64) -> Option { let dead = DEAD_SLOTS.lock().unwrap_or_else(|poisoned| poisoned.into_inner()); dead.get(operator_pubkey).copied().filter(|dead| *dead < slot) @@ -152,9 +161,82 @@ pub async fn scan_graphs_on_chain( graphs.into_iter().partition(|row| row.status == GraphStatus::Obsoleted.to_string()); let scan = ScanPass { local_db, btc_client, goat_client, engine, height }; scan.rows(&active, SCAN_BATCH, &SCAN_CURSOR, false).await?; + if !seed_dead_slots(local_db, btc_client, &obsoleted).await { + return Ok(()); + } scan.rows(&obsoleted, OBSOLETED_SCAN_BATCH, &OBSOLETED_SCAN_CURSOR, true).await } +/// Rebuild [`DEAD_SLOTS`] from the Obsoleted graphs (sorted by operator and +/// slot): per operator, the lowest slot whose Pre-Kickoff input another +/// confirmed transaction spent. Returns true once every operator is done; a +/// failed lookup leaves its operator for the next tick. +async fn seed_dead_slots(local_db: &LocalDB, btc_client: &BTCClient, obsoleted: &[Graph]) -> bool { + if DEAD_SLOTS_SEEDED.load(Ordering::Acquire) { + return true; + } + let mut complete = true; + let mut index = 0; + while index < obsoleted.len() { + let operator = obsoleted[index].operator_pubkey.clone(); + let end = obsoleted[index..] + .iter() + .position(|row| row.operator_pubkey != operator) + .map_or(obsoleted.len(), |offset| index + offset); + let rows = &obsoleted[index..end]; + index = end; + if SEEDED_OPERATORS + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .contains(&operator) + { + continue; + } + match lowest_dead_slot(local_db, btc_client, rows).await { + Ok(dead) => { + if let Some(slot) = dead { + record_dead_slot(&operator, slot); + } + SEEDED_OPERATORS + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(operator); + } + Err(error) => { + complete = false; + warn!(event = "graph_scan", operator_pubkey = %operator, error = %error, "dead slot seed failed; Obsoleted graphs wait for the next tick"); + } + } + } + if complete { + DEAD_SLOTS_SEEDED.store(true, Ordering::Release); + } + complete +} + +/// The lowest of `rows` (one operator, ascending slots) whose Pre-Kickoff can +/// never confirm. +async fn lowest_dead_slot( + local_db: &LocalDB, + btc_client: &BTCClient, + rows: &[Graph], +) -> anyhow::Result> { + for row in rows { + let definition = { + let mut storage = local_db.acquire().await?; + load_validated_graph_definition(&mut storage, row.instance_id, row.graph_id).await? + }; + let Some(definition) = definition else { + continue; + }; + let graph = BitvmGcGraph::from_simplified(&definition)?; + if prekickoff_input_spent(btc_client, &graph).await?.is_some() { + return Ok(Some(row.kickoff_index as u64)); + } + } + Ok(None) +} + struct ScanPass<'a> { local_db: &'a LocalDB, btc_client: &'a BTCClient, diff --git a/node/src/error.rs b/node/src/error.rs index e929f085..328badb1 100644 --- a/node/src/error.rs +++ b/node/src/error.rs @@ -16,6 +16,11 @@ pub enum SpecialError { #[error("Insufficient Balance: {0}")] InsufficientBalance(String), + + /// A backend has not caught up with a fact another source already proved + /// (a transaction Goat names that esplora does not know yet). Retried. + #[error("Backend Lag: {0}")] + BackendLag(String), } /// How a failed step is retried. @@ -50,7 +55,7 @@ pub fn permanent_rejection(error: &anyhow::Error) -> Option { | SpecialError::InvalidPeginData(message) | SpecialError::InvalidGraph(message) | SpecialError::InvalidCommittee(message) => Some(message.clone()), - SpecialError::InsufficientBalance(_) => None, + SpecialError::InsufficientBalance(_) | SpecialError::BackendLag(_) => None, }; } if let Some(message) = revert_message(cause) { @@ -79,9 +84,10 @@ fn revert_message(cause: &(dyn std::error::Error + 'static)) -> Option { fn is_transient(error: &anyhow::Error) -> bool { store::is_storage_error(error) || error.chain().any(|cause| { - cause.downcast_ref::().is_some() + matches!(cause.downcast_ref::(), Some(SpecialError::BackendLag(_))) + || cause.downcast_ref::().is_some() || cause.downcast_ref::().is_some() - || cause.downcast_ref::().is_some() + || cause.downcast_ref::().is_some_and(is_transient_io) || cause.downcast_ref::().is_some() || cause.downcast_ref::().is_some_and(transport_down) || cause.downcast_ref::().is_some_and(|contract| { @@ -91,6 +97,24 @@ fn is_transient(error: &anyhow::Error) -> bool { }) } +/// Network-shaped io errors pass; a missing file, a permission error or bad data +/// stays wrong on retry and must reach the give-up cap and its alert. +fn is_transient_io(error: &std::io::Error) -> bool { + use std::io::ErrorKind::*; + matches!( + error.kind(), + TimedOut + | ConnectionRefused + | ConnectionReset + | ConnectionAborted + | NotConnected + | BrokenPipe + | Interrupted + | WouldBlock + | UnexpectedEof + ) +} + fn transport_down(error: &alloy::transports::TransportError) -> bool { matches!( error, @@ -133,3 +157,26 @@ pub(crate) fn panic_message(payload: &(dyn std::any::Any + Send)) -> &str { "non-string panic payload" } } + +#[cfg(test)] +mod tests { + use super::*; + use std::io::{Error, ErrorKind}; + + #[test] + fn only_network_io_errors_are_transient() { + let wrapped = |kind| anyhow::Error::new(Error::from(kind)).context("read setup state"); + assert!(is_transient(&wrapped(ErrorKind::ConnectionReset))); + assert!(is_transient(&wrapped(ErrorKind::TimedOut))); + assert!(!is_transient(&wrapped(ErrorKind::NotFound))); + assert!(!is_transient(&wrapped(ErrorKind::PermissionDenied))); + assert!(!is_transient(&wrapped(ErrorKind::InvalidData))); + } + + #[test] + fn backend_lag_is_transient() { + let error = anyhow::Error::new(SpecialError::BackendLag("tx not seen".to_string())) + .context("validate graph"); + assert_eq!(classify_failure(&error), FailureClass::Transient); + } +} diff --git a/node/src/utils/graph_tx.rs b/node/src/utils/graph_tx.rs index 9c0b6d41..3aac6313 100644 --- a/node/src/utils/graph_tx.rs +++ b/node/src/utils/graph_tx.rs @@ -262,7 +262,10 @@ pub async fn build_graph_params( }) } -pub async fn operator_skip_graph(btc_client: &BTCClient, graph: &mut BitvmGcGraph) -> Result<()> { +/// Broadcasts the Pre-Kickoff and the Skip-Kickoff of a slot the operator gives up. +/// Returns false when the Kickoff connector cannot pay the Skip-Kickoff fee at the +/// current fee rate: then only the Pre-Kickoff goes out and the slot stays open. +pub async fn operator_skip_graph(btc_client: &BTCClient, graph: &mut BitvmGcGraph) -> Result { let graph_nonce = graph.parameters.graph_nonce; let operator_master_key = OperatorMasterKey::new(get_bitvm_key()?); let operator_master_keypair = operator_master_key.master_keypair(); @@ -310,21 +313,24 @@ pub async fn operator_skip_graph(btc_client: &BTCClient, graph: &mut BitvmGcGrap broadcast_tx(btc_client, &skip_kickoff_tx).await?; } } + Ok(true) } - None => match child_tx { - Some(tx) => { - if !tx_visible(btc_client, &prekickoff_txid).await? { - broadcast_package(btc_client, &[prekickoff_tx, tx], true).await?; + None => { + match child_tx { + Some(tx) => { + if !tx_visible(btc_client, &prekickoff_txid).await? { + broadcast_package(btc_client, &[prekickoff_tx, tx], true).await?; + } } - } - None => { - if !tx_visible(btc_client, &prekickoff_txid).await? { - broadcast_tx(btc_client, &prekickoff_tx).await?; + None => { + if !tx_visible(btc_client, &prekickoff_txid).await? { + broadcast_tx(btc_client, &prekickoff_tx).await?; + } } } - }, - }; - Ok(()) + Ok(false) + } + } } pub async fn operator_kickoff(btc_client: &BTCClient, graph: &mut BitvmGcGraph) -> Result<()> { diff --git a/node/src/utils/graph_validate.rs b/node/src/utils/graph_validate.rs index 78a84087..1b9aefb3 100644 --- a/node/src/utils/graph_validate.rs +++ b/node/src/utils/graph_validate.rs @@ -126,7 +126,10 @@ pub async fn validate_graph_instance_parameters( // The parameters already match GoatChain: a transaction the Bitcoin // backend does not know yet is the backend lagging, not a bad graph. let Some(funding_tx) = funding_tx else { - bail!("peg-in funding transaction {} not found on Bitcoin", input.outpoint.txid); + bail!(SpecialError::BackendLag(format!( + "peg-in funding transaction {} not found on Bitcoin", + input.outpoint.txid + ))); }; let funding_output = funding_tx.output.get(input.outpoint.vout as usize).ok_or_else(|| { @@ -160,7 +163,9 @@ pub async fn validate_graph_instance_parameters( .with_context(|| format!("failed to load peg-in deposit transaction {pegin_deposit_txid}"))? .is_none() { - bail!("peg-in deposit transaction {pegin_deposit_txid} not found on Bitcoin"); + bail!(SpecialError::BackendLag(format!( + "peg-in deposit transaction {pegin_deposit_txid} not found on Bitcoin" + ))); } Ok(()) diff --git a/node/src/utils/setup_state.rs b/node/src/utils/setup_state.rs index 1b2be61d..3a783d69 100644 --- a/node/src/utils/setup_state.rs +++ b/node/src/utils/setup_state.rs @@ -20,8 +20,8 @@ use bitvm_lib::babe_adapter::{ BabeVerifierPrivateState, CACSetupPackage, FinalizedInstanceData, Seed, SolderingData, compact_soldering_proof_payload, }; -use store::GraphStatus; -use tracing::warn; +use store::{EngineJobState, GraphStatus}; +use tracing::{info, warn}; use uuid::Uuid; const BABE_SETUP_STATE_ORPHAN_RETENTION: Duration = Duration::from_secs(24 * 60 * 60); @@ -194,6 +194,7 @@ pub(crate) async fn cleanup_babe_setup_states( let mut storage = local_db.acquire().await?; let now = SystemTime::now(); let mut deleted = 0; + let mut s3_payloads_left = 0; for instance_dir in instance_dirs { let instance_dir = instance_dir?; if !instance_dir.file_type()?.is_dir() { @@ -228,13 +229,21 @@ pub(crate) async fn cleanup_babe_setup_states( Some(status) } Some(graph) => { + // Not an orphan: the graph exists. Leave the mismatch for an operator. warn!( - "BABE setup state {graph_id} belongs to {instance_id}, but the stored graph belongs to {}", + "Skip BABE setup cleanup: state {graph_id} belongs to {instance_id}, but the stored graph belongs to {}", graph.instance_id ); + continue; + } + // No graph row yet: a setup is saved long before the operator presigns + // and the row exists. Only state with no live engine work is an orphan. + None => { + if !is_stale || graph_engine_is_live(&mut storage, graph_id, now).await? { + continue; + } None } - None => None, }; if !should_cleanup_babe_setup_state(status, is_stale) { continue; @@ -269,6 +278,12 @@ pub(crate) async fn cleanup_babe_setup_states( ready.candidate_index, &ready.payload_hash, )?; + if is_soldering_proof_s3_path(&payload_path) { + // Several nodes may share a bucket prefix; S3 objects are left to + // the bucket's lifecycle rules. + s3_payloads_left += 1; + continue; + } delete_soldering_proof_store_payload(&payload_path).await?; } std::fs::remove_file(&path) @@ -276,9 +291,39 @@ pub(crate) async fn cleanup_babe_setup_states( deleted += 1; } } + if s3_payloads_left > 0 { + info!( + s3_payloads_left, + "BABE setup cleanup left S3 soldering proof payloads to the bucket lifecycle" + ); + } Ok(deleted) } +/// Whether the graph still has engine work: a pending or running job, or an +/// aggregate updated within the orphan retention. +async fn graph_engine_is_live( + storage: &mut store::localdb::StorageProcessor<'_>, + graph_id: Uuid, + now: SystemTime, +) -> Result { + let aggregate_id = graph_id.to_string(); + let job_running = + storage.list_engine_jobs_for_aggregate("graph", &aggregate_id).await?.iter().any(|job| { + job.state == EngineJobState::Pending.to_string() + || job.state == EngineJobState::Running.to_string() + }); + if job_running { + return Ok(true); + } + let now_secs = now.duration_since(SystemTime::UNIX_EPOCH).unwrap_or_default().as_secs() as i64; + let retention = BABE_SETUP_STATE_ORPHAN_RETENTION.as_secs() as i64; + Ok(storage + .find_engine_aggregate("graph", &aggregate_id) + .await? + .is_some_and(|aggregate| now_secs.saturating_sub(aggregate.updated_at) < retention)) +} + #[derive(Clone, Default, Serialize, Deserialize)] pub struct BabeSetupState { pub verifier: Option, @@ -325,3 +370,54 @@ pub struct OperatorBabeSetupState { #[serde(default)] pub asserted_operator_proof: Option>, } + +#[cfg(test)] +mod cleanup_tests { + use super::*; + use store::NewEngineJob; + + fn age(path: &Path, by: Duration) { + let file = std::fs::File::options().write(true).open(path).unwrap(); + file.set_modified(SystemTime::now() - by).unwrap(); + } + + #[tokio::test] + async fn stale_state_without_a_graph_row_is_kept_while_its_engine_work_is_live() { + let dir = tempfile::tempdir().unwrap(); + let db = LocalDB::new_with_busy_timeout( + &format!("sqlite:{}", dir.path().join("cleanup.db").display()), + false, + store::DEFAULT_BUSY_TIMEOUT, + ) + .await; + db.migrate().await; + let (instance_id, graph_id) = (Uuid::new_v4(), Uuid::new_v4()); + save_babe_setup_state(&db, instance_id, graph_id, &BabeSetupState::default()).unwrap(); + let path = babe_setup_state_path(&db, instance_id, graph_id); + age(&path, BABE_SETUP_STATE_ORPHAN_RETENTION * 2); + + let job = NewEngineJob { + job_id: format!("graph:{graph_id}:setup"), + aggregate_kind: "graph".to_string(), + aggregate_id: graph_id.to_string(), + phase: "setup".to_string(), + action: "build_graph".to_string(), + pool: "default".to_string(), + params: Vec::new(), + deadline_at: None, + first_at: 0, + }; + db.acquire().await.unwrap().insert_engine_job(&job, 0).await.unwrap(); + assert_eq!(cleanup_babe_setup_states(&db, "/unused").await.unwrap(), 0); + assert!(path.exists(), "a setup with a pending job is not an orphan"); + + db.acquire() + .await + .unwrap() + .cancel_engine_jobs_for_aggregate("graph", &graph_id.to_string(), 0) + .await + .unwrap(); + assert_eq!(cleanup_babe_setup_states(&db, "/unused").await.unwrap(), 1); + assert!(!path.exists()); + } +}