Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 31 additions & 8 deletions node/src/aggregates/graph_jobs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ async fn run_step(
JOB_POST_GRAPH_DATA => post_graph_data(deps, graph_id, params).await,
JOB_FETCH_GRAPH => fetch_graph(deps, graph_id, params, attempts).await,
pegout if PEGOUT_JOBS.contains(&pegout) => {
run_pegout_step(deps, pegout, graph_id, params, attempts).await
run_pegout_step(deps, pegout, graph_id, params).await
}
other => bail!("unknown graph job {other}"),
}
Expand Down Expand Up @@ -615,9 +615,12 @@ async fn ingest_soldering_proof(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -
let Some(frozen) = operator_state.candidate_verifier_pubkeys.clone() else {
return Step::retry("operator verifier membership is not frozen", 30);
};
let verifier_pubkey = *frozen.get(candidate_index).ok_or_else(|| {
anyhow!("SolderingProofReady candidate index {candidate_index} out of range")
})?;
// The membership is frozen, so an index outside it can never become valid.
let Some(&verifier_pubkey) = frozen.get(candidate_index) else {
return invalid(format!(
"SolderingProofReady candidate index {candidate_index} out of range"
));
};
let candidate = operator_state
.candidates
.iter()
Expand Down Expand Up @@ -1561,6 +1564,8 @@ async fn verifier_validate_graph(deps: &JobDeps, graph_id: Uuid, params: &[u8])
else {
return Step::ignore("local verifier is not in this graph");
};
// Only a graph this verifier ran the setup for is worth the RPC checks and the
// graph rebuild below.
let Some(verifier_state) = load_babe_setup_state(&deps.local_db, instance_id, graph_id)?
.and_then(|state| state.verifier)
else {
Expand Down Expand Up @@ -1682,6 +1687,11 @@ async fn committee_ingest_graph(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -
if local_member(deps, instance_id).await?.is_none() {
return Step::ignore("not in this instance's committee");
}
// Checked first: validation retries a lagging Bitcoin backend without limit,
// and the gate ends those retries at T0 or when the instance fails.
if let Some(reason) = gate_closed(deps, instance_id).await? {
return Step::ignore(format!("signing gate closed: {reason}"));
}
// Structure, policy, prekickoff continuity, instance parameters against
// both chains, watchtower set and operator stake; verifier endorsements
// may arrive later.
Expand Down Expand Up @@ -1794,6 +1804,11 @@ async fn committee_nonces(deps: &JobDeps, graph_id: Uuid, _params: &[u8]) -> Res
graph.parameters.gc_data.len()
));
}
// Checked first: validation retries a lagging Bitcoin backend without limit,
// and the gate ends those retries at T0 or when the instance fails.
if let Some(reason) = gate_closed(deps, instance_id).await? {
return Step::ignore(format!("signing gate closed: {reason}"));
}
let validation = validate_init_graph(
&deps.local_db,
&deps.btc_client,
Expand All @@ -1804,9 +1819,6 @@ async fn committee_nonces(deps: &JobDeps, graph_id: Uuid, _params: &[u8]) -> Res
.await;
deps.metrics_state.record_graph_validation(validation.is_ok());
validation?;
if let Some(reason) = gate_closed(deps, instance_id).await? {
return Step::ignore(format!("signing gate closed: {reason}"));
}
let Some(instance_keypair) = local_member(deps, instance_id).await? else {
return Step::ignore("not in this instance's committee");
};
Expand Down Expand Up @@ -2363,11 +2375,22 @@ const FETCH_GRAPH_FAST_ATTEMPTS: i64 = 5;
const FETCH_GRAPH_SLOW_RETRY_SECS: i64 = 600;
const FETCH_GRAPH_MAX_ATTEMPTS: i64 = FETCH_GRAPH_FAST_ATTEMPTS
+ super::common::PARKED_MESSAGE_TTL_SECS / FETCH_GRAPH_SLOW_RETRY_SECS;
/// Slow retries between two alerts for a posted graph nobody serves (about a day).
const FETCH_GRAPH_POSTED_ALERT_EVERY: i64 = 24 * 60 * 60 / FETCH_GRAPH_SLOW_RETRY_SECS;

async fn fetch_graph(deps: &JobDeps, graph_id: Uuid, params: &[u8], attempts: i64) -> Result<Step> {
let lookup: GraphLookup = bincode::deserialize(params).context("decode graph lookup")?;
// A graph posted on Goat exists, and only the event watcher asks for it, once:
// that pull never gives up. It keeps retrying slowly and alerts about once a day.
let posted_on_chain = matches!(lookup, GraphLookup::Id { finalized: true, .. });
let give_up = |reason: String| {
if attempts + 1 >= FETCH_GRAPH_MAX_ATTEMPTS {
if posted_on_chain && attempts + 1 >= FETCH_GRAPH_FAST_ATTEMPTS {
if (attempts + 1 - FETCH_GRAPH_FAST_ATTEMPTS) % FETCH_GRAPH_POSTED_ALERT_EVERY == 0 {
warn!(event = "graph_fetch", graph_id = %graph_id, ?lookup, attempts, reason, "posted graph still not served by any peer");
deps.metrics_state.record_instance_alert("posted_graph_unavailable");
}
Step::retry(reason, FETCH_GRAPH_SLOW_RETRY_SECS)
} else if attempts + 1 >= FETCH_GRAPH_MAX_ATTEMPTS {
warn!(event = "graph_fetch", graph_id = %graph_id, ?lookup, attempts, reason, "giving up on the pull");
Step::ignore(format!("gave up pulling {lookup:?}: {reason}"))
} else if attempts + 1 >= FETCH_GRAPH_FAST_ATTEMPTS {
Expand Down
90 changes: 57 additions & 33 deletions node/src/aggregates/graph_pegout_jobs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ pub(super) async fn run_pegout_step(
action: &str,
graph_id: Uuid,
params: &[u8],
attempts: i64,
) -> Result<Step> {
match action {
JOB_KICKOFF => kickoff(deps, graph_id, params).await,
Expand All @@ -68,7 +67,7 @@ pub(super) async fn run_pegout_step(
let Some((row, graph)) = load_graph(&deps.local_db, graph_id).await? else {
return Step::ignore("graph not stored locally");
};
settlement_step(deps, action, &row, graph, signal, attempts).await
settlement_step(deps, action, &row, graph, signal).await
}
}
}
Expand All @@ -79,7 +78,6 @@ async fn settlement_step(
row: &Graph,
graph: BitvmGcGraph,
signal: PegoutSignal,
attempts: i64,
) -> Result<Step> {
use PegoutSignal::*;
// A closed graph's branches are over; it may still owe the relayer's Goat
Expand All @@ -97,7 +95,7 @@ async fn settlement_step(
}
match (action, signal) {
(JOB_ENFORCE_PREVIOUS_SLOT, _) => enforce_previous_slot(deps, &graph).await,
(JOB_PROCEED_WITHDRAW, _) => proceed_withdraw(deps, row, &graph, attempts).await,
(JOB_PROCEED_WITHDRAW, _) => proceed_withdraw(deps, row, &graph).await,
(JOB_CHALLENGE_KICKOFF, _) => challenge_kickoff(deps, &graph).await,
(JOB_WATCHTOWER_CHALLENGE_INIT, ChallengeSent { challenge_txid }) => {
watchtower_challenge_init(deps, row, graph, challenge_txid).await
Expand Down Expand Up @@ -139,9 +137,9 @@ async fn settlement_step(
.await
}
(JOB_TAKE1, _) => take1(deps, row, graph).await,
(JOB_FINISH_HAPPY, _) => finish_happy(deps, &graph, attempts).await,
(JOB_FINISH_HAPPY, _) => finish_happy(deps, &graph).await,
(JOB_TAKE2, _) => take2(deps, row, graph).await,
(JOB_FINISH_UNHAPPY, _) => finish_unhappy(deps, &graph, attempts).await,
(JOB_FINISH_UNHAPPY, _) => finish_unhappy(deps, &graph).await,
(action, signal) => bail!("{action} does not act on {signal:?}"),
}
}
Expand Down Expand Up @@ -372,7 +370,14 @@ async fn settle_predecessors(deps: &JobDeps, graph_id: Uuid, params: &[u8]) -> R
else {
bail!("slot {slot} definition disappeared");
};
operator_skip_graph(&deps.btc_client, &mut skipped).await?;
if !operator_skip_graph(&deps.btc_client, &mut skipped).await? {
let reason = format!(
"slot {slot}: Kickoff connector cannot pay the Skip-Kickoff fee at the current rate"
);
warn!(event = "graph_skip", graph_id = %graph_id, skipped_graph_id = %local.graph_id, slot, reason = %reason, "earlier slot cannot be skipped yet");
alert("pegout_predecessor_stuck");
return after_blocks(deps, reason, 10);
}
info!(
event = "graph_skip",
graph_id = %graph_id,
Expand Down Expand Up @@ -1720,38 +1725,57 @@ async fn watchtower_proof_timeout(row: &Graph, graph: &BitvmGcGraph) -> Result<S
// ------------------------------------------------------------------ relayer

/// Kickoff confirmed and covered by the SPV client: `proceedWithdraw`.
/// Blocks a relayer job waits for a lagging Goat node to show the withdraw
/// before it gives up on it.
const WITHDRAW_LAG_RETRIES: i64 = 30;
/// Runs (six blocks apart) a finish job waits for `proceedWithdraw` to land.
const WITHDRAW_PROCEED_RETRIES: i64 = 60;
/// Bitcoin blocks past the settling transaction a relayer job waits for a
/// lagging Goat node to show the withdraw before it gives up on it.
const WITHDRAW_LAG_BLOCKS: u64 = 30;
/// Bitcoin blocks past the settling transaction a finish job waits for
/// `proceedWithdraw` to land.
const WITHDRAW_PROCEED_BLOCKS: u64 = 360;

/// Confirmations of `txid`, or None while it is unconfirmed. The give-up
/// windows below count chain depth, not job attempts: a Goat or Bitcoin RPC
/// outage retries the job many times without the chain moving.
async fn confirmations(deps: &JobDeps, txid: &Txid) -> Result<Option<u64>> {
let Some(height) = deps.btc_client.get_tx_status(txid).await?.block_height else {
return Ok(None);
};
let tip = deps.btc_client.get_height().await?;
Ok(Some(u64::from(tip.saturating_add(1).saturating_sub(height))))
}

/// For proceed/happy/unhappy jobs, wait for the withdraw to become visible.
/// Disproved settlement must not use this gate: it also handles rogue kickoffs
/// without initWithdraw. These jobs run once per graph, so an early
/// `ignore` would leave the Goat side behind for good.
fn withdraw_missing(deps: &JobDeps, graph_id: Uuid, call: &str, attempts: i64) -> Result<Step> {
if attempts < WITHDRAW_LAG_RETRIES {
fn withdraw_missing(
deps: &JobDeps,
graph_id: Uuid,
call: &str,
depth: Option<u64>,
) -> Result<Step> {
if depth.is_none_or(|depth| depth < WITHDRAW_LAG_BLOCKS) {
return after_blocks(deps, format!("withdraw is not visible on Goat yet; {call} waits"), 1);
}
warn!(event = "graph_withdraw", graph_id = %graph_id, call, attempts, "withdraw never appeared on Goat; giving up");
warn!(event = "graph_withdraw", graph_id = %graph_id, call, ?depth, "withdraw never appeared on Goat; giving up");
alert("pegout_withdraw_missing");
Step::ignore(format!("withdraw not on Goat after {attempts} attempts; {call} skipped"))
Step::ignore(format!("withdraw not on Goat {depth:?} blocks after settlement; {call} skipped"))
}

/// `proceedWithdraw` has not landed: wait for it, but not forever.
fn withdraw_not_proceeded(
deps: &JobDeps,
graph_id: Uuid,
call: &str,
attempts: i64,
depth: Option<u64>,
) -> Result<Step> {
if attempts < WITHDRAW_PROCEED_RETRIES {
if depth.is_none_or(|depth| depth < WITHDRAW_PROCEED_BLOCKS) {
return after_blocks(deps, "withdraw is initialized but the kickoff is not posted", 6);
}
warn!(event = "graph_withdraw", graph_id = %graph_id, call, attempts, "proceedWithdraw never landed; giving up");
warn!(event = "graph_withdraw", graph_id = %graph_id, call, ?depth, "proceedWithdraw never landed; giving up");
alert("pegout_withdraw_stuck");
Step::ignore(format!("withdraw still Initialized after {attempts} attempts; {call} skipped"))
Step::ignore(format!(
"withdraw still Initialized {depth:?} blocks after settlement; {call} skipped"
))
}

/// Goat shows a status the chain contradicts: report it, nothing to send.
Expand All @@ -1766,12 +1790,7 @@ fn withdraw_inconsistent(
Step::ignore(format!("withdraw is {status:?} although {settled_by} confirmed; {call} skipped"))
}

async fn proceed_withdraw(
deps: &JobDeps,
row: &Graph,
graph: &BitvmGcGraph,
attempts: i64,
) -> Result<Step> {
async fn proceed_withdraw(deps: &JobDeps, row: &Graph, graph: &BitvmGcGraph) -> Result<Step> {
if !deps.relayer {
return Step::ignore("not a relayer");
}
Expand All @@ -1784,7 +1803,8 @@ async fn proceed_withdraw(
return Step::ignore("graph closed without a withdraw on Goat");
}
WithdrawStatus::None => {
return withdraw_missing(deps, graph_id, "proceedWithdraw", attempts);
let depth = confirmations(deps, &graph.kickoff.tx().compute_txid()).await?;
return withdraw_missing(deps, graph_id, "proceedWithdraw", depth);
}
WithdrawStatus::Canceled => {
// Cancelling needs the committee; a cancelled withdraw is handled by hand.
Expand Down Expand Up @@ -1833,7 +1853,7 @@ async fn spv_covered(deps: &JobDeps, txid: &Txid, what: &str) -> Result<Option<S
}

/// Take-1 confirmed: `finishWithdrawHappyPath`.
async fn finish_happy(deps: &JobDeps, graph: &BitvmGcGraph, attempts: i64) -> Result<Step> {
async fn finish_happy(deps: &JobDeps, graph: &BitvmGcGraph) -> Result<Step> {
if !deps.relayer {
return Step::ignore("not a relayer");
}
Expand All @@ -1846,10 +1866,12 @@ async fn finish_happy(deps: &JobDeps, graph: &BitvmGcGraph, attempts: i64) -> Re
match withdraw_status {
WithdrawStatus::Processing => {}
WithdrawStatus::Initialized => {
return withdraw_not_proceeded(deps, graph_id, "finishWithdrawHappyPath", attempts);
let depth = confirmations(deps, &take1_txid).await?;
return withdraw_not_proceeded(deps, graph_id, "finishWithdrawHappyPath", depth);
}
WithdrawStatus::None => {
return withdraw_missing(deps, graph_id, "finishWithdrawHappyPath", attempts);
let depth = confirmations(deps, &take1_txid).await?;
return withdraw_missing(deps, graph_id, "finishWithdrawHappyPath", depth);
}
WithdrawStatus::Complete => return Step::ignore("withdraw already complete on Goat"),
other => {
Expand All @@ -1867,7 +1889,7 @@ async fn finish_happy(deps: &JobDeps, graph: &BitvmGcGraph, attempts: i64) -> Re
}

/// Take-2 confirmed: `finishWithdrawUnhappyPath`.
async fn finish_unhappy(deps: &JobDeps, graph: &BitvmGcGraph, attempts: i64) -> Result<Step> {
async fn finish_unhappy(deps: &JobDeps, graph: &BitvmGcGraph) -> Result<Step> {
if !deps.relayer {
return Step::ignore("not a relayer");
}
Expand All @@ -1880,10 +1902,12 @@ async fn finish_unhappy(deps: &JobDeps, graph: &BitvmGcGraph, attempts: i64) ->
match withdraw_status {
WithdrawStatus::Processing => {}
WithdrawStatus::Initialized => {
return withdraw_not_proceeded(deps, graph_id, "finishWithdrawUnhappyPath", attempts);
let depth = confirmations(deps, &take2_txid).await?;
return withdraw_not_proceeded(deps, graph_id, "finishWithdrawUnhappyPath", depth);
}
WithdrawStatus::None => {
return withdraw_missing(deps, graph_id, "finishWithdrawUnhappyPath", attempts);
let depth = confirmations(deps, &take2_txid).await?;
return withdraw_missing(deps, graph_id, "finishWithdrawUnhappyPath", depth);
}
WithdrawStatus::Complete => return Step::ignore("withdraw already complete on Goat"),
other => {
Expand Down
Loading
Loading