diff --git a/.claude/skills/grafana-dashboards/SKILL.md b/.claude/skills/grafana-dashboards/SKILL.md index 617bc889..efa5634e 100644 --- a/.claude/skills/grafana-dashboards/SKILL.md +++ b/.claude/skills/grafana-dashboards/SKILL.md @@ -40,11 +40,11 @@ the iroh relay's in `monitoring/iroh-relay/`, the iroh DNS server's in process, iroh transport, DHT and probe, logs, traces. - `prometheus-alerts.yml` — three groups: `decdn-slash-safety`, `decdn-liveness`, `decdn-delivery`. Every rule carries a `component` label. Only rules with a matching - section in upstream's `docs/runbook.md` (20 of 57) also carry a `runbook_url`. + section in upstream's `docs/runbook.md` (21 of 58) also carry a `runbook_url`. `DecdnNodeDown`, `DecdnCapacityBondRegistryResyncFailing`, `DecdnBuyerTopupUnresolved`, `DecdnRedemptionMaybeBroadcast`, `DecdnNodeClockSkew`, - `DecdnSettlementLedgerWriteFailing`, `DecdnSettlementReanchorBacklog` and - `DecdnSettlementReanchorLowerDeposit` have promtool unit tests in + `DecdnSettlementLedgerWriteFailing`, `DecdnSettlementReanchorBacklog`, + `DecdnSettlementReanchorLowerDeposit` and `DecdnNodeBootStuck` have promtool unit tests in `prometheus-alerts_test.yml` (the chart's `.helmignore` keeps that file out of the package): keep them in step. diff --git a/monitoring/README.md b/monitoring/README.md index 055611bc..792372aa 100644 --- a/monitoring/README.md +++ b/monitoring/README.md @@ -16,7 +16,7 @@ this repo's MIT license. sponsord's were written here. | `decdn-node/dashboard-chain.json` | Chain, payments and slash safety (`uid: decdn-chain`): watcher liveness, chain RPC, registries, payments, settlement re-anchor. | | `decdn-node/dashboard-node.json` | Single-node drilldown (`uid: decdn-node`): host, process, iroh transport, DHT and probe, logs, traces. | | `decdn-node/prometheus-alerts.yml` | Rule groups `decdn-slash-safety`, `decdn-liveness`, `decdn-delivery`. A rule with a matching runbook section carries a `runbook_url` into upstream's `docs/runbook.md`. | -| `decdn-node/prometheus-alerts_test.yml` | promtool unit tests for `DecdnNodeDown`, `DecdnCapacityBondRegistryResyncFailing`, `DecdnBuyerTopupUnresolved`, `DecdnRedemptionMaybeBroadcast`, `DecdnNodeClockSkew`, `DecdnSettlementLedgerWriteFailing`, `DecdnSettlementReanchorBacklog` and `DecdnSettlementReanchorLowerDeposit` (run by `make lint-helm`; the chart's `.helmignore` keeps them out of the package). | +| `decdn-node/prometheus-alerts_test.yml` | promtool unit tests for `DecdnNodeDown`, `DecdnCapacityBondRegistryResyncFailing`, `DecdnBuyerTopupUnresolved`, `DecdnRedemptionMaybeBroadcast`, `DecdnNodeClockSkew`, `DecdnSettlementLedgerWriteFailing`, `DecdnSettlementReanchorBacklog`, `DecdnSettlementReanchorLowerDeposit` and `DecdnNodeBootStuck` (run by `make lint-helm`; the chart's `.helmignore` keeps them out of the package). | | `sponsord/` | The onboarding sponsor's dashboard and alert rules ([below](#sponsord)). Not rendered by the chart. | | `iroh-relay/` | The iroh relay's dashboard and alert rules ([below](#iroh-relay)). Not rendered by the chart. | | `iroh-dns-server/` | The iroh DNS server's dashboard and alert rules ([below](#iroh-dns-server)). Not rendered by the chart. | diff --git a/monitoring/decdn-node/dashboard-chain.json b/monitoring/decdn-node/dashboard-chain.json index 71224be7..169610cf 100644 --- a/monitoring/decdn-node/dashboard-chain.json +++ b/monitoring/decdn-node/dashboard-chain.json @@ -676,6 +676,15 @@ "expr": "max(decdn_fee_shares_watcher_unregistered{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", "refId": "D", "legendFormat": "fee-shares watcher unregistered" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "min(decdn_node_boot_ready{deployment_environment=\"$env\",region=~\"$region\",instance=~\"$instance\"})", + "refId": "E", + "legendFormat": "min boot ready" } ], "fieldConfig": { @@ -753,7 +762,7 @@ "y": 10 }, "id": 6, - "description": "Boot read retries counts transient-error retries of boot-time chain reads; the metrics listener binds after them, so it reads as a value since start, not a rate. A rise after a restart points at the RPC provider. Fee-shares watcher unregistered is 1 when the startup feeRouter() read failed: the watcher never started, its downtime gauge reads healthy, and the operator share stays at the floor until restart." + "description": "Boot read retries counts transient-error retries of boot-time chain reads since the process started; a rise after a restart points at the RPC provider. Min boot ready is 0 while any selected node is still in bring-up and 1 once all are ready; the metrics listener serves during boot, so a stuck boot shows here rather than as a down target. Fee-shares watcher unregistered is 1 when the startup feeRouter() read failed: the watcher never started, its downtime gauge reads healthy, and the operator share stays at the floor until restart." }, { "datasource": { diff --git a/monitoring/decdn-node/prometheus-alerts.yml b/monitoring/decdn-node/prometheus-alerts.yml index ea6446df..3205b12e 100644 --- a/monitoring/decdn-node/prometheus-alerts.yml +++ b/monitoring/decdn-node/prometheus-alerts.yml @@ -231,8 +231,8 @@ groups: See docs/runbook.md § Cache coalescing mutex poisoned. # The value, not increase(): the slash watcher's bring-up backfill counts - # every still-appealable slash before the first scrape, so increase() - # would miss a slash found at startup. A non-zero value means this process + # every still-appealable slash at startup, which can be before the first + # scrape, so increase() could miss a slash found then. A non-zero value means this process # has seen a slash; after a restart it re-counts only the slashes whose # appeal window is still open, so the alert clears once the last window # closes and the node restarts. @@ -530,6 +530,30 @@ groups: life of this process. Check the RPC provider, then restart the node. + # The metrics listener serves before bring-up, so a node stuck in boot + # scrapes fine (up == 1) and DecdnNodeDown stays silent: this rule covers + # it. A node whose boot chain reads keep failing exits at upstream's boot + # deadline and restarts, a crash loop whose failed scrapes write staleness + # markers; `max_over_time`, not the bare gauge, keeps the for running + # across them. It fires 10m (the for) after a first boot's first scrape, + # and up to 25m after a ready node restarts (its last 1 stays in the 15m + # window, then the for). A node that exits before its first scrape has no + # series at all; DecdnNodeDown covers that. + - alert: DecdnNodeBootStuck + expr: max_over_time(decdn_node_boot_ready[15m]) == 0 + for: 10m + labels: + component: node + severity: warning + annotations: + runbook_url: "https://github.com/decdn/decdn/blob/main/docs/runbook.md#rpc-unreachable" + summary: "{{ $labels.instance }} has not finished boot" + description: | + decdn_node_boot_ready has had no 1 sample for 25 m (or since its + first scrape, on a first boot, for 10 m). If decdn_chain_boot_read_retries_total rises, a boot chain + read is retrying. If it does not rise, the node is stuck before the + chain reads: read the first WARN or ERROR line after "deCDN node starting". + - alert: DecdnRpcUnhealthy expr: decdn_rpc_healthy == 0 for: 2m @@ -1303,8 +1327,8 @@ groups: for the buyer leg, decdn_node_pull_no_providers_total for discovery, and decdn_buyer_wallet_usdc for funding. - # Seeding runs once at cache open, before the first scrape, so the value - # is read rather than increase(). Only a restart clears it. + # Seeding runs once at cache open, which can be before the first scrape, + # so the value is read rather than increase(). Only a restart clears it. - alert: DecdnCacheRecencySeedFailed expr: decdn_cache_recency_seed_failures_total > 0 for: 0m diff --git a/monitoring/decdn-node/prometheus-alerts_test.yml b/monitoring/decdn-node/prometheus-alerts_test.yml index aa80db0a..575cac64 100644 --- a/monitoring/decdn-node/prometheus-alerts_test.yml +++ b/monitoring/decdn-node/prometheus-alerts_test.yml @@ -6,8 +6,10 @@ # DecdnBuyerTopupUnresolved and DecdnSettlementReanchorBacklog pin their for, # DecdnSettlementLedgerWriteFailing its 1h resolve and a failure already counted # at a node's first scrape, DecdnNodeClockSkew its window against its for (a -# one-off event stays quiet), and DecdnSettlementReanchorLowerDeposit a sparse -# retry, a healed reorg and retries that stop. The test clock starts at 0. The chart's .helmignore keeps +# one-off event stays quiet), DecdnSettlementReanchorLowerDeposit a sparse +# retry, a healed reorg and retries that stop, and DecdnNodeBootStuck its window +# (a lone 1 holds it off; a restart fires 25m after the last 1), a crash loop's +# staleness markers and its resolve. The test clock starts at 0. The chart's .helmignore keeps # this file out of the package. # # It does not run in place: render-test.sh copies it into a scratch directory as @@ -100,6 +102,86 @@ tests: alertname: DecdnNodeDown exp_alerts: [] + # Boot stuck. Range windows are left-open (Prometheus 3), so a sample leaves a + # 15m window at exactly 15m. a reads 0 from its first scrape (a first boot) and + # fires at exactly the 10m for. b reads its one 1 at 6m, so it is silent while + # that sample is in the window (checked at 10m and 20m); the 1 leaves at 21m + # and b fires at 31m. c was ready until 10m, restarted and is stuck at 0: its + # last 1 leaves at 25m and it fires at 35m, 25m after the last 1. d is a crash + # loop: 4 scrapes, then a failed one (a staleness marker) and a missed one. It + # fires at 10m, at a staleness marker, and keeps firing through the gap (11m) + # and later ones (34m); the bare gauge would restart its for at every marker. + # e boots for 20m and comes up: firing at 20m30s, resolved at its first 1 (21m). + - interval: 1m + input_series: + - series: 'decdn_node_boot_ready{job="decdn-node", instance="a"}' + values: '0x40' + - series: 'decdn_node_boot_ready{job="decdn-node", instance="b"}' + values: '0x5 1 0x34' + - series: 'decdn_node_boot_ready{job="decdn-node", instance="c"}' + values: '1x10 0x30' + - series: 'decdn_node_boot_ready{job="decdn-node", instance="d"}' + values: '0x3 stale _x1 0x3 stale _x1 0x3 stale _x1 0x3 stale _x1 0x3 stale _x1 0x3 stale _x1 0x3' + - series: 'decdn_node_boot_ready{job="decdn-node", instance="e"}' + values: '0x20 1x20' + alert_rule_test: + - eval_time: 9m30s + alertname: DecdnNodeBootStuck + exp_alerts: [] + - eval_time: 10m + alertname: DecdnNodeBootStuck + exp_alerts: + - exp_labels: {job: decdn-node, instance: a, component: node, severity: warning} + - exp_labels: {job: decdn-node, instance: d, component: node, severity: warning} + - exp_labels: {job: decdn-node, instance: e, component: node, severity: warning} + - eval_time: 11m + alertname: DecdnNodeBootStuck + exp_alerts: + - exp_labels: {job: decdn-node, instance: a, component: node, severity: warning} + - exp_labels: {job: decdn-node, instance: d, component: node, severity: warning} + - exp_labels: {job: decdn-node, instance: e, component: node, severity: warning} + - eval_time: 20m + alertname: DecdnNodeBootStuck + exp_alerts: + - exp_labels: {job: decdn-node, instance: a, component: node, severity: warning} + - exp_labels: {job: decdn-node, instance: d, component: node, severity: warning} + - exp_labels: {job: decdn-node, instance: e, component: node, severity: warning} + - eval_time: 20m30s + alertname: DecdnNodeBootStuck + exp_alerts: + - exp_labels: {job: decdn-node, instance: a, component: node, severity: warning} + - exp_labels: {job: decdn-node, instance: d, component: node, severity: warning} + - exp_labels: {job: decdn-node, instance: e, component: node, severity: warning} + - eval_time: 21m + alertname: DecdnNodeBootStuck + exp_alerts: + - exp_labels: {job: decdn-node, instance: a, component: node, severity: warning} + - exp_labels: {job: decdn-node, instance: d, component: node, severity: warning} + - eval_time: 30m30s + alertname: DecdnNodeBootStuck + exp_alerts: + - exp_labels: {job: decdn-node, instance: a, component: node, severity: warning} + - exp_labels: {job: decdn-node, instance: d, component: node, severity: warning} + - eval_time: 31m + alertname: DecdnNodeBootStuck + exp_alerts: + - exp_labels: {job: decdn-node, instance: a, component: node, severity: warning} + - exp_labels: {job: decdn-node, instance: b, component: node, severity: warning} + - exp_labels: {job: decdn-node, instance: d, component: node, severity: warning} + - eval_time: 34m + alertname: DecdnNodeBootStuck + exp_alerts: + - exp_labels: {job: decdn-node, instance: a, component: node, severity: warning} + - exp_labels: {job: decdn-node, instance: b, component: node, severity: warning} + - exp_labels: {job: decdn-node, instance: d, component: node, severity: warning} + - eval_time: 35m + alertname: DecdnNodeBootStuck + exp_alerts: + - exp_labels: {job: decdn-node, instance: a, component: node, severity: warning} + - exp_labels: {job: decdn-node, instance: b, component: node, severity: warning} + - exp_labels: {job: decdn-node, instance: c, component: node, severity: warning} + - exp_labels: {job: decdn-node, instance: d, component: node, severity: warning} + # Capacity-bond registry resync: attempts are 15m apart. a fails once (15m) and # never fires: one failure reads about 1.0. b fails twice (15m, 30m), so the # expression turns true at 30m and fires after the 15m for. It resolves once the