fix: readiness waits for decdn_node_boot_ready, not a 200

ansible/Makefile

@@ -199,7 +199,7 @@ decommission:
 # validation-sponsord (bad knobs, every role, must be rejected by their own asserts;
 # split by role so they run side by side), generate-keystore (opt-in host-side
 # wallet), host-env (host-provisioned /etc/decdn/decdn.env), slow-readiness (advisory
-# /metrics probe timeout), grafana-cloud and grafana-cloud-token (opt-in Grafana
+# boot-readiness probe timeout), grafana-cloud and grafana-cloud-token (opt-in Grafana
 # Cloud observability, host-provisioned vs inventory token), os-matrix (default's
 # plays on Debian 13, Ubuntu 24.04 and 26.04), lifecycle (network profile,
 # backup + decommission), sponsord, sponsord-install and sponsord-wallet (the sponsord

ansible/README.md

@@ -472,7 +472,7 @@ make lint-alloy     # render grafana_alloy's templates, then `alloy validate` th
 `validation-alloy` and `validation-sponsord` (bad knobs, for every role, must be rejected
 by their own asserts; one scenario per role group), `generate-keystore` (opt-in
 host-side wallet), `host-env` (host-provisioned `/etc/decdn/decdn.env`),
-`slow-readiness` (advisory `/metrics` probe timeout), `grafana-cloud` (the opt-in
+`slow-readiness` (advisory boot-readiness probe timeout), `grafana-cloud` (the opt-in
 observability wiring — see [Grafana Cloud observability](#grafana-cloud-observability-opt-in)),
 `grafana-cloud-token` (the same wiring with the API token carried through
 git-ignored inventory instead: role-authored env file + provenance record),

ansible/molecule/default/converge.yml

@@ -137,3 +137,22 @@
     - role: grafana_alloy
       tags: [observability, decdn_node, node]
     - role: decdn_node
+  post_tasks:
+    # The stub serves /metrics at once with decdn_node_boot_ready 0 until 3s
+    # after the first scrape, then 1 (decdn-node-stub's BOOT_SECONDS): the probe
+    # must end on the 1 and stop before its window runs out (attempts would then
+    # equal decdn_readiness_retries), with no warning. That it retries a 0 is
+    # slow-readiness's to prove: on the idempotence run here the node has long
+    # finished booting, so one attempt is right.
+    - name: Assert the readiness probe waited for decdn_node_boot_ready 1
+      ansible.builtin.assert:
+        that:
+          - decdn_metrics_probe.status == 200
+          - decdn_metrics_probe.content is search('(?m)^decdn_node_boot_ready 1$')
+          - decdn_metrics_probe.attempts | int < decdn_readiness_retries | int
+          - decdn_readiness_warning is skipped
+        fail_msg: >-
+          The readiness probe ended on status
+          {{ decdn_metrics_probe.status | default('none') }} after
+          {{ decdn_metrics_probe.attempts | default('?') }} attempt(s):
+          {{ decdn_metrics_probe.content | default('') }}

ansible/molecule/default/files/decdn-node-stub

@@ -6,19 +6,28 @@ can be exercised end-to-end without a published release or a live chain:
 
   * `--version`   -> print a version line and exit 0 (satisfies the role's
                      install-time `--version` backstop / version-match check).
-  * `run ...`     -> serve HTTP 200 on 127.0.0.1:9090/metrics (404 elsewhere) so
-                     the role's `/metrics` readiness probe succeeds, and block
-                     forever under systemd (Type=simple). CLI args (`--config`,
-                     `--keystore-password-file`) are accepted and ignored.
+  * `run ...`     -> serve HTTP 200 on 127.0.0.1:9090/metrics (404 elsewhere) and
+                     block forever under systemd (Type=simple). Like a release
+                     carrying decdn/decdn#2821, it binds at once and exports
+                     `decdn_node_boot_ready 0` until BOOT_SECONDS after the first
+                     scrape, then `1`, so the role's readiness probe always sees
+                     the flip, however slow the runner. CLI
+                     args (`--config`, `--keystore-password-file`) are accepted
+                     and ignored.
                      Under a unit with WatchdogSec it heartbeats WATCHDOG=1 to
                      NOTIFY_SOCKET every WATCHDOG_USEC/2, as the real daemon
                      does, so systemd does not kill it at the deadline.
-                     If the marker file NO_METRICS_MARKER exists, it instead
-                     blocks forever WITHOUT binding metrics — standing in for a
-                     healthy node whose metrics listener binds late (behind the
-                     startup PaymentPool bootstrap). The `slow-readiness`
-                     scenario uses this to prove the role's advisory probe warns
-                     (not fails) on timeout while the unit stays `running`.
+                     The first line of BOOT_MODE_MARKER, when the file exists,
+                     picks another boot:
+                       stuck      -> `decdn_node_boot_ready 0` forever (a boot
+                                     that never finishes; `slow-readiness`).
+                       legacy     -> no gauge at all, as v0.3.0 and v0.3.1,
+                                     which bind /metrics only after boot.
+                       no-metrics -> stays alive but never binds (a release
+                                     without the gauge, still booting, or a
+                                     daemon wedged before its listener).
+                       error      -> /metrics answers 500 (a broken handler,
+                                     which the role must fail on at once).
   * `config validate ...`
                   -> parse the rendered node.toml as TOML and exit 0. In the default
                      scenario it also checks the exact literal DECDN_RPC_URL passed
@@ -50,10 +59,13 @@ METRICS_ADDR = ("127.0.0.1", 9090)  # matches the role's default metrics_bind/po
 # Matches decdn_node_version in the cloud-init scenario's mirror. Printed after the
 # name it runs as (decdn-node or decdn), as clap prints the real binaries' line.
 VERSION = "0.0.0-molecule-stub"
-# When this file exists, `run` blocks WITHOUT binding metrics — simulating a
-# healthy node whose metrics listener binds late. The slow-readiness scenario
-# stages it (see that scenario's prepare.yml) to exercise the advisory-timeout path.
-NO_METRICS_MARKER = "/etc/decdn/stub-no-metrics"
+# How long a default boot reports `decdn_node_boot_ready 0`, from the first
+# scrape, before it flips to 1.
+BOOT_SECONDS = 3
+# Its first line picks a non-default boot (see the docstring); scenarios stage it
+# in prepare.yml.
+BOOT_MODE_MARKER = "/etc/decdn/stub-boot"
+BOOT_MODES = ("stuck", "legacy", "no-metrics", "error")
 # Forces `config validate` to exit 1, so a scenario can prove the role's config
 # gate still fails the deploy (the gate itself runs with failed_when: false).
 BAD_CONFIG_MARKER = "/etc/decdn/stub-bad-config"
@@ -67,7 +79,16 @@ class _Handler(BaseHTTPRequestHandler):
             self.send_response(404)
             self.end_headers()
             return
+        if self.server.boot_mode == "error":
+            self.send_response(500)
+            self.end_headers()
+            return
         body = b"# stub decdn-node metrics\ndecdn_stub_up 1\n"
+        if self.server.boot_mode != "legacy":
+            if self.server.ready_at is None:
+                self.server.ready_at = time.monotonic() + BOOT_SECONDS
+            ready = self.server.boot_mode != "stuck" and time.monotonic() >= self.server.ready_at
+            body += b"decdn_node_boot_ready %d\n" % ready
         self.send_response(200)
         self.send_header("Content-Type", "text/plain; version=0.0.4")
         self.send_header("Content-Length", str(len(body)))
@@ -122,10 +143,11 @@ def config_validate(args):
     key-level drift; see its README.
     """
     config = _opt(args, "--config")
-    # Forced-failure marker, mirroring NO_METRICS_MARKER. The role's config gate
-    # deliberately runs with `failed_when: false` and defers to a following `fail`
-    # task, so a regression in THAT task's `when:` would silently no-op the whole
-    # gate. This lets a scenario prove the gate still fires.
+    # Forced-failure marker (a presence check, unlike BOOT_MODE_MARKER's mode
+    # line). The role's config gate deliberately runs with `failed_when: false`
+    # and defers to a following `fail` task, so a regression in THAT task's
+    # `when:` would silently no-op the whole gate. This lets a scenario prove the
+    # gate still fires.
     if os.path.isfile(BAD_CONFIG_MARKER):
         print(f"stub config validate: refusing {config} (forced by "
               f"{BAD_CONFIG_MARKER})", file=sys.stderr)
@@ -185,6 +207,21 @@ def _start_watchdog_heartbeat():
     threading.Thread(target=beat, daemon=True).start()
 
 
+def boot_mode():
+    """'' when BOOT_MODE_MARKER is absent (the default boot), its mode, or None
+    when its first line names no mode (`run` then exits 1)."""
+    try:
+        with open(BOOT_MODE_MARKER, encoding="utf-8") as fh:
+            mode = fh.readline().strip()
+    except FileNotFoundError:
+        return ""
+    if mode not in BOOT_MODES:
+        print(f"stub: {BOOT_MODE_MARKER} names no boot mode: {mode!r} (want one of {BOOT_MODES})",
+              file=sys.stderr)
+        return None
+    return mode
+
+
 def main():
     args = sys.argv[1:]
     if "--version" in args:
@@ -196,17 +233,22 @@ def main():
         return config_validate(args[2:])
     if args and args[0] == "run":
         _start_watchdog_heartbeat()
-        # Late-bind simulation: stay alive (Type=simple => unit stays `running`)
-        # but never bind metrics, so the role's advisory probe times out.
-        if os.path.isfile(NO_METRICS_MARKER):
+        mode = boot_mode()
+        if mode is None:
+            return 1
+        if mode == "no-metrics":
+            # Stay alive (Type=simple => unit stays `running`) but never bind.
             while True:
                 time.sleep(3600)
         try:
-            HTTPServer(METRICS_ADDR, _Handler).serve_forever()
+            server = HTTPServer(METRICS_ADDR, _Handler)
         except OSError as exc:
             addr = f"{METRICS_ADDR[0]}:{METRICS_ADDR[1]}"
             print(f"stub: cannot bind {addr}: {exc}", file=sys.stderr)
             return 1
+        server.boot_mode = mode
+        server.ready_at = None  # set by the first scrape
+        server.serve_forever()
         return 0
     print(f"stub: unrecognised invocation: {args!r}", file=sys.stderr)
     return 2

ansible/molecule/grafana-cloud/converge.yml

@@ -6,9 +6,9 @@
 # sponsord_hosts), so Alloy must scrape both daemons. ../sponsord covers a
 # sponsord-only host.
 #
-# The stub does not serve /metrics, so decdn_node's readiness probe would sit out
-# its full advisory window (~60s of retries); decdn_readiness_* below bounds that
-# to ~3s without changing what the role asserts.
+# The stub reports decdn_node_boot_ready 0 for 3s after its first scrape, so
+# decdn_node's advisory readiness probe may warn here; decdn_readiness_* below
+# bounds its window to ~3s without changing what the role asserts.
 - name: Converge
   hosts: all
   become: true
@@ -21,7 +21,7 @@
     # environment labels exercise the "set" branch of the attribute rendering.
     grafana_alloy_deployment_environment: production
     grafana_alloy_region: US
-    # Keep the advisory metrics probe short (stub serves nothing).
+    # Keep the advisory readiness probe short (see the header).
     decdn_readiness_retries: 3
     decdn_readiness_delay: 1
 

ansible/molecule/host-env/converge.yml

@@ -31,3 +31,18 @@
     decdn_cache_origin_s3_use_default_chain: true
   roles:
     - role: decdn_node
+  post_tasks:
+    # prepare.yml boots the stub without the gauge: a 200 must count as ready
+    # (stopping the probe before its window runs out, when attempts would equal
+    # decdn_readiness_retries), not be polled for a gauge that never comes.
+    - name: Assert the readiness probe took a gauge-less 200 as ready
+      ansible.builtin.assert:
+        that:
+          - decdn_metrics_probe.status == 200
+          - decdn_metrics_probe.content is not search('(?m)^decdn_node_boot_ready[ {]')
+          - decdn_metrics_probe.attempts | int < decdn_readiness_retries | int
+          - decdn_readiness_warning is skipped
+        fail_msg: >-
+          Expected the probe to stop on a 200 without decdn_node_boot_ready, with no
+          warning; got status {{ decdn_metrics_probe.status | default('none') }}
+          after {{ decdn_metrics_probe.attempts | default('?') }} attempt(s).

ansible/molecule/host-env/prepare.yml

@@ -54,3 +54,12 @@
           DECDN_RPC_URL=https://rpc.example.invalid/
           DECDN_MOLECULE_HOST_LINE=written-by-the-operator
         mode: "0644"
+
+    # The stub's `legacy` boot (decdn-node-stub's BOOT_MODE_MARKER): no
+    # decdn_node_boot_ready line, as v0.3.0 and v0.3.1, which bind /metrics only
+    # once boot has finished. converge.yml asserts the role takes that 200 as ready.
+    - name: Boot the stub daemon like a release without decdn_node_boot_ready
+      ansible.builtin.copy:
+        dest: "{{ decdn_etc }}/stub-boot"
+        content: "legacy\n"
+        mode: "0644"

ansible/molecule/slow-readiness/converge.yml

@@ -1,34 +1,30 @@
 ---
-# Converge the decdn_node role against the stub in late-bind mode with a tiny
-# readiness window. The advisory /metrics probe MUST time out and WARN without
-# failing — so this play reaching the end (and the idempotence re-run) is itself
-# the core assertion of the fix. verify.yml then confirms the running-but-no-
-# metrics end state. Chain/contract knobs mirror the default scenario's
-# well-formed placeholders (the role's fail-loud asserts still run against them).
+# Converge the decdn_node role against the stub in its `stuck` boot with a tiny
+# readiness window (the role's variables are molecule.yml's inventory group_vars,
+# shared with side_effect.yml). The advisory probe MUST keep polling the 200 that
+# says decdn_node_boot_ready 0, then time out and WARN without failing — so this
+# play reaching the end (and the idempotence re-run) is itself the core
+# assertion; post_tasks prove the probe did not stop at the first 200.
+# verify.yml then confirms the running-but-booting end state.
 - name: Converge
   hosts: all
   become: true
-  vars:
-    # Reuse the default scenario's stub binary rather than duplicating it.
-    stub_bin: "{{ lookup('ansible.builtin.env', 'MOLECULE_PROJECT_DIRECTORY') }}/molecule/default/files/decdn-node-stub"
-    decdn_node_install_method: manual
-    decdn_node_manual_bin_src: "{{ stub_bin }}"
-    decdn_cli_manual_bin_src: "{{ stub_bin }}"
-    decdn_node_version: "0.0.0-molecule-stub"
-    decdn_rpc_url: "https://rpc.example.invalid/"
-    decdn_chain_id: 421614
-    decdn_region: "US"
-    decdn_payment_pool_address: "0x1111111111111111111111111111111111111111"
-    decdn_capacity_bond_address: "0x2222222222222222222222222222222222222222"
-    decdn_slash_judge_address: "0x3333333333333333333333333333333333333333"
-    # REQUIRED upstream (ADR 011/031) — the role asserts it with the other three.
-    decdn_content_blacklist_address: "0x4444444444444444444444444444444444444444"
-    decdn_cache_origin_kind: "http"
-    decdn_cache_origin_url: "https://origin.example.invalid/"
-    # Tiny probe window (~2s) so the advisory timeout is exercised fast. The stub
-    # never binds metrics in this scenario, so any window times out — this just
-    # keeps the scenario quick.
-    decdn_readiness_retries: 2
-    decdn_readiness_delay: 1
   roles:
     - role: decdn_node
+  post_tasks:
+    # A probe that took any 200 as ready would stop at its first attempt; one that
+    # reads the gauge retries until the window runs out on decdn_node_boot_ready 0
+    # (attempts then equals the retries), and its warning names the booting state.
+    - name: Assert the readiness probe kept polling a node that is still booting
+      ansible.builtin.assert:
+        that:
+          - decdn_metrics_probe.status == 200
+          - decdn_metrics_probe.content is search('(?m)^decdn_node_boot_ready 0$')
+          - decdn_metrics_probe.attempts | int == decdn_readiness_retries | int
+          - decdn_readiness_warning is not skipped
+          - "'`decdn_node_boot_ready 0`:' in decdn_readiness_warning.msg"
+        fail_msg: >-
+          Expected the probe to retry a 200 carrying decdn_node_boot_ready 0 until
+          its window ran out, then warn that the node is booting; got status
+          {{ decdn_metrics_probe.status | default('none') }} after
+          {{ decdn_metrics_probe.attempts | default('?') }} attempt(s).

ansible/molecule/slow-readiness/molecule.yml

@@ -1,11 +1,14 @@
 ---
-# Regression scenario for the ADVISORY /metrics readiness probe. Runs the stub
-# daemon in its "late bind" mode (it stays alive but never binds metrics — see
-# molecule/default/files/decdn-node-stub, gated on /etc/decdn/stub-no-metrics),
-# with a deliberately tiny readiness window. It proves the fix: the role's probe
-# WARNS (does not fail) on timeout, the deploy still succeeds, and the hard
-# service-state gate confirms the unit is `running`. Reuses the default
-# scenario's stub binary rather than duplicating it.
+# Regression scenario for the ADVISORY boot-readiness probe. Runs the stub daemon
+# in its `stuck` boot (it serves /metrics with decdn_node_boot_ready 0 forever —
+# see molecule/default/files/decdn-node-stub, BOOT_MODE_MARKER) with a
+# deliberately tiny readiness window. It proves the role keeps polling a 200 that
+# is still booting instead of taking it as ready, WARNS (does not fail) when the
+# window runs out, and that the hard service-state gate confirms the unit is
+# `running`. side_effect.yml re-runs the role against a daemon that never binds
+# /metrics (a release without the gauge, booting), which must stay advisory too,
+# and against a /metrics that answers 500, which must fail at once.
+# Reuses the default scenario's stub binary rather than duplicating it.
 #
 # `baseline` is NOT exercised here (same rationale as the default scenario:
 # host-level hardening is real-host-only — see molecule/default/molecule.yml).
@@ -37,6 +40,32 @@ provisioner:
     # round trip per remote task (measured at -22% on the `default` scenario, which
     # also carries the coverage note). Safe here: the containers have no sudo requiretty.
     ANSIBLE_PIPELINING: "true"
+  inventory:
+    group_vars:
+      # Inventory group_vars, not converge vars: side_effect.yml runs the role too.
+      # Chain/contract knobs mirror the default scenario's well-formed placeholders
+      # (the role's fail-loud asserts still run against them).
+      all:
+        # Reuse the default scenario's stub binary rather than duplicating it.
+        decdn_node_install_method: manual
+        decdn_node_manual_bin_src: "${MOLECULE_PROJECT_DIRECTORY}/molecule/default/files/decdn-node-stub"
+        decdn_cli_manual_bin_src: "${MOLECULE_PROJECT_DIRECTORY}/molecule/default/files/decdn-node-stub"
+        decdn_node_version: "0.0.0-molecule-stub"
+        decdn_rpc_url: "https://rpc.example.invalid/"
+        decdn_chain_id: 421614
+        decdn_region: "US"
+        decdn_payment_pool_address: "0x1111111111111111111111111111111111111111"
+        decdn_capacity_bond_address: "0x2222222222222222222222222222222222222222"
+        decdn_slash_judge_address: "0x3333333333333333333333333333333333333333"
+        # REQUIRED upstream (ADR 011/031) — the role asserts it with the other three.
+        decdn_content_blacklist_address: "0x4444444444444444444444444444444444444444"
+        decdn_cache_origin_kind: "http"
+        decdn_cache_origin_url: "https://origin.example.invalid/"
+        # Tiny probe window (~2s) so the advisory timeout is exercised fast. The
+        # stub never finishes booting (or never binds) here, so any window times
+        # out — this just keeps the scenario quick.
+        decdn_readiness_retries: 2
+        decdn_readiness_delay: 1
 verifier:
   name: ansible
 scenario:
@@ -48,5 +77,6 @@ scenario:
     - prepare
     - converge
     - idempotence
+    - side_effect
     - verify
     - destroy