diff --git a/localnet/e2e/flows/backtoback.sh b/localnet/e2e/flows/backtoback.sh index de312307e..fde06fbe4 100755 --- a/localnet/e2e/flows/backtoback.sh +++ b/localnet/e2e/flows/backtoback.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash # Two peg-ins back to back with a slow verifier soldering (MOCK_GC_SOLDER_DELAY_SECS=E2E_BACKTOBACK_SOLDER_SECS: -# 120 s on anvil, 240 s on testnet3, whose committee answers take ~70 s longer). B is prepared while A's +# 180 s on anvil, 240 s on testnet3, whose committee answers take ~70 s longer). B is prepared while A's # ceremony still runs; the engine's cpu pool runs one GC job per node at a time (ENGINE_CPU_POOL_SLOTS, default 1), # so the verifier queues B's setup behind A's solder. Asserted: A is still in its ceremony on every node when B's # Prepare confirms, the verifier created B's setup job before A's solder finished, both end L2Minted / L2Recorded, # and their graphs take consecutive slots. source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh" "$ROOT/scripts/miner.sh" stop >/dev/null 2>&1 || true -SLOW=${E2E_BACKTOBACK_SOLDER_SECS:-$([[ $GOAT_BACKEND == testnet3 ]] && echo 240 || echo 120)} +SLOW=${E2E_BACKTOBACK_SOLDER_SECS:-$([[ $GOAT_BACKEND == testnet3 ]] && echo 240 || echo 180)} SOLDER_BEFORE=$(tuning_get MOCK_GC_SOLDER_DELAY_SECS); set_tuning MOCK_GC_SOLDER_DELAY_SECS "$SLOW" NODES_SH restart verifier_0 >/dev/null 2>&1; log "verifier_0 restarted with a $SLOW s solder delay" sleep 8 @@ -38,12 +38,15 @@ wait_all inst "$B" RelayerL1Broadcasted 240 || exit 1 mine 1 wait_all inst "$A" RelayerL2Minted 600 || exit 1 wait_all inst "$B" RelayerL2Minted 600 || exit 1 -gA=$(graph_of_instance "$A" | awk '{print $1}' | head -1); gB=$(graph_of_instance "$B" | awk '{print $1}' | head -1) -log "graphs: A $(graph_of_instance "$A" | tr '\n' ';') B $(graph_of_instance "$B" | tr '\n' ';')" +graphsA=$(graph_of_instance "$A") || exit 1 +graphsB=$(graph_of_instance "$B") || exit 1 +read -r gA kA _ <<<"$graphsA" +read -r gB kB _ <<<"$graphsB" +[[ -n "$gA" && -n "$gB" ]] || { log "FAIL: missing graph for A=$A or B=$B"; exit 1; } +log "graphs: A $(tr '\n' ';' <<<"$graphsA") B $(tr '\n' ';' <<<"$graphsB")" wait_all graph "$gA" L2Recorded 300 || exit 1 wait_all graph "$gB" L2Recorded 300 || exit 1 echo "$gB" > "$E2E_RUN/graph.txt" -kA=$(graph_of_instance "$A" | awk '{print $2}' | head -1); kB=$(graph_of_instance "$B" | awk '{print $2}' | head -1) bset=$(db verifier_0 "select created_at from engine_job where aggregate_id='$gB' and action='verifier_setup' order by created_at limit 1") asol=$(db verifier_0 "select finished_at from engine_job where aggregate_id='$gA' and action='verifier_solder' order by created_at limit 1") log "slots A=$kA B=$kB; verifier: B setup queued at $bset, A solder finished at $asol" diff --git a/localnet/e2e/flows/challenge.sh b/localnet/e2e/flows/challenge.sh index 23f59ecf2..110afd37c 100755 --- a/localnet/e2e/flows/challenge.sh +++ b/localnet/e2e/flows/challenge.sh @@ -40,7 +40,7 @@ wait_log operator_0 "$gid.*Prover-Assert broadcast\|Prover-Assert broadcast.*$gi wait_mempool 60 || exit 1 mine 1; A=$(tip); log "assert confirmed at A=$A" if [[ $MODE == valid ]]; then - wait_log verifier_0 "operator assert proof is valid" 240 || { engine_summary; exit 1; } + wait_log verifier_0 "$gid.*operator assert proof and committed pubin are valid\|operator assert proof and committed pubin are valid.*$gid" 240 || { engine_summary; exit 1; } for i in $(seq 1 15); do if fresh_log operator_0 | grep -q "Take-2 broadcast"; then break; fi sleep 20; mine 1 diff --git a/localnet/e2e/flows/pegin-restart.sh b/localnet/e2e/flows/pegin-restart.sh index a3cc66516..ed70fccbd 100755 --- a/localnet/e2e/flows/pegin-restart.sh +++ b/localnet/e2e/flows/pegin-restart.sh @@ -46,8 +46,11 @@ mine 1 wait_all inst "$id" RelayerL1Broadcasted 180 || exit 1 mine 1 wait_all inst "$id" RelayerL2Minted 600 || exit 1 -gid=$(graph_of_instance "$id" | awk '{print $1}' | head -1); echo "$gid" > "$E2E_RUN/graph.txt" -log "graph(s): $(graph_of_instance "$id" | tr '\n' ';')" +graphs=$(graph_of_instance "$id") || { log "graph query failed for instance $id"; exit 1; } +gid=$(awk 'NR==1 {print $1}' <<<"$graphs") +[[ -n "$gid" ]] || { log "no graph found for instance $id"; exit 1; } +echo "$gid" > "$E2E_RUN/graph.txt" +log "graph(s): $(tr '\n' ';' <<<"$graphs")" wait_all graph "$gid" L2Recorded 300 || exit 1 log "$role recovery lines: $(fresh_log $role | grep -E 'engine_recovery|lost its holder|re-claimed|resumed' | grep -oE '"message":"[^"]*"' | sort | uniq -c | tr '\n' '|')" engine_summary @@ -59,7 +62,8 @@ if [[ $jst != Done ]]; then log "FAIL: the interrupted job $jact did not end Don elif (( jatt < 2 && jfin > T_UP )); then log "FAIL: $jact finished after the restart without being re-run (attempts=$jatt)"; rc=1 elif (( jatt < 2 )); then log "$jact finished in the old process during its graceful shutdown" else log "$jact was re-run by the new process"; fi -ng=$(graph_of_instance "$id" | wc -l) +graphs=$(graph_of_instance "$id") || exit 1 +ng=$(awk 'NF {n++} END {print n+0}' <<<"$graphs") [[ $ng == 1 ]] || { log "FAIL: $ng graphs for the instance, expected 1"; rc=1; } nc=$(fresh_log committee_0 | jq -r --arg id "$id" 'select(.message=="Peg-In-Confirm broadcast" and .instance_id==$id) | .txid' 2>/dev/null | sort -u | wc -l) [[ $nc == 1 ]] || { log "FAIL: $nc distinct Peg-In-Confirm transactions for the instance, expected 1"; rc=1; } diff --git a/localnet/e2e/flows/pegin.sh b/localnet/e2e/flows/pegin.sh index 737629883..4485683d4 100755 --- a/localnet/e2e/flows/pegin.sh +++ b/localnet/e2e/flows/pegin.sh @@ -27,8 +27,11 @@ wait_all inst "$id" RelayerL1Broadcasted 180 || exit 1 mine 1 log "SPV fed; waiting for postPeginData" wait_all inst "$id" RelayerL2Minted 600 || exit 1 -gid=$(graph_of_instance "$id" | awk '{print $1}' | head -1); echo "$gid" > "$E2E_RUN/graph.txt" -log "graph(s): $(graph_of_instance "$id" | tr '\n' ';')" +graphs=$(graph_of_instance "$id") || { log "graph query failed for instance $id"; exit 1; } +gid=$(awk 'NR==1 {print $1}' <<<"$graphs") +[[ -n "$gid" ]] || { log "no graph found for instance $id"; exit 1; } +echo "$gid" > "$E2E_RUN/graph.txt" +log "graph(s): $(tr '\n' ';' <<<"$graphs")" wait_all graph "$gid" L2Recorded 300 || exit 1 engine_summary log "PEGIN DONE instance=$id graph=$gid tip=$(tip)" diff --git a/localnet/e2e/lib.sh b/localnet/e2e/lib.sh index c3c01390a..3aeee5193 100644 --- a/localnet/e2e/lib.sh +++ b/localnet/e2e/lib.sh @@ -75,18 +75,50 @@ alerts() { curl -s --max-time 5 "$(metrics_url "$1")" | grep '^bitvm_node_instan db() { sqlite3 -noheader -list -cmd ".timeout 8000" "$NODES_DIR/$1/bitvm-node.db" "$2"; } # warn_summary role -> new WARN/ERROR lines since the script started, grouped by event/outcome warn_summary() { fresh_log "$1" | grep -E ' (WARN|ERROR) ' | sed -E 's/\x1b\[[0-9;]*m//g' | grep -oE 'event=[a-z_]+( outcome=[a-z_]+)?|(WARN|ERROR) [^ ]+' | sort | uniq -c | sort -rn | head -20 | sed 's/^/ /'; } -graphs_on() { curl -s --max-time 5 "$(rpc_url "$1")/v1/graphs?limit=200" | python3 -c ' -import sys,json +# Fetch bounded pages and fail on HTTP/API errors before exposing any results. +graphs_json() { + python3 - "$(rpc_url "$1")" <<'PYGRAPH' +import json +import sys +import urllib.request + try: - for g in json.load(sys.stdin)["graphs"]: - g=g["graph"] - if g["instance_id"]=="'"$2"'": print(g["graph_id"]) -except Exception: pass'; } -graph_of_instance() { curl -s --max-time 5 "$(rpc_url committee_0)/v1/graphs?limit=200" | python3 -c ' -import sys,json -for g in json.load(sys.stdin)["graphs"]: - g=g["graph"] - if g["instance_id"]=="'"$1"'": print(g["graph_id"], g["kickoff_index"], g["status"])'; } + graphs = [] + offset = 0 + while True: + with urllib.request.urlopen( + f"{sys.argv[1]}/v1/graphs?limit=100&offset={offset}", timeout=10 + ) as response: + page = json.load(response) + rows = page["graphs"] + if not isinstance(rows, list): + raise ValueError("graphs must be a list") + graphs.extend(rows) + offset += len(rows) + if offset >= page["total"]: + break + if not rows: + raise ValueError("empty graph page before total was reached") + print(json.dumps(graphs)) +except Exception as error: + print(f"graph query failed: {error}", file=sys.stderr) + sys.exit(1) +PYGRAPH +} +graphs_on() { + local graphs; graphs=$(graphs_json "$1") || return 1 + python3 -c 'import sys,json +for item in json.load(sys.stdin): + g=item["graph"] + if g["instance_id"]==sys.argv[1]: print(g["graph_id"])' "$2" <<<"$graphs" +} +graph_of_instance() { + local graphs; graphs=$(graphs_json committee_0) || return 1 + python3 -c 'import sys,json +for item in json.load(sys.stdin): + g=item["graph"] + if g["instance_id"]==sys.argv[1]: print(g["graph_id"], g["kickoff_index"], g["status"])' "$1" <<<"$graphs" +} pegbtc_sats() { $CAST call $PEG "balanceOf(address)(uint256)" "$OP0_GOAT_ADDRESS" --rpc-url "$GOAT_CHAIN_URL" | awk '{printf "%d\n", $1/1e10}'; } topup_pegbtc() { # ensure the operator holds >= 1_100_000 sats of pegBTC local have; have=$(pegbtc_sats); if (( have < 1100000 )); then @@ -112,11 +144,13 @@ scenario_end() { log "=== SCENARIO $1 END"; snapshot_heights; engine_summary; lo # ---- e2e helpers ---- # next_posted_graph -> lowest-slot graph still L2Recorded on the operator (peg-outs go in slot order) -next_posted_graph() { curl -s --max-time 5 "$(rpc_url operator_0)/v1/graphs?limit=500" | python3 -c ' -import sys,json -gs=[g["graph"] for g in json.load(sys.stdin)["graphs"] if g["graph"]["status"]=="L2Recorded"] +next_posted_graph() { + local graphs; graphs=$(graphs_json operator_0) || return 1 + python3 -c 'import sys,json +gs=[item["graph"] for item in json.load(sys.stdin) if item["graph"]["status"]=="L2Recorded"] gs.sort(key=lambda g: g["kickoff_index"]) -print(gs[0]["graph_id"] if gs else "")'; } +print(gs[0]["graph_id"] if gs else "")' <<<"$graphs" +} # set_tuning VAR VALUE / clear_tuning VAR -> runtime override of one MOCK_* knob in run/tuning.env (sourced by # nodes.sh at launch, so it takes effect on the next (re)start of a node; localnet.env itself is never edited) set_tuning() { local f="$RUN_DIR/tuning.env"; touch "$f"; sed -i "/^export $1=/d" "$f"; echo "export $1=$2" >> "$f"; } diff --git a/localnet/e2e/run.sh b/localnet/e2e/run.sh index b0e8b6e63..fdce8ef11 100755 --- a/localnet/e2e/run.sh +++ b/localnet/e2e/run.sh @@ -29,7 +29,7 @@ for a in "$@"; do case $a in --list) printf '%s\n' "${CASES_ALL[@]}"; exit 0;; - for c in "${CASES[@]}"; do [[ " ${CASES_ALL[*]} " == *" $c "* ]] || { echo "unknown case: $c (see --list)" >&2; rm -rf "$RUN"; exit 2; }; done # ---- cases: each returns 0 on pass; graph ids for peg-out cases come from the lowest posted slot ---- -need_graph() { local g; g=$(next_posted_graph); if [[ -z $g ]]; then log "no posted graph; running a peg-in first" >&2; "$E/flows/pegin.sh" >&2 || return 1; g=$(cat "$E2E_RUN/graph.txt"); fi; echo "$g"; } +need_graph() { local g; g=$(next_posted_graph) || return 1; if [[ -z $g ]]; then log "no posted graph; running a peg-in first" >&2; "$E/flows/pegin.sh" >&2 || return 1; g=$(cat "$E2E_RUN/graph.txt"); fi; echo "$g"; } case_pegin_happy() { "$E/flows/pegin.sh"; } case_pegout_happy() { local g; g=$(need_graph) || return 1; "$E/flows/pegout.sh" "$g"; } case_challenge_take2() { local g; g=$(need_graph) || return 1; "$E/flows/challenge.sh" "$g" valid; } diff --git a/localnet/e2e/scenarios/N9.sh b/localnet/e2e/scenarios/N9.sh index af9956fd7..e4ac78326 100755 --- a/localnet/e2e/scenarios/N9.sh +++ b/localnet/e2e/scenarios/N9.sh @@ -2,7 +2,24 @@ # N9: subgraph latency 8 s (< 10 s timeout) on every node for a full peg-in. Exits with the peg-in flow's status. source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh" scenario_begin N9 subgraph-latency-8s pegin -"$CHAOS" fault all graph latency 8000 --ttl 2400 +# The fast localnet window (50 blocks = 100 s on Anvil) can expire before +# every committee discovers the request under 8 s latency. Use at least the +# upstream 200-block window for this scenario, then restore the live value. +window_before=$($CAST call $GW "responseWindowBlocks()(uint256)" --rpc-url "$GOAT_CHAIN_URL" | awk '{print $1}') || exit 1 +[[ $window_before =~ ^[0-9]+$ ]] || { log "invalid response window: $window_before"; exit 1; } +cleanup() { + local rc=$? + trap - EXIT + "$CHAOS" clear all graph >/dev/null 2>&1 || rc=1 + $CAST send $GW "debugSetResponseWindowBlocks(uint256)" "$window_before" --private-key "$C0_GOAT_PRIVATE_KEY" --rpc-url "$GOAT_CHAIN_URL" >/dev/null || rc=1 + exit "$rc" +} +trap cleanup EXIT +trap 'exit 130' INT TERM +window=$(( window_before > 200 ? window_before : 200 )) +$CAST send $GW "debugSetResponseWindowBlocks(uint256)" "$window" --private-key "$C0_GOAT_PRIVATE_KEY" --rpc-url "$GOAT_CHAIN_URL" >/dev/null || exit 1 +log "response window: $window blocks (restored to $window_before on exit)" +"$CHAOS" fault all graph latency 8000 --ttl 2400 || exit 1 "$ROOT/e2e/flows/pegin.sh"; rc=$? log "event watcher lag/ages:"; for r in committee_0 operator_0; do backend_probe "$r" | grep -E 'event_watch|ready'; done "$CHAOS" clear all graph diff --git a/localnet/e2e/test_graph_queries.py b/localnet/e2e/test_graph_queries.py new file mode 100644 index 000000000..a33cacbec --- /dev/null +++ b/localnet/e2e/test_graph_queries.py @@ -0,0 +1,74 @@ +"""Regression checks for graph pagination without a running localnet. + +Run: python3 -m unittest discover -s localnet/e2e -p 'test_*.py' +""" +import http.server +import json +from pathlib import Path +import subprocess +import threading +import unittest +from urllib.parse import parse_qs, urlparse + + +class GraphQueriesTest(unittest.TestCase): + def query(self, mode, function='graph_of_instance instance'): + requests = [] + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self): + params = parse_qs(urlparse(self.path).query) + offset, limit = int(params['offset'][0]), int(params['limit'][0]) + requests.append((offset, limit)) + self.send_response(400 if mode == 'http_error' or (mode == 'later_error' and offset) else 200) + self.end_headers() + rows = [{'graph': {'instance_id': 'instance', 'graph_id': str(i), + 'kickoff_index': i, 'status': 'L2Recorded'}} + for i in range(offset, min(offset + limit, 101))] + body = {'graphs': rows, 'total': 101} + if mode == 'http_error': + body = {'error': 'INVALID_LIMIT'} + elif mode == 'empty': + body['graphs'] = [] + elif mode == 'invalid': + body = {'error': 'unexpected API error'} + self.wfile.write(json.dumps(body).encode()) + + def log_message(self, *args): + pass + + source = Path(__file__).with_name('lib.sh').read_text() + helpers = source[source.index('graphs_json() {'):source.index('pegbtc_sats()')] + helpers += source[source.index('next_posted_graph() {'):source.index('# set_tuning')] + with http.server.ThreadingHTTPServer(('127.0.0.1', 0), Handler) as server: + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + script = f'rpc_url() {{ echo http://127.0.0.1:{server.server_port}; }}\n' + result = subprocess.run(['bash', '-c', script + helpers + '\n' + function], + capture_output=True, text=True, timeout=15) + server.shutdown() + worker.join() + return result, requests + + def test_all_pages_and_selection(self): + result, requests = self.query('ok') + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(requests, [(0, 100), (100, 100)]) + self.assertEqual(len(result.stdout.splitlines()), 101) + self.assertIn('100 100 L2Recorded', result.stdout) + result, _ = self.query('ok', 'graphs_on committee_0 instance') + self.assertEqual(result.stdout.splitlines(), [str(i) for i in range(101)]) + result, _ = self.query('ok', 'next_posted_graph') + self.assertEqual(result.stdout, '0\n') + + def test_query_errors_fail_without_results(self): + for mode in ('http_error', 'later_error', 'empty', 'invalid'): + with self.subTest(mode=mode): + result, _ = self.query(mode) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, '') + self.assertIn('graph query failed:', result.stderr) + + +if __name__ == '__main__': + unittest.main()