Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions localnet/e2e/flows/backtoback.sh
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
#!/usr/bin/env bash
# Two peg-ins back to back with a slow verifier soldering (MOCK_GC_SOLDER_DELAY_SECS=E2E_BACKTOBACK_SOLDER_SECS:
# 120 s on anvil, 240 s on testnet3, whose committee answers take ~70 s longer). B is prepared while A's
# 180 s on anvil, 240 s on testnet3, whose committee answers take ~70 s longer). B is prepared while A's
# ceremony still runs; the engine's cpu pool runs one GC job per node at a time (ENGINE_CPU_POOL_SLOTS, default 1),
# so the verifier queues B's setup behind A's solder. Asserted: A is still in its ceremony on every node when B's
# Prepare confirms, the verifier created B's setup job before A's solder finished, both end L2Minted / L2Recorded,
# and their graphs take consecutive slots.
source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh"
"$ROOT/scripts/miner.sh" stop >/dev/null 2>&1 || true
SLOW=${E2E_BACKTOBACK_SOLDER_SECS:-$([[ $GOAT_BACKEND == testnet3 ]] && echo 240 || echo 120)}
SLOW=${E2E_BACKTOBACK_SOLDER_SECS:-$([[ $GOAT_BACKEND == testnet3 ]] && echo 240 || echo 180)}
SOLDER_BEFORE=$(tuning_get MOCK_GC_SOLDER_DELAY_SECS); set_tuning MOCK_GC_SOLDER_DELAY_SECS "$SLOW"
NODES_SH restart verifier_0 >/dev/null 2>&1; log "verifier_0 restarted with a $SLOW s solder delay"
sleep 8
Expand Down Expand Up @@ -38,12 +38,15 @@ wait_all inst "$B" RelayerL1Broadcasted 240 || exit 1
mine 1
wait_all inst "$A" RelayerL2Minted 600 || exit 1
wait_all inst "$B" RelayerL2Minted 600 || exit 1
gA=$(graph_of_instance "$A" | awk '{print $1}' | head -1); gB=$(graph_of_instance "$B" | awk '{print $1}' | head -1)
log "graphs: A $(graph_of_instance "$A" | tr '\n' ';') B $(graph_of_instance "$B" | tr '\n' ';')"
graphsA=$(graph_of_instance "$A") || exit 1
graphsB=$(graph_of_instance "$B") || exit 1
read -r gA kA _ <<<"$graphsA"
read -r gB kB _ <<<"$graphsB"
[[ -n "$gA" && -n "$gB" ]] || { log "FAIL: missing graph for A=$A or B=$B"; exit 1; }
log "graphs: A $(tr '\n' ';' <<<"$graphsA") B $(tr '\n' ';' <<<"$graphsB")"
wait_all graph "$gA" L2Recorded 300 || exit 1
wait_all graph "$gB" L2Recorded 300 || exit 1
echo "$gB" > "$E2E_RUN/graph.txt"
kA=$(graph_of_instance "$A" | awk '{print $2}' | head -1); kB=$(graph_of_instance "$B" | awk '{print $2}' | head -1)
bset=$(db verifier_0 "select created_at from engine_job where aggregate_id='$gB' and action='verifier_setup' order by created_at limit 1")
asol=$(db verifier_0 "select finished_at from engine_job where aggregate_id='$gA' and action='verifier_solder' order by created_at limit 1")
log "slots A=$kA B=$kB; verifier: B setup queued at $bset, A solder finished at $asol"
Expand Down
2 changes: 1 addition & 1 deletion localnet/e2e/flows/challenge.sh
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ wait_log operator_0 "$gid.*Prover-Assert broadcast\|Prover-Assert broadcast.*$gi
wait_mempool 60 || exit 1
mine 1; A=$(tip); log "assert confirmed at A=$A"
if [[ $MODE == valid ]]; then
wait_log verifier_0 "operator assert proof is valid" 240 || { engine_summary; exit 1; }
wait_log verifier_0 "$gid.*operator assert proof and committed pubin are valid\|operator assert proof and committed pubin are valid.*$gid" 240 || { engine_summary; exit 1; }
for i in $(seq 1 15); do
if fresh_log operator_0 | grep -q "Take-2 broadcast"; then break; fi
sleep 20; mine 1
Expand Down
10 changes: 7 additions & 3 deletions localnet/e2e/flows/pegin-restart.sh
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,11 @@ mine 1
wait_all inst "$id" RelayerL1Broadcasted 180 || exit 1
mine 1
wait_all inst "$id" RelayerL2Minted 600 || exit 1
gid=$(graph_of_instance "$id" | awk '{print $1}' | head -1); echo "$gid" > "$E2E_RUN/graph.txt"
log "graph(s): $(graph_of_instance "$id" | tr '\n' ';')"
graphs=$(graph_of_instance "$id") || { log "graph query failed for instance $id"; exit 1; }
gid=$(awk 'NR==1 {print $1}' <<<"$graphs")
[[ -n "$gid" ]] || { log "no graph found for instance $id"; exit 1; }
echo "$gid" > "$E2E_RUN/graph.txt"
log "graph(s): $(tr '\n' ';' <<<"$graphs")"
wait_all graph "$gid" L2Recorded 300 || exit 1
log "$role recovery lines: $(fresh_log $role | grep -E 'engine_recovery|lost its holder|re-claimed|resumed' | grep -oE '"message":"[^"]*"' | sort | uniq -c | tr '\n' '|')"
engine_summary
Expand All @@ -59,7 +62,8 @@ if [[ $jst != Done ]]; then log "FAIL: the interrupted job $jact did not end Don
elif (( jatt < 2 && jfin > T_UP )); then log "FAIL: $jact finished after the restart without being re-run (attempts=$jatt)"; rc=1
elif (( jatt < 2 )); then log "$jact finished in the old process during its graceful shutdown"
else log "$jact was re-run by the new process"; fi
ng=$(graph_of_instance "$id" | wc -l)
graphs=$(graph_of_instance "$id") || exit 1
ng=$(awk 'NF {n++} END {print n+0}' <<<"$graphs")
[[ $ng == 1 ]] || { log "FAIL: $ng graphs for the instance, expected 1"; rc=1; }
nc=$(fresh_log committee_0 | jq -r --arg id "$id" 'select(.message=="Peg-In-Confirm broadcast" and .instance_id==$id) | .txid' 2>/dev/null | sort -u | wc -l)
[[ $nc == 1 ]] || { log "FAIL: $nc distinct Peg-In-Confirm transactions for the instance, expected 1"; rc=1; }
Expand Down
7 changes: 5 additions & 2 deletions localnet/e2e/flows/pegin.sh
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,11 @@ wait_all inst "$id" RelayerL1Broadcasted 180 || exit 1
mine 1
log "SPV fed; waiting for postPeginData"
wait_all inst "$id" RelayerL2Minted 600 || exit 1
gid=$(graph_of_instance "$id" | awk '{print $1}' | head -1); echo "$gid" > "$E2E_RUN/graph.txt"
log "graph(s): $(graph_of_instance "$id" | tr '\n' ';')"
graphs=$(graph_of_instance "$id") || { log "graph query failed for instance $id"; exit 1; }
gid=$(awk 'NR==1 {print $1}' <<<"$graphs")
[[ -n "$gid" ]] || { log "no graph found for instance $id"; exit 1; }
echo "$gid" > "$E2E_RUN/graph.txt"
log "graph(s): $(tr '\n' ';' <<<"$graphs")"
wait_all graph "$gid" L2Recorded 300 || exit 1
engine_summary
log "PEGIN DONE instance=$id graph=$gid tip=$(tip)"
64 changes: 49 additions & 15 deletions localnet/e2e/lib.sh
Original file line number Diff line number Diff line change
Expand Up @@ -75,18 +75,50 @@ alerts() { curl -s --max-time 5 "$(metrics_url "$1")" | grep '^bitvm_node_instan
db() { sqlite3 -noheader -list -cmd ".timeout 8000" "$NODES_DIR/$1/bitvm-node.db" "$2"; }
# warn_summary role -> new WARN/ERROR lines since the script started, grouped by event/outcome
warn_summary() { fresh_log "$1" | grep -E ' (WARN|ERROR) ' | sed -E 's/\x1b\[[0-9;]*m//g' | grep -oE 'event=[a-z_]+( outcome=[a-z_]+)?|(WARN|ERROR) [^ ]+' | sort | uniq -c | sort -rn | head -20 | sed 's/^/ /'; }
graphs_on() { curl -s --max-time 5 "$(rpc_url "$1")/v1/graphs?limit=200" | python3 -c '
import sys,json
# Fetch bounded pages and fail on HTTP/API errors before exposing any results.
graphs_json() {
python3 - "$(rpc_url "$1")" <<'PYGRAPH'
import json
import sys
import urllib.request

try:
for g in json.load(sys.stdin)["graphs"]:
g=g["graph"]
if g["instance_id"]=="'"$2"'": print(g["graph_id"])
except Exception: pass'; }
graph_of_instance() { curl -s --max-time 5 "$(rpc_url committee_0)/v1/graphs?limit=200" | python3 -c '
import sys,json
for g in json.load(sys.stdin)["graphs"]:
g=g["graph"]
if g["instance_id"]=="'"$1"'": print(g["graph_id"], g["kickoff_index"], g["status"])'; }
graphs = []
offset = 0
while True:
with urllib.request.urlopen(
f"{sys.argv[1]}/v1/graphs?limit=100&offset={offset}", timeout=10
) as response:
page = json.load(response)
rows = page["graphs"]
if not isinstance(rows, list):
raise ValueError("graphs must be a list")
graphs.extend(rows)
offset += len(rows)
if offset >= page["total"]:
break
if not rows:
raise ValueError("empty graph page before total was reached")
print(json.dumps(graphs))
except Exception as error:
print(f"graph query failed: {error}", file=sys.stderr)
sys.exit(1)
PYGRAPH
}
graphs_on() {
local graphs; graphs=$(graphs_json "$1") || return 1
python3 -c 'import sys,json
for item in json.load(sys.stdin):
g=item["graph"]
if g["instance_id"]==sys.argv[1]: print(g["graph_id"])' "$2" <<<"$graphs"
}
graph_of_instance() {
local graphs; graphs=$(graphs_json committee_0) || return 1
python3 -c 'import sys,json
for item in json.load(sys.stdin):
g=item["graph"]
if g["instance_id"]==sys.argv[1]: print(g["graph_id"], g["kickoff_index"], g["status"])' "$1" <<<"$graphs"
}
pegbtc_sats() { $CAST call $PEG "balanceOf(address)(uint256)" "$OP0_GOAT_ADDRESS" --rpc-url "$GOAT_CHAIN_URL" | awk '{printf "%d\n", $1/1e10}'; }
topup_pegbtc() { # ensure the operator holds >= 1_100_000 sats of pegBTC
local have; have=$(pegbtc_sats); if (( have < 1100000 )); then
Expand All @@ -112,11 +144,13 @@ scenario_end() { log "=== SCENARIO $1 END"; snapshot_heights; engine_summary; lo

# ---- e2e helpers ----
# next_posted_graph -> lowest-slot graph still L2Recorded on the operator (peg-outs go in slot order)
next_posted_graph() { curl -s --max-time 5 "$(rpc_url operator_0)/v1/graphs?limit=500" | python3 -c '
import sys,json
gs=[g["graph"] for g in json.load(sys.stdin)["graphs"] if g["graph"]["status"]=="L2Recorded"]
next_posted_graph() {
local graphs; graphs=$(graphs_json operator_0) || return 1
python3 -c 'import sys,json
gs=[item["graph"] for item in json.load(sys.stdin) if item["graph"]["status"]=="L2Recorded"]
gs.sort(key=lambda g: g["kickoff_index"])
print(gs[0]["graph_id"] if gs else "")'; }
print(gs[0]["graph_id"] if gs else "")' <<<"$graphs"
}
# set_tuning VAR VALUE / clear_tuning VAR -> runtime override of one MOCK_* knob in run/tuning.env (sourced by
# nodes.sh at launch, so it takes effect on the next (re)start of a node; localnet.env itself is never edited)
set_tuning() { local f="$RUN_DIR/tuning.env"; touch "$f"; sed -i "/^export $1=/d" "$f"; echo "export $1=$2" >> "$f"; }
Expand Down
2 changes: 1 addition & 1 deletion localnet/e2e/run.sh
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ for a in "$@"; do case $a in --list) printf '%s\n' "${CASES_ALL[@]}"; exit 0;; -
for c in "${CASES[@]}"; do [[ " ${CASES_ALL[*]} " == *" $c "* ]] || { echo "unknown case: $c (see --list)" >&2; rm -rf "$RUN"; exit 2; }; done

# ---- cases: each returns 0 on pass; graph ids for peg-out cases come from the lowest posted slot ----
need_graph() { local g; g=$(next_posted_graph); if [[ -z $g ]]; then log "no posted graph; running a peg-in first" >&2; "$E/flows/pegin.sh" >&2 || return 1; g=$(cat "$E2E_RUN/graph.txt"); fi; echo "$g"; }
need_graph() { local g; g=$(next_posted_graph) || return 1; if [[ -z $g ]]; then log "no posted graph; running a peg-in first" >&2; "$E/flows/pegin.sh" >&2 || return 1; g=$(cat "$E2E_RUN/graph.txt"); fi; echo "$g"; }
case_pegin_happy() { "$E/flows/pegin.sh"; }
case_pegout_happy() { local g; g=$(need_graph) || return 1; "$E/flows/pegout.sh" "$g"; }
case_challenge_take2() { local g; g=$(need_graph) || return 1; "$E/flows/challenge.sh" "$g" valid; }
Expand Down
19 changes: 18 additions & 1 deletion localnet/e2e/scenarios/N9.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,24 @@
# N9: subgraph latency 8 s (< 10 s timeout) on every node for a full peg-in. Exits with the peg-in flow's status.
source "$(cd "$(dirname "$0")/.." && pwd)/lib.sh"
scenario_begin N9 subgraph-latency-8s pegin
"$CHAOS" fault all graph latency 8000 --ttl 2400
# The fast localnet window (50 blocks = 100 s on Anvil) can expire before
# every committee discovers the request under 8 s latency. Use at least the
# upstream 200-block window for this scenario, then restore the live value.
window_before=$($CAST call $GW "responseWindowBlocks()(uint256)" --rpc-url "$GOAT_CHAIN_URL" | awk '{print $1}') || exit 1
[[ $window_before =~ ^[0-9]+$ ]] || { log "invalid response window: $window_before"; exit 1; }
cleanup() {
local rc=$?
trap - EXIT
"$CHAOS" clear all graph >/dev/null 2>&1 || rc=1
$CAST send $GW "debugSetResponseWindowBlocks(uint256)" "$window_before" --private-key "$C0_GOAT_PRIVATE_KEY" --rpc-url "$GOAT_CHAIN_URL" >/dev/null || rc=1
exit "$rc"
}
trap cleanup EXIT
trap 'exit 130' INT TERM
window=$(( window_before > 200 ? window_before : 200 ))
$CAST send $GW "debugSetResponseWindowBlocks(uint256)" "$window" --private-key "$C0_GOAT_PRIVATE_KEY" --rpc-url "$GOAT_CHAIN_URL" >/dev/null || exit 1
log "response window: $window blocks (restored to $window_before on exit)"
"$CHAOS" fault all graph latency 8000 --ttl 2400 || exit 1
"$ROOT/e2e/flows/pegin.sh"; rc=$?
log "event watcher lag/ages:"; for r in committee_0 operator_0; do backend_probe "$r" | grep -E 'event_watch|ready'; done
"$CHAOS" clear all graph
Expand Down
74 changes: 74 additions & 0 deletions localnet/e2e/test_graph_queries.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
"""Regression checks for graph pagination without a running localnet.

Run: python3 -m unittest discover -s localnet/e2e -p 'test_*.py'
"""
import http.server
import json
from pathlib import Path
import subprocess
import threading
import unittest
from urllib.parse import parse_qs, urlparse


class GraphQueriesTest(unittest.TestCase):
def query(self, mode, function='graph_of_instance instance'):
requests = []

class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self):
params = parse_qs(urlparse(self.path).query)
offset, limit = int(params['offset'][0]), int(params['limit'][0])
requests.append((offset, limit))
self.send_response(400 if mode == 'http_error' or (mode == 'later_error' and offset) else 200)
self.end_headers()
rows = [{'graph': {'instance_id': 'instance', 'graph_id': str(i),
'kickoff_index': i, 'status': 'L2Recorded'}}
for i in range(offset, min(offset + limit, 101))]
body = {'graphs': rows, 'total': 101}
if mode == 'http_error':
body = {'error': 'INVALID_LIMIT'}
elif mode == 'empty':
body['graphs'] = []
elif mode == 'invalid':
body = {'error': 'unexpected API error'}
self.wfile.write(json.dumps(body).encode())

def log_message(self, *args):
pass

source = Path(__file__).with_name('lib.sh').read_text()
helpers = source[source.index('graphs_json() {'):source.index('pegbtc_sats()')]
helpers += source[source.index('next_posted_graph() {'):source.index('# set_tuning')]
with http.server.ThreadingHTTPServer(('127.0.0.1', 0), Handler) as server:
worker = threading.Thread(target=server.serve_forever, daemon=True)
worker.start()
script = f'rpc_url() {{ echo http://127.0.0.1:{server.server_port}; }}\n'
result = subprocess.run(['bash', '-c', script + helpers + '\n' + function],
capture_output=True, text=True, timeout=15)
server.shutdown()
worker.join()
return result, requests

def test_all_pages_and_selection(self):
result, requests = self.query('ok')
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(requests, [(0, 100), (100, 100)])
self.assertEqual(len(result.stdout.splitlines()), 101)
self.assertIn('100 100 L2Recorded', result.stdout)
result, _ = self.query('ok', 'graphs_on committee_0 instance')
self.assertEqual(result.stdout.splitlines(), [str(i) for i in range(101)])
result, _ = self.query('ok', 'next_posted_graph')
self.assertEqual(result.stdout, '0\n')

def test_query_errors_fail_without_results(self):
for mode in ('http_error', 'later_error', 'empty', 'invalid'):
with self.subTest(mode=mode):
result, _ = self.query(mode)
self.assertNotEqual(result.returncode, 0)
self.assertEqual(result.stdout, '')
self.assertIn('graph query failed:', result.stderr)


if __name__ == '__main__':
unittest.main()
Loading