From d9d374b255fbae9d3061ae2dc81da76ce7412065 Mon Sep 17 00:00:00 2001 From: dnth Date: Fri, 16 Oct 2026 17:07:52 +0800 Subject: [PATCH 1/4] fix(boat-lanes): honor lane policies in one run Use each lane TTL for spend reservation and box creation. Avoid retries for listed failed case IDs and allow opted-in ordered lanes to continue after non-pass selections without changing legacy defaults. --- bin/fm-boat-lanes.py | 85 +++++++++++----- tests/boat-lane-runner-cases.py | 162 +++++++++++++++++++++++++++++- tests/boat-lane-runner-fixture.py | 5 +- 3 files changed, 222 insertions(+), 30 deletions(-) diff --git a/bin/fm-boat-lanes.py b/bin/fm-boat-lanes.py index 8ca78b2ed65..e7ce45767b7 100755 --- a/bin/fm-boat-lanes.py +++ b/bin/fm-boat-lanes.py @@ -12,19 +12,28 @@ name is fm-lanes--. Never adopt or delete another run's box. Spec v1 is JSON: {"version":1,"size":"large","ttl_seconds":1800, - "pilot":"pilot","env":{"NAME":"value"},"prepare":[STEP],"guard":STEP, - "lanes":[{"id":"pilot","selections":[STEP]}],"artifacts":["/tmp/results"], - "finish":[STEP]}. STEP is {"command":"shell script","timeout_seconds":60} - or, in prepare only, {"upload":"/absolute/local/file","destination":"/tmp/file", - "timeout_seconds":60}. size accepts fm-boat.py's RATES keys; ttl_seconds is an - integer from 60 through 86400. Lane IDs are unique and match - [a-zA-Z0-9][a-zA-Z0-9_-]{0,31}; pilot names one of them. Each lane has a nonempty - selections list and an optional report-only profile (default "default"). - prepare, finish, artifacts and env default to empty; known_harness_defects and - infra_signatures are optional top-level lists of nonempty literal substrings. - Every step needs a positive finite timeout_seconds. env maps shell variable - names to strings without NUL; FM_ names are reserved. Artifact and evidence - paths must be beneath /tmp without '..' components. Upload sources must exist. + "pilot":"pilot","known_product_failures":["case-id"],"env":{"NAME":"value"}, + "prepare":[STEP],"guard":STEP,"lanes":[{"id":"pilot","ttl_seconds":3600, + "continue_on_failure":true,"selections":[STEP]}],"artifacts":["/tmp/results"], + "finish":[STEP]}. +STEP is {"command":"shell script","timeout_seconds":60} or, in prepare only, + {"upload":"/absolute/local/file","destination":"/tmp/file","timeout_seconds":60}. +size accepts fm-boat.py's RATES keys; spec-wide ttl_seconds is an integer from +60 through 86400. Lane IDs are unique and match [a-zA-Z0-9][a-zA-Z0-9_-]{0,31}; +pilot names one of them. Each lane has nonempty selections, an optional +report-only profile (default "default"), and an optional ttl_seconds override +using the same range; omitted lane TTLs inherit the spec value. +continue_on_failure is an optional boolean that defaults to false. When true, +later selections continue after a selection reports failed or blocked cases; +harness errors, timeouts, cancellation, and invalid evidence still stop the lane. +prepare, finish, artifacts and env default to empty; known_harness_defects and +infra_signatures are optional top-level lists of nonempty literal substrings. +known_product_failures is an optional top-level list of unique case IDs matching +[A-Za-z0-9_.:-]{1,120}; failed matching cases are product failures and are never retried. +Dependency-blocked evidence keeps its existing classification. +Every step needs a positive finite timeout_seconds. env maps shell variable +names to strings without NUL; FM_ names are reserved. Artifact and evidence +paths must be beneath /tmp without '..' components. Upload sources must exist. Keep project recipes private under the operator home's config/boat-lanes/; tests/boat-lane-runner-cases.py contains offline fake-provider spec examples. Commands execute remotely in bash with strict errors, @@ -372,6 +381,12 @@ def validate(spec): steps += [(spec.get("guard"), False)] steps += [(step, False) for step in spec.get("finish", [])] for lane in lanes: + lane_ttl = lane.get("ttl_seconds", ttl) + if not isinstance(lane_ttl, int) or isinstance(lane_ttl, bool) or not 60 <= lane_ttl <= 86400: + raise Refusal("lane TTL must be 60..86400 seconds") + continue_on_failure = lane.get("continue_on_failure", False) + if type(continue_on_failure) is not bool: + raise Refusal("continue_on_failure must be a boolean") selections = lane.get("selections") if not isinstance(selections, list) or not selections: raise Refusal("each lane requires real selections") @@ -419,11 +434,19 @@ def validate(spec): if not isinstance(spec.get(key, []), list) or any(not isinstance(x, str) or not x for x in spec.get(key, [])): raise Refusal(f"{key} must contain nonempty literal signatures") + known_products = spec.get("known_product_failures", []) + if (not isinstance(known_products, list) + or any(not isinstance(value, str) + or not re.fullmatch(r"[A-Za-z0-9_.:-]{1,120}", value) + for value in known_products) + or len(set(known_products)) != len(known_products)): + raise Refusal("known_product_failures must be unique case IDs") class Runner: def __init__(self, args, spec): self.args, self.spec = args, spec + self.known_product_failures = frozenset(spec.get("known_product_failures", [])) self.stop = threading.Event() self.executor = Executor(self.stop) # Use the existing Boat client's JSON/API/credential implementation. @@ -464,19 +487,21 @@ def handle_signal(self, number, _frame): self.stop.set() def create(self, lane): + lane_id = lane["id"] + ttl = lane.get("ttl_seconds", self.spec["ttl_seconds"]) with self.lock: if self.stop.is_set() or self.systemic: raise Interrupted() - reserve = client.RATES[self.spec["size"]] * self.spec["ttl_seconds"] / 3600 + reserve = client.RATES[self.spec["size"]] * ttl / 3600 if self.spent + self.reserved + reserve > self.args.cap_usd + 1e-12: - self.event("refused", lane=lane, reason="projected spend exceeds cap", + self.event("refused", lane=lane_id, reason="projected spend exceeds cap", spent_usd=self.spent, reserved_usd=self.reserved, new_usd=reserve) self.stop.set() raise Refusal("projected spend exceeds cap") - name = f"fm-lanes-{self.token}-{lane}" + name = f"fm-lanes-{self.token}-{lane_id}" self.reserved += reserve - self.event("create_intent", lane=lane, name=name, reserved_usd=reserve, - ttl_seconds=self.spec["ttl_seconds"]) + self.event("create_intent", lane=lane_id, name=name, reserved_usd=reserve, + ttl_seconds=ttl) # Do not interrupt creation between its response and durable ID publication. # If transport dies ambiguously, the intent/name and reservation remain. cleaning = self.executor.cleaning @@ -484,16 +509,17 @@ def create(self, lane): try: body = client.cli("new", "--json", "--personal", "--no-env", "--environment", "base", "--type", self.spec["size"], - "--ttl", str(self.spec["ttl_seconds"]), "--no-update") + "--ttl", str(ttl), "--no-update") nested = body.get("sandbox", {}) box = body.get("id") or (nested.get("id") if isinstance(nested, dict) else None) if not isinstance(box, str) or not re.fullmatch(r"bx_[A-Za-z0-9]+", box): raise Refusal("creation returned invalid identity; inspect create intent") - self.boxes[box] = dict(lane=lane, name=name, reserve=reserve, deleted=False) - self.event("created", lane=lane, box=box, name=name, reserved_usd=reserve) + self.boxes[box] = dict(lane=lane_id, name=name, reserve=reserve, + ttl_seconds=ttl, deleted=False) + self.event("created", lane=lane_id, box=box, name=name, reserved_usd=reserve) except Exception: self.stop.set() # An ambiguous create must never be followed by fan-out. - self.event("creation_uncertain", lane=lane, name=name) + self.event("creation_uncertain", lane=lane_id, name=name) raise finally: self.executor.cleaning = cleaning @@ -644,8 +670,11 @@ def evidence(self, box, lane, selection, phase, code): raise ValueError() if status == "pass": classification = "pass" - elif blocked or status == "dependency-blocked" or any( - x in signature for x in self.spec.get("known_harness_defects", [])): + elif blocked or status == "dependency-blocked": + classification = "known-harness-defect" + elif identity in self.known_product_failures and status == "fail": + classification = "product" + elif any(x in signature for x in self.spec.get("known_harness_defects", [])): classification = "known-harness-defect" elif kind == "infra" or any( x in signature for x in self.spec.get("infra_signatures", [])): @@ -843,9 +872,10 @@ def lane(self, lane): started = time.monotonic() phase = "create" try: - box = self.create(name) + box = self.create(lane) owned_name = self.boxes[box]["name"] - client.api("PATCH", box, {"name": owned_name, "ttlSeconds": self.spec["ttl_seconds"]}) + client.api("PATCH", box, {"name": owned_name, + "ttlSeconds": self.boxes[box]["ttl_seconds"]}) self.event("named", box=box, name=owned_name) phase = "clean_substrate" result = self.remote(box, name, "docker ps -aq", 30) @@ -866,7 +896,8 @@ def lane(self, lane): verdict = failure_verdict with self.lock: signature = sorted(value for value, seen in self.signatures.items() if name in seen) - break + if not lane.get("continue_on_failure", False) or self.stop.is_set(): + break except Deadline: verdict, signature = "timeout", phase + ":timeout" except Interrupted: diff --git a/tests/boat-lane-runner-cases.py b/tests/boat-lane-runner-cases.py index 7be7f715b9f..8ac5d48f02c 100644 --- a/tests/boat-lane-runner-cases.py +++ b/tests/boat-lane-runner-cases.py @@ -150,6 +150,114 @@ def test_budget_counts_spent_and_all_reserved_ttls(self): for row in self.ledger())) self.assert_cleanup() + def test_lane_ttl_overrides_creation_reservation_and_runs_concurrently(self): + body = spec(3) + body["ttl_seconds"] = 3600 + body["lanes"][1]["ttl_seconds"] = 14400 + barrier = ("touch /tmp/boat-lane-tests/$FM_LANE.started; " + "for attempt in {1..100}; do " + "if test -f /tmp/boat-lane-tests/lane1.started && " + "test -f /tmp/boat-lane-tests/lane2.started; then break; fi; " + "sleep .05; done; " + "test -f /tmp/boat-lane-tests/lane1.started && " + "test -f /tmp/boat-lane-tests/lane2.started # SELECT") + body["lanes"][1]["selections"] = [step(barrier, 8)] + body["lanes"][2]["selections"] = [step(barrier, 8)] + self.assertEqual(self.run_case(body, "--cap-usd", ".37", "--jobs", "2"), 0) + intents = {row["lane"]: row for row in self.ledger() if row["event"] == "create_intent"} + self.assertEqual(intents["lane1"]["ttl_seconds"], 14400) + self.assertAlmostEqual(intents["lane1"]["reserved_usd"], .288) + self.assertEqual(intents["lane2"]["ttl_seconds"], 3600) + self.assertAlmostEqual(intents["lane2"]["reserved_usd"], .072) + boxes = {row["lane"]: row["box"] for row in self.ledger() if row["event"] == "created"} + for lane, ttl in (("lane1", 14400), ("lane2", 3600)): + box = self.state()["boxes"][boxes[lane]] + self.assertEqual(box["create_ttl"], ttl) + self.assertEqual(box["ttlSeconds"], ttl) + self.assert_cleanup() + + def test_lane_ttl_is_used_by_cap_check(self): + body = spec(3) + body["ttl_seconds"] = 3600 + body["lanes"][1]["ttl_seconds"] = 14400 + self.assertEqual(self.run_case(body, "--cap-usd", ".29", "--jobs", "2"), 3) + intents = {row["lane"]: row for row in self.ledger() if row["event"] == "create_intent"} + self.assertEqual(intents["lane1"]["ttl_seconds"], 14400) + self.assertAlmostEqual(intents["lane1"]["reserved_usd"], .288) + refusal = next(row for row in self.ledger() if row["event"] == "refused") + self.assertEqual(refusal["lane"], "lane2") + self.assertAlmostEqual(refusal["new_usd"], .072) + self.assertGreaterEqual(refusal["reserved_usd"], .288) + self.assert_cleanup() + + def test_invalid_lane_ttl_is_refused_before_box_creation(self): + body = spec() + body["lanes"][0]["ttl_seconds"] = True + self.assertEqual(self.run_case(body), 2) + self.assertEqual(self.state()["boxes"], {}) + + + def test_continue_on_failure_runs_later_selection_and_fetches_artifacts(self): + body = spec() + path = "/tmp/boat-lane-tests/cases.json" + initial = dict(cases=[dict(id="resilience-fail", status="fail"), + dict(id="resilience-blocked", status="blocked")]) + later = dict(cases=[dict(id="maintenance", status="pass")]) + import shlex + emit = lambda payload: "printf %s " + shlex.quote(json.dumps(payload)) + " > " + shlex.quote(path) + body["lanes"][0]["continue_on_failure"] = True + body["lanes"][0]["selections"] = [ + dict(command=emit(initial) + "; exit 0 # SELECT", + timeout_seconds=2, evidence_file=path), + dict(command=emit(later) + "; touch /tmp/boat-lane-tests/maintenance-artifact # SELECT", + timeout_seconds=2, evidence_file=path)] + body["artifacts"] = ["/tmp/boat-lane-tests/maintenance-artifact"] + self.assertEqual(self.run_case(body), 4) + reports = [json.loads(line) for line in (self.home / "run/report.jsonl").read_text().splitlines()] + selections = [row for row in reports if row["attempt"] == "initial"] + self.assertEqual([row["selection"] for row in selections], [0, 1]) + first = {case["id"]: case for case in selections[0]["cases"]} + self.assertEqual(first["resilience-fail"]["status"], "fail") + self.assertEqual(first["resilience-blocked"]["status"], "blocked") + self.assertEqual(selections[1]["cases"][0]["id"], "maintenance") + self.assertEqual(selections[1]["cases"][0]["status"], "pass") + complete = next(row for row in reports if row["attempt"] == "complete") + self.assertEqual(complete["verdict"], "uncertain_failure") + self.assertEqual({case["id"] for case in complete["failures"]}, + {"resilience-fail", "resilience-blocked"}) + steps = [row for row in self.ledger() if row["event"] == "step" and row.get("phase") == "selection"] + artifact = next(row for row in self.ledger() + if row["event"] == "artifact" and row["path"] == body["artifacts"][0]) + self.assertEqual(len(steps), 2) + self.assertGreater(artifact["sequence"], steps[1]["sequence"]) + self.assertEqual(artifact["exit"], 0) + self.assert_cleanup() + + def test_lane_still_stops_after_nonpass_by_default(self): + body = spec() + path = "/tmp/boat-lane-tests/cases.json" + initial = dict(cases=[dict(id="first", status="fail")]) + later = dict(cases=[dict(id="later", status="pass")]) + import shlex + emit = lambda payload: "printf %s " + shlex.quote(json.dumps(payload)) + " > " + shlex.quote(path) + body["lanes"][0]["selections"] = [ + dict(command=emit(initial) + "; exit 0 # SELECT", + timeout_seconds=2, evidence_file=path), + dict(command=emit(later) + "; touch /tmp/boat-lane-tests/should-not-run # SELECT", + timeout_seconds=2, evidence_file=path)] + self.assertEqual(self.run_case(body), 4) + self.assertFalse((self.home / "should-not-run").exists()) + reports = [json.loads(line) for line in (self.home / "run/report.jsonl").read_text().splitlines()] + self.assertEqual([row["selection"] for row in reports if row["attempt"] == "initial"], [0]) + self.assert_cleanup() + + def test_invalid_continue_on_failure_is_refused_before_box_creation(self): + body = spec() + body["lanes"][0]["continue_on_failure"] = "yes" + self.assertEqual(self.run_case(body), 2) + self.assertEqual(self.state()["boxes"], {}) + + def test_product_failure_cleanup_and_pilot_stops_fanout(self): body = spec(4) body["lanes"][0]["selections"] = [step("false # SELECT")] @@ -257,6 +365,56 @@ def test_case_retry_excludes_known_defect(self): self.assertEqual(next(case for case in final["cases"] if case["id"] == "uncertain")["retry_decision"], "exhausted") self.assert_cleanup() + def test_known_product_failure_ids_are_not_retried(self): + body = spec() + path = "/tmp/boat-lane-tests/cases.json" + known = "QA-QUAL-RANKING-01" + blocked = "QA-KNOWN-BLOCKED" + initial = dict(cases=[dict(id=known, status="fail", failure_signature="ranking mismatch", + request_ids=["req-known"]), + dict(id=blocked, status="blocked", dependency_blocked=True, + request_ids=["req-blocked"]), + dict(id="eligible", status="fail", request_ids=["req-eligible"])]) + retried = dict(cases=[dict(id="eligible", status="pass", request_ids=["req-retry"])]) + import shlex + emit = lambda payload: "printf %s " + shlex.quote(json.dumps(payload)) + " > " + shlex.quote(path) + body["known_product_failures"] = [known, blocked] + body["lanes"][0]["selections"] = [dict(command=emit(initial) + "; false # SELECT", + timeout_seconds=2, evidence_file=path, retry_reset=[step("true")], + retry_cases={known: step("touch /tmp/boat-lane-tests/unsafe-known-retry; false"), + "eligible": step(emit(retried) + " # RETRY_uncertain")})] + self.assertEqual(self.run_case(body), 4) + self.assertFalse((self.home / "unsafe-known-retry").exists()) + reports = [json.loads(line) for line in (self.home / "run/report.jsonl").read_text().splitlines()] + initial_report = next(row for row in reports if row["attempt"] == "initial") + self.assertEqual([case["id"] for case in initial_report["failures"]], [known, blocked, "eligible"]) + cases = {row["id"]: row for row in initial_report["cases"]} + self.assertEqual(cases[known]["status"], "fail") + self.assertEqual(cases[known]["classification"], "product") + self.assertEqual(cases[known]["retry_decision"], "never") + self.assertEqual(cases[blocked]["status"], "blocked") + self.assertEqual(cases[blocked]["classification"], "known-harness-defect") + self.assertEqual(cases[blocked]["retry_decision"], "never") + self.assertEqual(cases[blocked]["request_ids"], ["req-blocked"]) + self.assertEqual(cases[known]["request_ids"], ["req-known"]) + self.assertEqual(cases["eligible"]["retry_decision"], "once-case") + retries = [row for row in reports if row["attempt"] == "retry"] + self.assertEqual(len(retries), 1) + self.assertEqual([case["id"] for case in retries[0]["cases"]], ["eligible"]) + final = next(row for row in reports if row["attempt"] == "complete") + final_cases = {row["id"]: row for row in final["cases"]} + self.assertEqual(final_cases[known]["status"], "fail") + self.assertEqual(final_cases[known]["retry_decision"], "never") + self.assertEqual(final_cases["eligible"]["retry_decision"], "exhausted") + self.assertEqual(final["verdict"], "product_failure") + self.assert_cleanup() + + def test_invalid_known_product_failure_ids_are_refused_before_box_creation(self): + body = spec() + body["known_product_failures"] = ["not a valid case id"] + self.assertEqual(self.run_case(body), 2) + self.assertEqual(self.state()["boxes"], {}) + def test_streaming_report_valid_before_last_lane_finishes(self): body = spec(3) body["lanes"][1]["selections"] = [step("sleep .15 # SELECT")] @@ -937,7 +1095,9 @@ def test_closed_pipes_with_live_process_still_honor_deadline(self): def test_help_owns_schema_and_exits(self): result = subprocess.run([sys.executable, str(ROOT / "bin/fm-boat-lanes.py"), "--help"], capture_output=True, text=True, check=True) - for term in ("Spec v1", "ledger.jsonl", "Exit codes", "124", "SIGKILL", "pilot", "cap-usd"): + for term in ("Spec v1", "ledger.jsonl", "Exit codes", "124", "SIGKILL", + "pilot", "cap-usd", "ttl_seconds", "known_product_failures", + "continue_on_failure"): self.assertIn(term, result.stdout) diff --git a/tests/boat-lane-runner-fixture.py b/tests/boat-lane-runner-fixture.py index 7573f11794f..7772522f5e0 100755 --- a/tests/boat-lane-runner-fixture.py +++ b/tests/boat-lane-runner-fixture.py @@ -48,10 +48,11 @@ def main(): if command == "new": assert "--no-env" in args and "--personal" in args assert args[args.index("--environment") + 1] == "base" + ttl = int(args[args.index("--ttl") + 1]) def create(state): box = "bx_" + str(len(state["boxes"]) + 1) - state["boxes"][box] = {"deleted": False} - state["events"].append({"event": "new", "box": box}) + state["boxes"][box] = {"deleted": False, "create_ttl": ttl} + state["events"].append({"event": "new", "box": box, "ttl_seconds": ttl}) return box box = mutate(create) print(json.dumps({"id": box}), flush=True) From abb2780196a741008cd1c122002ebe967eb1a48c Mon Sep 17 00:00:00 2001 From: dnth Date: Fri, 16 Oct 2026 17:19:25 +0800 Subject: [PATCH 2/4] no-mistakes(review): Fix known-failure protection, lane verdict aggregation, and cap tests --- bin/fm-boat-lanes.py | 5 ++- tests/boat-lane-runner-cases.py | 78 +++++++++++++++++++++++++++++---- 2 files changed, 72 insertions(+), 11 deletions(-) diff --git a/bin/fm-boat-lanes.py b/bin/fm-boat-lanes.py index e7ce45767b7..2288988b46d 100755 --- a/bin/fm-boat-lanes.py +++ b/bin/fm-boat-lanes.py @@ -672,7 +672,7 @@ def evidence(self, box, lane, selection, phase, code): classification = "pass" elif blocked or status == "dependency-blocked": classification = "known-harness-defect" - elif identity in self.known_product_failures and status == "fail": + elif identity in self.known_product_failures: classification = "product" elif any(x in signature for x in self.spec.get("known_harness_defects", [])): classification = "known-harness-defect" @@ -893,7 +893,8 @@ def lane(self, lane): phase = "selection" code, failure_verdict = self.selection(box, lane, selection, index) if code: - verdict = failure_verdict + verdict = max(verdict, failure_verdict, key=( + "pass", "known_harness_defect", "uncertain_failure", "product_failure").index) with self.lock: signature = sorted(value for value, seen in self.signatures.items() if name in seen) if not lane.get("continue_on_failure", False) or self.stop.is_set(): diff --git a/tests/boat-lane-runner-cases.py b/tests/boat-lane-runner-cases.py index 8ac5d48f02c..e6c22e2140a 100644 --- a/tests/boat-lane-runner-cases.py +++ b/tests/boat-lane-runner-cases.py @@ -180,14 +180,21 @@ def test_lane_ttl_is_used_by_cap_check(self): body = spec(3) body["ttl_seconds"] = 3600 body["lanes"][1]["ttl_seconds"] = 14400 + for lane in body["lanes"][1:]: + lane["selections"] = [step("sleep 120 # SELECT", 120)] self.assertEqual(self.run_case(body, "--cap-usd", ".29", "--jobs", "2"), 3) intents = {row["lane"]: row for row in self.ledger() if row["event"] == "create_intent"} - self.assertEqual(intents["lane1"]["ttl_seconds"], 14400) - self.assertAlmostEqual(intents["lane1"]["reserved_usd"], .288) refusal = next(row for row in self.ledger() if row["event"] == "refused") - self.assertEqual(refusal["lane"], "lane2") - self.assertAlmostEqual(refusal["new_usd"], .072) - self.assertGreaterEqual(refusal["reserved_usd"], .288) + admitted = ({"lane1", "lane2"} - {refusal["lane"]}).pop() + self.assertEqual(set(intents), {"pilot", admitted}) + for lane, ttl, reserve in (("lane1", 14400, .288), ("lane2", 3600, .072)): + if lane == admitted: + self.assertEqual(intents[lane]["ttl_seconds"], ttl) + self.assertAlmostEqual(intents[lane]["reserved_usd"], reserve) + self.assertGreaterEqual(refusal["reserved_usd"], reserve) + else: + self.assertAlmostEqual(refusal["new_usd"], reserve) + self.assertGreater(refusal["spent_usd"] + refusal["reserved_usd"] + refusal["new_usd"], .29) self.assert_cleanup() def test_invalid_lane_ttl_is_refused_before_box_creation(self): @@ -233,6 +240,39 @@ def test_continue_on_failure_runs_later_selection_and_fetches_artifacts(self): self.assertEqual(artifact["exit"], 0) self.assert_cleanup() + def test_continued_lane_verdict_preserves_failure_precedence_in_both_orders(self): + import itertools + import shlex + pairs = list(itertools.permutations(("product", "uncertain", "dependency-blocked"), 2)) + body = spec(len(pairs) + 1) + expected = {} + path = "/tmp/boat-lane-tests/cases.json" + for lane, kinds in zip(body["lanes"][1:], pairs): + lane["continue_on_failure"] = True + lane["selections"] = [] + expected[lane["id"]] = "product_failure" if "product" in kinds else "uncertain_failure" + for index, kind in enumerate((*kinds, "pass")): + case = dict(id=f"case-{index}", status="pass" if kind == "pass" else + "dependency-blocked" if kind == "dependency-blocked" else "failed", + failure_signature=f"{lane['id']}-{index}") + if kind == "product": + case["kind"] = "product" + command = "printf %s " + shlex.quote(json.dumps(dict(cases=[case]))) + lane["selections"].append(dict(command=command + " > " + shlex.quote(path) + " # SELECT", + timeout_seconds=2, evidence_file=path)) + self.assertEqual(self.run_case(body, "--jobs", "1"), 4) + reports = [json.loads(line) for line in (self.home / "run/report.jsonl").read_text().splitlines()] + results = {row["lane"]: row for row in self.ledger() if row["event"] == "lane_result"} + for lane, verdict in expected.items(): + with self.subTest(lane=lane): + complete = next(row for row in reports if row["lane"] == lane and row["attempt"] == "complete") + self.assertEqual(complete["verdict"], verdict) + self.assertEqual(results[lane]["verdict"], verdict) + self.assertEqual(len(complete["failures"]), 2) + self.assertEqual([row["selection"] for row in reports + if row["lane"] == lane and row["attempt"] == "initial"], [0, 1, 2]) + self.assert_cleanup() + def test_lane_still_stops_after_nonpass_by_default(self): body = spec() path = "/tmp/boat-lane-tests/cases.json" @@ -375,19 +415,27 @@ def test_known_product_failure_ids_are_not_retried(self): dict(id=blocked, status="blocked", dependency_blocked=True, request_ids=["req-blocked"]), dict(id="eligible", status="fail", request_ids=["req-eligible"])]) - retried = dict(cases=[dict(id="eligible", status="pass", request_ids=["req-retry"])]) + other_known = [dict(id=f"known-{status}", status=status, request_ids=[f"req-{status}"]) + for status in ("failed", "error", "blocked", "dependency-blocked", "pass")] + initial["cases"].extend(other_known) + retried = dict(cases=[dict(id="eligible", status="pass", request_ids=["req-retry"]), + dict(id="retry-known", status="failed", request_ids=["req-new"])]) import shlex emit = lambda payload: "printf %s " + shlex.quote(json.dumps(payload)) + " > " + shlex.quote(path) - body["known_product_failures"] = [known, blocked] + body["known_product_failures"] = [known, blocked, "retry-known"] + [case["id"] for case in other_known] body["lanes"][0]["selections"] = [dict(command=emit(initial) + "; false # SELECT", timeout_seconds=2, evidence_file=path, retry_reset=[step("true")], retry_cases={known: step("touch /tmp/boat-lane-tests/unsafe-known-retry; false"), "eligible": step(emit(retried) + " # RETRY_uncertain")})] + for case in other_known: + body["lanes"][0]["selections"][0]["retry_cases"][case["id"]] = step( + "touch /tmp/boat-lane-tests/unsafe-known-retry; false") self.assertEqual(self.run_case(body), 4) self.assertFalse((self.home / "unsafe-known-retry").exists()) reports = [json.loads(line) for line in (self.home / "run/report.jsonl").read_text().splitlines()] initial_report = next(row for row in reports if row["attempt"] == "initial") - self.assertEqual([case["id"] for case in initial_report["failures"]], [known, blocked, "eligible"]) + self.assertEqual([case["id"] for case in initial_report["failures"]], + [case["id"] for case in initial["cases"] if case["status"] != "pass"]) cases = {row["id"]: row for row in initial_report["cases"]} self.assertEqual(cases[known]["status"], "fail") self.assertEqual(cases[known]["classification"], "product") @@ -400,9 +448,21 @@ def test_known_product_failure_ids_are_not_retried(self): self.assertEqual(cases["eligible"]["retry_decision"], "once-case") retries = [row for row in reports if row["attempt"] == "retry"] self.assertEqual(len(retries), 1) - self.assertEqual([case["id"] for case in retries[0]["cases"]], ["eligible"]) + self.assertEqual([case["id"] for case in retries[0]["cases"]], ["eligible", "retry-known"]) final = next(row for row in reports if row["attempt"] == "complete") final_cases = {row["id"]: row for row in final["cases"]} + for case in other_known: + identity, status = case["id"], case["status"] + classification = ("pass" if status == "pass" else "known-harness-defect" + if status == "dependency-blocked" else "product") + for observed in (cases[identity], final_cases[identity]): + self.assertEqual(observed["status"], status) + self.assertEqual(observed["classification"], classification) + self.assertEqual(observed["request_ids"], case["request_ids"]) + self.assertEqual(observed["retry_decision"], "not-needed" if status == "pass" else "never") + self.assertEqual(final_cases["retry-known"]["classification"], "product") + self.assertEqual(final_cases["retry-known"]["retry_decision"], "never") + self.assertEqual(final_cases["retry-known"]["request_ids"], ["req-new"]) self.assertEqual(final_cases[known]["status"], "fail") self.assertEqual(final_cases[known]["retry_decision"], "never") self.assertEqual(final_cases["eligible"]["retry_decision"], "exhausted") From 9df0b7ae83c662da8d2c7f9c48a353edb65857a2 Mon Sep 17 00:00:00 2001 From: dnth Date: Fri, 16 Oct 2026 17:25:15 +0800 Subject: [PATCH 3/4] no-mistakes(review): Preserve listed failures and evidence across retry merges --- bin/fm-boat-lanes.py | 2 ++ tests/boat-lane-runner-cases.py | 60 +++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/bin/fm-boat-lanes.py b/bin/fm-boat-lanes.py index 2288988b46d..ba6716666b6 100755 --- a/bin/fm-boat-lanes.py +++ b/bin/fm-boat-lanes.py @@ -810,6 +810,8 @@ def selection(self, box, lane, selection, index): cases.append(row) for row in cases: if row["id"] in latest: + if row["id"] in self.known_product_failures and row["status"] != "pass": + continue decision = ("exhausted" if row["id"] in attempted else "not-needed" if latest[row["id"]]["status"] == "pass" else "never" if latest[row["id"]]["classification"] in ("known-harness-defect", "product") diff --git a/tests/boat-lane-runner-cases.py b/tests/boat-lane-runner-cases.py index e6c22e2140a..845577d7e27 100644 --- a/tests/boat-lane-runner-cases.py +++ b/tests/boat-lane-runner-cases.py @@ -469,6 +469,66 @@ def test_known_product_failure_ids_are_not_retried(self): self.assertEqual(final["verdict"], "product_failure") self.assert_cleanup() + def test_known_failures_survive_overlapping_retry_evidence_and_stop_fanout(self): + import shlex + body = spec(3) + path = "/tmp/boat-lane-tests/cases.json" + retained = [dict(id="known", status="failed", failure_signature="original product failure", + request_ids=["req-original"]), + dict(id="blocked", status="blocked", dependency_blocked=True, + failure_signature="original dependency failure", request_ids=["req-blocked"])] + initial = dict(cases=retained + [dict(id="known-pass", status="pass"), + dict(id="eligible-1", status="failed"), + dict(id="eligible-2", status="failed"), + dict(id="unlisted", status="failed", kind="product")]) + discovered = [dict(id="new-known", status="error", failure_signature="new failure", + request_ids=["req-new"]), + dict(id="known-pass", status="failed", failure_signature="newly failing", + request_ids=["req-newly-failing"])] + first_retry = dict(cases=[dict(id=case["id"], status="pass", request_ids=["req-replacement"]) + for case in retained] + discovered + + [dict(id="eligible-1", status="pass"), + dict(id="eligible-2", status="failed"), + dict(id="unlisted", status="pass")]) + second_retry = dict(cases=[dict(id=case["id"], status="pass", request_ids=["req-erased"]) + for case in retained + discovered] + + [dict(id="eligible-2", status="pass")]) + emit = lambda payload: "printf %s " + shlex.quote(json.dumps(payload)) + " > " + shlex.quote(path) + body["known_product_failures"] = [case["id"] for case in retained + discovered] + body["lanes"][0]["selections"] = [dict( + command=emit(initial) + "; false # SELECT", timeout_seconds=2, evidence_file=path, + retry_reset=[step("true")], + retry_cases={"eligible-1": step(emit(first_retry)), "eligible-2": step(emit(second_retry))})] + self.assertEqual(self.run_case(body), 4) + reports = [json.loads(line) for line in (self.home / "run/report.jsonl").read_text().splitlines()] + initial_report = next(row for row in reports if row["attempt"] == "initial") + retries = [row for row in reports if row["attempt"] == "retry"] + self.assertEqual(len(retries), 2) + final = next(row for row in reports if row["attempt"] == "complete") + originals = {case["id"]: case for case in initial_report["cases"]} + first_observed = {case["id"]: case for case in retries[0]["cases"]} + for report in retries + [final]: + cases = {case["id"]: case for case in report["cases"]} + for case in retained: + self.assertEqual(cases[case["id"]], originals[case["id"]]) + for case in discovered: + self.assertEqual(cases[case["id"]], first_observed[case["id"]]) + self.assertEqual(cases[case["id"]]["status"], case["status"]) + self.assertEqual(cases[case["id"]]["retry_decision"], "never") + self.assertEqual(report["overall_totals"], + {"failed": 3, "blocked": 1, "error": 1, "pass": 2} + if report is retries[0] else + {"failed": 2, "blocked": 1, "error": 1, "pass": 3}) + final_cases = {case["id"]: case for case in final["cases"]} + self.assertEqual(final_cases["unlisted"]["status"], "pass") + self.assertEqual(final_cases["eligible-1"]["status"], "pass") + self.assertEqual(final_cases["eligible-2"]["status"], "pass") + self.assertEqual(final["verdict"], "product_failure") + results = [row for row in self.ledger() if row["event"] == "lane_result"] + self.assertEqual([(row["lane"], row["verdict"]) for row in results], [("pilot", "product_failure")]) + self.assertEqual(len(self.state()["boxes"]), 1) + self.assert_cleanup() + def test_invalid_known_product_failure_ids_are_refused_before_box_creation(self): body = spec() body["known_product_failures"] = ["not a valid case id"] From 96a8da4abfac091f5a725516ad22229758326490 Mon Sep 17 00:00:00 2001 From: dnth Date: Fri, 16 Oct 2026 17:34:09 +0800 Subject: [PATCH 4/4] no-mistakes(document): Clarify Boat lane TTL, continuation, and failure preservation --- bin/fm-boat-lanes.py | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/bin/fm-boat-lanes.py b/bin/fm-boat-lanes.py index ba6716666b6..70304097574 100755 --- a/bin/fm-boat-lanes.py +++ b/bin/fm-boat-lanes.py @@ -23,14 +23,17 @@ pilot names one of them. Each lane has nonempty selections, an optional report-only profile (default "default"), and an optional ttl_seconds override using the same range; omitted lane TTLs inherit the spec value. -continue_on_failure is an optional boolean that defaults to false. When true, +The effective lane TTL controls reservation, cap admission, box creation and TTL update. +continue_on_failure is an optional boolean that defaults to false, stopping after +a selection remains non-pass following any permitted retries. When true, later selections continue after a selection reports failed or blocked cases; harness errors, timeouts, cancellation, and invalid evidence still stop the lane. +Continued selections retain earlier failures; product_failure takes precedence +over uncertain_failure, then known_harness_defect, then pass in the lane verdict. prepare, finish, artifacts and env default to empty; known_harness_defects and infra_signatures are optional top-level lists of nonempty literal substrings. known_product_failures is an optional top-level list of unique case IDs matching -[A-Za-z0-9_.:-]{1,120}; failed matching cases are product failures and are never retried. -Dependency-blocked evidence keeps its existing classification. +[A-Za-z0-9_.:-]{1,120}, defaulting to empty; classification and retry rules follow below. Every step needs a positive finite timeout_seconds. env maps shell variable names to strings without NUL; FM_ names are reserved. Artifact and evidence paths must be beneath /tmp without '..' components. Upload sources must exist. @@ -62,9 +65,10 @@ dependency_blocked defaults to false and must be a boolean, request_ids defaults to [] and must contain strings matching [A-Za-z0-9_.:-]{1,160}, and kind (when present) must be infra or product. - Malformed fields are harness evidence errors. Non-pass cases classify as known-harness-defect when - dependency_blocked=true, status="dependency-blocked", or matching a literal - substring in spec known_harness_defects; + Malformed fields are harness evidence errors. Non-pass cases classify in order: + known-harness-defect when dependency_blocked=true or status="dependency-blocked"; + then product when the ID is in known_product_failures; + then known-harness-defect when matching a literal substring in known_harness_defects; then infra when kind=infra or matching infra_signatures (e.g. a known outage); then product when kind=product; otherwise uncertain. Passes classify as pass. Only infra/uncertain are eligible for one retry. Selection retry_reset:[STEP] @@ -73,7 +77,10 @@ skipped when it would rerun an attempted case or a currently known-defect/product case. Never rerun all lane selections. Missing reset/selection records a skipped retry. Retry commands must write updated evidence_file; known defects/product are never - retry targets. Newly reported retry cases remain in verdicts and reports; newly + retry targets. For IDs in known_product_failures, the first non-pass status and + evidence are retained within the selection, including dependency-blocked cases; + overlapping retry evidence cannot replace them in retry or final reports. + Newly reported retry cases remain in verdicts and reports; newly observed infra/uncertain cases are recorded without adding retry targets. Infra transport/timeout failures without readable case evidence remain harness/timeout verdicts, not invented product cases. @@ -107,7 +114,8 @@ lane finish: lane, box, profile, duration_seconds, cases (id/status/classification/ request_ids/retry_decision), case_counts, failures and spend_so_far_usd. Each append recomputes overall_totals from the latest case result per lane/selection/id - (retries replace their original result). report.md is atomically replaced after + (retries replace their original result except for retained listed failures). + report.md is atomically replaced after every append and readable mid-run. Only structural evidence is recorded, never arbitrary request bodies, credentials, commands or remote stdout/stderr.