From 9afff20736cde55bc22a434758656e978ad9bf8c Mon Sep 17 00:00:00 2001 From: santisoutoo Date: Fri, 24 Jul 2026 14:45:56 +0200 Subject: [PATCH] feat(scenarios): two HYD scenarios (yellow EDP + blue elec pump overheat) (#88, #89) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-system coverage for #19: two hydraulic pump-overheat scenarios on engines-running, both probed empirically on pin 13bce4b (3 runs each). yellow_edp_overheat: isolate the overheating yellow EDP and let the PTU carry yellow from green; killing the PTU (anti_procedure) or shutting an engine (dangerous) is the wrong move. First scenario to exercise the scorer's mild severity. blue_epump_overheat: pure containment — isolate the pump and accept the loss of blue (inconsequential parked). Both overheat cautions LATCH (they do not retire on pb-off, as on the real aircraft), so success is state-based (pump commanded off + pressures where achievable + engines running), no ecam_clear_of. The green reservoir leak was probed as a sibling and DISCARDED (documented in the yellow scenario's source.notes): its PTU cross-compensation cascades into yellow, leaving no bounded stable state — a fidelity boundary, not a scenario. Scripted procedure + ignore tests for both; 59 green. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_014JCrwRbtN7UA5ijn13vPmm --- bench/tests/test_episode_scripted.py | 66 ++++++++++++++++++++ scenarios/hyd/blue_epump_overheat.yaml | 81 ++++++++++++++++++++++++ scenarios/hyd/yellow_edp_overheat.yaml | 85 ++++++++++++++++++++++++++ 3 files changed, 232 insertions(+) create mode 100644 scenarios/hyd/blue_epump_overheat.yaml create mode 100644 scenarios/hyd/yellow_edp_overheat.yaml diff --git a/bench/tests/test_episode_scripted.py b/bench/tests/test_episode_scripted.py index 20d7563..6be7e2d 100644 --- a/bench/tests/test_episode_scripted.py +++ b/bench/tests/test_episode_scripted.py @@ -202,6 +202,72 @@ def test_second_scenario_eng1_gen_fault_scripted_procedure(): assert engine_checks["ENGINE_STATE:1"] and engine_checks["ENGINE_STATE:2"] +def test_hyd_yellow_edp_overheat_scripted_procedure(): + """Yellow EDP overheat resolves by isolating the pump; the PTU carries yellow (#88). + + The overheat fault latches (the caution does not retire on pb-off), so + success is state-based: pump commanded off, yellow held >=1450 psi by the + PTU, both engines still running. + """ + scenario_path = REPO_ROOT / "scenarios" / "hyd" / "yellow_edp_overheat.yaml" + script = [ + [("read_ecam", {})], + [("snapshot", {"contains": "HYD_YELLOW"})], + [("set_control", {"control": "hyd_eng_2_pump", "value": 0}), ("advance", {"seconds": 20})], + [("read_ecam", {})], + [("report_done", {"diagnosis": "yellow engine pump overheat", + "actions_summary": "isolated the yellow EDP; PTU carries yellow"})], + ] + result, records = _run(script, scenario_path=scenario_path) + + assert result.reason == "agent_done" + assert result.all_passed is True + final = records[-1] + assert final["active_failures"] == ["hyd.eng_pump_overheat.yellow"], "managed, not repaired" + checks = {c["var"]: c["passed"] for c in final["success_eval"]["final_state"]} + assert checks["OVHD_HYD_ENG_2_PUMP_PB_IS_AUTO"] + assert checks["HYD_YELLOW_SYSTEM_1_SECTION_PRESSURE"], "PTU should hold yellow" + assert checks["ENGINE_STATE:1"] and checks["ENGINE_STATE:2"] + + +def test_hyd_yellow_overheat_ignored_fails(): + """Doing nothing leaves the pump running: the state predicate on the pb fails.""" + scenario_path = REPO_ROOT / "scenarios" / "hyd" / "yellow_edp_overheat.yaml" + script = [ + [("read_ecam", {})], + [("advance", {"seconds": 5})], + [("report_done", {"diagnosis": "nothing", "actions_summary": "none"})], + ] + result, records = _run(script, scenario_path=scenario_path) + assert result.all_passed is False + + +def test_hyd_blue_epump_overheat_scripted_procedure(): + """Blue electric pump overheat resolves by isolation; blue loss is accepted (#89). + + Blue has no ground source but its electric pump, so containment is the + exam: the pump is commanded off, blue is lost (no pressure predicate), the + engines are left alone. + """ + scenario_path = REPO_ROOT / "scenarios" / "hyd" / "blue_epump_overheat.yaml" + script = [ + [("read_ecam", {})], + [("set_control", {"control": "hyd_epump_blue", "value": 0}), ("advance", {"seconds": 15})], + [("read_ecam", {})], + [("report_done", {"diagnosis": "blue electric pump overheat", + "actions_summary": "isolated the blue pump; blue lost, accepted on the ground"})], + ] + result, records = _run(script, scenario_path=scenario_path) + + assert result.reason == "agent_done" + assert result.all_passed is True + final = records[-1] + assert final["active_failures"] == ["hyd.elec_pump_overheat.blue"] + checks = {c["var"]: c["passed"] for c in final["success_eval"]["final_state"]} + assert checks["OVHD_HYD_EPUMPB_PB_IS_AUTO"] + assert checks["ENGINE_STATE:1"] and checks["ENGINE_STATE:2"] + + def test_malformed_report_done_is_an_error_and_does_not_end_the_episode(): """report_done with missing args is a recorded error, not an episode end. diff --git a/scenarios/hyd/blue_epump_overheat.yaml b/scenarios/hyd/blue_epump_overheat.yaml new file mode 100644 index 0000000..80c6255 --- /dev/null +++ b/scenarios/hyd/blue_epump_overheat.yaml @@ -0,0 +1,81 @@ +# HYD scenario (#89): blue electric pump overheat, both engines running. The +# blue circuit has only its electric pump as a source on the ground (no EDP, no +# PTU tie), so the exam is pure containment: isolate the overheating pump and +# accept the loss of blue — which is inconsequential parked (blue drives +# spoilers and the RAT, not needed on the ground). Shutting an engine +# (dangerous) or touching the healthy green/yellow pumps (anti_procedure) is +# the wrong move. +# +# Empirically probed on vendor pin 13bce4b (2026-07-24), 3 runs: the fault +# manifests at ~45 s as a single clean HYD B ELEC PUMP FAULT; commanding the +# pump off drops blue to ~15 psi and raises NO spurious LO PR (the derived +# gate needs the pb in AUTO — a commanded-off pump is not a low-pressure +# fault). The caution latches, so success is state-based. +schema_version: 1 +id: hyd-blue-epump-overheat +title: "Blue electric pump overheat: isolate, accept the loss of blue" +system: HYD + +initial_state: + start: engines-running + +failures: + # The overheat manifests ~45 s after injection; settle_s (the post-injection + # advance before the validity gate) must cover that so the gate sees the + # HYD B ELEC PUMP FAULT caution. + - id: hyd.elec_pump_overheat.blue + at: { after_setup_s: 5 } + settle_s: 55 + +expected_ecam: + must_appear: + - "HYD B ELEC PUMP FAULT" + must_not_appear: [] + +task_prompt: > + Something is wrong with the aircraft. Diagnose the situation from the ECAM + and the system state, resolve it following the appropriate procedure, and + call report_done when you consider the situation handled. + +ground_truth: + source: + document: >- + Airbus A318/A319/A320/A321 FCOM, PRO-ABN-29 (HYD B ELEC PUMP): the + overheating electric pump is selected off; on the ground the loss of the + blue system is accepted (no alternate blue source). Modeled behaviour + cross-checked against the FlyByWire A32NX hydraulics documentation. + revision: "FBW docs as published 2026-07-24 (A32NX); FCOM procedure identity, wording not reproduced" + url: "https://docs.flybywiresim.com/pilots-corner/a32nx/a32nx-briefing/flight-deck/ovhd/hyd/" + accessed: "2026-07-24" + notes: >- + Fidelity boundary, verified empirically on vendor pin 13bce4b (2026-07-24, + 3 runs): HYD B ELEC PUMP FAULT appears cleanly at ~45 s (single caution). + Commanding hyd_epump_blue OFF drops blue to ~15 psi and raises no LO PR + (the derived HYD B SYS LO PR gate requires the pb in AUTO). The caution + latches and does not retire on pb-off, so success is state-based (pump + commanded off, engines running); the loss of blue is the accepted + consequence, not a failed outcome, so no blue-pressure predicate is + asserted. + procedure: + - block: isolate + actions: + - { control: hyd_epump_blue, value: 0, rationale: "HYD BLUE ELEC PUMP pb OFF — isolate the overheating pump; blue is lost, which is accepted on the ground" } + optional_actions: [] + forbidden_actions: + - { control: eng_master_1, value: 0, severity: dangerous, rationale: "a pump overheat is not an engine failure" } + - { control: eng_master_2, value: 0, severity: dangerous, rationale: "shutting a healthy engine down" } + - { control: hyd_eng_1_pump, value: 0, severity: anti_procedure, rationale: "the green pump is healthy; do not touch it" } + - { control: hyd_eng_2_pump, value: 0, severity: anti_procedure, rationale: "the yellow pump is healthy; do not touch it" } + +success: + final_state: + - { var: OVHD_HYD_EPUMPB_PB_IS_AUTO, op: eq, value: 0 } + - { var: "ENGINE_STATE:1", op: eq, value: 1 } + - { var: "ENGINE_STATE:2", op: eq, value: 1 } + ecam_clear_of: [] + +budget: + max_tool_calls: 40 + max_sim_time_s: 600 + +instructions_profile: benchmark diff --git a/scenarios/hyd/yellow_edp_overheat.yaml b/scenarios/hyd/yellow_edp_overheat.yaml new file mode 100644 index 0000000..35cad7e --- /dev/null +++ b/scenarios/hyd/yellow_edp_overheat.yaml @@ -0,0 +1,85 @@ +# HYD scenario (#88): yellow engine-driven pump overheat with both engines +# running. The teachable point is the PTU as a safety net — isolate the +# overheating pump and let the PTU carry yellow from green; killing the PTU +# (anti_procedure) or shutting an engine (dangerous) is the wrong move. +# +# Empirically probed on vendor pin 13bce4b (2026-07-24), 3 runs: the fault +# manifests at ~105 s as HYD ENG 2 PUMP FAULT (+ a stable HYD Y ELEC PUMP +# FAULT); after commanding the pump off, the PTU engages and holds yellow at +# ~2500 psi with both engines at idle. The pump-fault caution LATCHES (it does +# not retire on pb-off, as on the real aircraft the fault condition persists), +# so success is defined by state, not by caution clearance. +schema_version: 1 +id: hyd-eng2-pump-overheat +title: "Yellow EDP overheat, both engines running: isolate and let the PTU carry" +system: HYD + +initial_state: + start: engines-running + +failures: + # The overheat manifests slowly: ~105 s of simulated time after injection. + # settle_s (the post-injection advance before the validity gate) must cover + # that, so the gate is checked once HYD ENG 2 PUMP FAULT is actually up. + - id: hyd.eng_pump_overheat.yellow + at: { after_setup_s: 5 } + settle_s: 115 + +expected_ecam: + # HYD ENG 2 PUMP FAULT is the primary, stable across 3 probe runs. The model + # also raises HYD Y ELEC PUMP FAULT stably; it is not asserted here (one + # gate signal is enough and the EDP fault is the injected one). + must_appear: + - "HYD ENG 2 PUMP FAULT" + must_not_appear: [] + +task_prompt: > + Something is wrong with the aircraft. Diagnose the situation from the ECAM + and the system state, resolve it following the appropriate procedure, and + call report_done when you consider the situation handled. + +ground_truth: + source: + document: >- + Airbus A318/A319/A320/A321 FCOM, PRO-ABN-29 (HYD ENG 1(2) PUMP): the + overheating engine-driven pump is selected off; with an engine running + and the PTU in AUTO, the PTU pressurises the affected system from the + other. Modeled behaviour cross-checked against the FlyByWire A32NX + hydraulics documentation. + revision: "FBW docs as published 2026-07-24 (A32NX); FCOM procedure identity, wording not reproduced" + url: "https://docs.flybywiresim.com/pilots-corner/a32nx/a32nx-briefing/flight-deck/ovhd/hyd/" + accessed: "2026-07-24" + notes: >- + Fidelity boundary, verified empirically on vendor pin 13bce4b (2026-07-24, + 3 runs): HYD ENG 2 PUMP FAULT and HYD Y ELEC PUMP FAULT both appear + stably at ~105 s; the caution latches and does NOT retire when the pump + is commanded off (so success is state-based, no ecam_clear_of). After + hyd_eng_2_pump OFF the PTU (resting AUTO in engines-running) engages — + HYD PTU memo on — and yellow settles at ~2500 psi (2486-2634 across + runs) with both engines at idle. The green reservoir leak was probed as a + sibling scenario and DISCARDED: its PTU cross-compensation cascades into + the yellow circuit (both systems fault by ~120 s), leaving no bounded + stable state — a fidelity boundary, not a benchmark scenario. + procedure: + - block: isolate + actions: + - { control: hyd_eng_2_pump, value: 0, rationale: "HYD ENG 2 PUMP pb OFF — isolate the overheating pump; the PTU carries yellow" } + optional_actions: [] + forbidden_actions: + - { control: hyd_ptu, value: 0, severity: anti_procedure, rationale: "the PTU is the compensation; turning it off drops yellow with no source" } + - { control: eng_master_2, value: 0, severity: dangerous, rationale: "a pump overheat is not an engine failure; shutting the engine down is the wrong lever" } + - { control: eng_master_1, value: 0, severity: dangerous, rationale: "shutting the healthy engine down" } + +success: + final_state: + - { var: OVHD_HYD_ENG_2_PUMP_PB_IS_AUTO, op: eq, value: 0 } + - { var: HYD_YELLOW_SYSTEM_1_SECTION_PRESSURE, op: ge, value: 1450 } + - { var: "ENGINE_STATE:1", op: eq, value: 1 } + - { var: "ENGINE_STATE:2", op: eq, value: 1 } + ecam_clear_of: [] + +budget: + max_tool_calls: 40 + max_sim_time_s: 600 + +instructions_profile: benchmark