diff --git a/CHANGELOG.md b/CHANGELOG.md index 979fc2b..1528532 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,14 @@ All notable changes to evidence-cli are documented here. This format follows [Keep a Changelog](https://keepachangelog.com/), and the project adheres to [Semantic Versioning](https://semver.org/). +## [0.1.4] — 2026-07-07 + +- **Failure `title`** — a step-level `failure.yaml` MAY carry a short + defect-style `title` (open string, never cross-checked); `finalize` lifts it + verbatim into the run-level failure index row, so triage queues and + dashboards get a name, not just a status. Purely additive — records without + a title index exactly as before. + ## [0.1.0] — Unreleased Initial public release of the `0.1` evidence contract. diff --git a/design/contract/04-L1.md b/design/contract/04-L1.md index eaa91e0..a3314ce 100644 --- a/design/contract/04-L1.md +++ b/design/contract/04-L1.md @@ -128,6 +128,7 @@ Authored by the framework (or a triage agent), sibling of the step's screenshot. ```yaml step: pay # REQUIRED — = folder id + a result.yaml step status: broken # REQUIRED — verdict enum; cross-checked +title: Payment API timeout on checkout # OPTIONAL — short defect-style name (open string) expected: receipt screen shown # ┐ evidence-of-what: error.message, actual: payment API timed out # ┘ or expected + actual (REQUIRED) page_state: @@ -155,6 +156,8 @@ triage: # OPTIONAL — strictly shaped when pre URLs are open strings, never checked. - **Triage is strict when present**, open vocabulary elsewhere (`locator_context`, `error.type`, `rca.category`, `linked_defects`, …). +- An optional **`title`** gives the failure a short defect-style *name* (the + label a triage queue shows) — an open string, never cross-checked. - A record whose `status` disagrees with its step's `result.yaml` status is an advisory **warning**; so is a failed/broken step with no record (at finalized). Neither fails the pack. @@ -175,6 +178,7 @@ failures: step: pay status: broken path: tests/checkout/steps/2-pay/failure.yaml # pack-root-relative pointer + title: Payment API timeout on checkout # lifted from the record when present triage_status: triaged # lifted from the record when present ``` diff --git a/design/decisions/0044-failure-records.md b/design/decisions/0044-failure-records.md index c69734c..ca7cfad 100644 --- a/design/decisions/0044-failure-records.md +++ b/design/decisions/0044-failure-records.md @@ -73,6 +73,16 @@ supersedes: [] ## Reasoning +> **Amendment (optional `title`, 2026-07-07).** A step record MAY carry a +> top-level `title` — a short defect-style *name* for the failure, typically +> the investigation verdict's bug title. It is an **open string, never +> cross-checked** (no new codes, no new required fields), and finalize **lifts +> it verbatim** into the index row when it is a non-empty string — the same +> mirror-never-validate posture as the `triage_status` lift. Rationale: triage +> queues and dashboards need a *label*, not just a status — `rca.root_cause` is +> a sentence, `title` is a name. A record-vs-row title mismatch is impossible +> on a freshly finalized pack, since finalize generates the index. + **Why not `issues/`.** The reserved directory imagined failures as a *sibling concept* beside steps. But a failure's forensics are *about a step* — the expected/actual, the page state, the console at the moment step N broke. Giving diff --git a/fixtures/0.1/L1/valid/failure-full.evidence/failure.yaml b/fixtures/0.1/L1/valid/failure-full.evidence/failure.yaml index 3cbd26d..dd7b382 100644 --- a/fixtures/0.1/L1/valid/failure-full.evidence/failure.yaml +++ b/fixtures/0.1/L1/valid/failure-full.evidence/failure.yaml @@ -6,4 +6,5 @@ failures: step: pay status: broken path: tests/checkout/steps/2-pay/failure.yaml + title: Payment API timeout on checkout triage_status: triaged diff --git a/fixtures/0.1/L1/valid/failure-full.evidence/tests/checkout/steps/2-pay/failure.yaml b/fixtures/0.1/L1/valid/failure-full.evidence/tests/checkout/steps/2-pay/failure.yaml index f0fd89a..d544a93 100644 --- a/fixtures/0.1/L1/valid/failure-full.evidence/tests/checkout/steps/2-pay/failure.yaml +++ b/fixtures/0.1/L1/valid/failure-full.evidence/tests/checkout/steps/2-pay/failure.yaml @@ -1,5 +1,6 @@ step: pay status: broken +title: Payment API timeout on checkout expected: receipt screen shown with a charge id actual: payment API timed out after 30s page_state: diff --git a/package.json b/package.json index ecf09c3..d9aaddc 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@testmuai/evidence-cli", - "version": "0.1.3", + "version": "0.1.4", "description": "An open, framework-agnostic format for what a test run produced — the .evidence pack, and the library + CLI that validate and seal it.", "license": "Apache-2.0", "author": "TestMu AI (formerly LambdaTest)", diff --git a/src/finalize/index.test.ts b/src/finalize/index.test.ts index decce85..265dc5e 100644 --- a/src/finalize/index.test.ts +++ b/src/finalize/index.test.ts @@ -123,6 +123,37 @@ describe("finalize failure index (decision 0044)", () => { ]); }); + it("lifts an optional title verbatim; omits it when absent, empty, or not a string", async () => { + const dir = await stageCopy(); + const steps = path.join(dir, "tests", "checkout", "steps"); + await fs.mkdir(path.join(steps, "1-open"), { recursive: true }); + await fs.mkdir(path.join(steps, "2-pay"), { recursive: true }); + await fs.mkdir(path.join(steps, "3-refund"), { recursive: true }); + // (a) present — lifted verbatim + await fs.writeFile( + path.join(steps, "1-open", "failure.yaml"), + "step: open\nstatus: failed\ntitle: Checkout page renders blank\nerror: { message: blank }\n", + ); + // (b) empty string — omitted, no `title: null` noise + await fs.writeFile( + path.join(steps, "2-pay", "failure.yaml"), + 'step: pay\nstatus: broken\ntitle: ""\nerror: { message: timeout }\n', + ); + // (c) an old record without title still indexes fine + await fs.writeFile( + path.join(steps, "3-refund", "failure.yaml"), + "step: refund\nstatus: failed\nerror: { message: no refund button }\n", + ); + + await finalize(dir, { endedAt: "2026-06-28T09:00:30Z" }); + const idx = parseYaml(await readSealed(dir, "failure.yaml")) as any; + + expect(idx.failures[0].title).toBe("Checkout page renders blank"); + expect("title" in idx.failures[1]).toBe(false); + expect("title" in idx.failures[2]).toBe(false); + expect(idx.failures.map((r: any) => r.step)).toEqual(["open", "pay", "refund"]); + }); + it("fails fast on a step record that is not valid YAML", async () => { const dir = await stageCopy(); const folder = path.join(dir, "tests", "checkout", "steps", "2-pay"); diff --git a/src/finalize/index.ts b/src/finalize/index.ts index e53acc8..097c904 100644 --- a/src/finalize/index.ts +++ b/src/finalize/index.ts @@ -12,6 +12,7 @@ interface FailureRow { step: string; status: string; path: string; + title?: string; triage_status?: string; } @@ -145,6 +146,7 @@ async function collectFailureRows(testsDir: string, id: string): Promise 0) row.title = rec.title; const ts = rec?.triage?.status; if (typeof ts === "string" && ts.length > 0) row.triage_status = ts; rows.push(row); diff --git a/src/schemas/0.1/L1/failure-index.schema.json b/src/schemas/0.1/L1/failure-index.schema.json index d71c39e..fa9e013 100644 --- a/src/schemas/0.1/L1/failure-index.schema.json +++ b/src/schemas/0.1/L1/failure-index.schema.json @@ -38,6 +38,11 @@ "triage_status": { "description": "Lifted verbatim from the record's triage.status when present.", "enum": ["untriaged", "triaged", "in_progress", "dismissed"] + }, + "title": { + "description": "Lifted verbatim from the record's title when present.", + "type": "string", + "minLength": 1 } }, "additionalProperties": true diff --git a/src/schemas/0.1/L1/failure.schema.json b/src/schemas/0.1/L1/failure.schema.json index 5d01af0..9915973 100644 --- a/src/schemas/0.1/L1/failure.schema.json +++ b/src/schemas/0.1/L1/failure.schema.json @@ -26,6 +26,11 @@ "description": "The step verdict — same vocabulary as result.yaml (decision 0006). Cross-checked against the matched step; disagreement is an advisory warning (l1.failure.status_disagrees).", "enum": ["passed", "failed", "broken", "skipped"] }, + "title": { + "description": "Short defect-style name for the failure, authored by the framework (typically the investigation verdict's bug title). Optional; open string, never cross-checked.", + "type": "string", + "minLength": 1 + }, "expected": { "type": "string" }, "actual": { "type": "string" }, "error": { diff --git a/src/schemas/failure.schema.test.ts b/src/schemas/failure.schema.test.ts index 6d0470a..ad350e7 100644 --- a/src/schemas/failure.schema.test.ts +++ b/src/schemas/failure.schema.test.ts @@ -29,6 +29,11 @@ describe("failure.schema.json — required core", () => { expect(failure({ ...core, status: "exploded" })).toBe(false); }); + it("accepts an optional title; rejects an empty one", () => { + expect(failure({ ...core, title: "Checkout page renders blank" })).toBe(true); + expect(failure({ ...core, title: "" })).toBe(false); + }); + it("keeps forensic blocks open (locator_context, trajectory_refs, extra keys)", () => { expect( failure({ @@ -91,6 +96,11 @@ describe("failure-index.schema.json", () => { expect(failureIndex({ failures: [] })).toBe(true); }); + it("accepts an optional lifted title on a row; rejects an empty one", () => { + expect(failureIndex({ failures: [{ ...row, title: "Checkout page renders blank" }] })).toBe(true); + expect(failureIndex({ failures: [{ ...row, title: "" }] })).toBe(false); + }); + it("rejects a row missing a required field or with a bad enum", () => { const { path: _p, ...noPath } = row; expect(failureIndex({ failures: [noPath] })).toBe(false);