From 555379ecfc327f7bb1e43d109b5e9e37077a1847 Mon Sep 17 00:00:00 2001 From: alphali <5236230+alphali@users.noreply.github.com> Date: Wed, 16 Sep 2026 20:01:12 +0800 Subject: [PATCH] feat: declare the journey map and keep results observable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A round can already order its findings with journeyRef/step/role, and agents do: a replay against apob-react-web-app returned 38 findings, 82% of them ordered into 8 journeys. What no field carries is the journey's own name, goal and actor, so Doable titles each flow with one raw finding sentence and takes its goal from another. Add `journeys` to the answer candidate. A declaration names journeys the findings already ordered; it cannot introduce one. `normalizeJourneys` rejects a journeyRef with no ordered findings, rejects duplicates, and puts name, goal, actor and alternatePaths through the same provenance check as any other prose crossing the boundary. `journeyMapSkipReason` declines when a map would add nothing — an empty map with a reason beats a guessed one. Two fixes alongside it: - The observable-result rule did not say observable to whom. That same replay produced verifications a tester cannot check, such as "the vote is submitted through the Live scene-vote request" — the mechanism, not the result. State that where a behavior has a user-visible surface, the result is what that surface shows, and make the pre-submit review restate such findings. - The watch checklist expected `stop` for `consumed`. A pre-create Round now hands its code to the follow-up Rounds of the TRD it created, so the watch continues on the same DQ and the server answers `wait`. Only `cancelled`, or a Round bound to no session, stops. Co-Authored-By: Claude Opus 5 --- TESTING.md | 2 +- .../scripts/doable-code-context.mjs | 42 +++++++++++++++++++ .../skills/doable-answer-questions/SKILL.md | 5 ++- .../references/answer-contract.md | 14 ++++++- tests/doable-code-context-helper.test.mjs | 41 ++++++++++++++++++ 5 files changed, 100 insertions(+), 4 deletions(-) diff --git a/TESTING.md b/TESTING.md index a393125..62da08b 100644 --- a/TESTING.md +++ b/TESTING.md @@ -15,7 +15,7 @@ For every scenario, confirm that the agent inspects only evidence needed for the 7. **Mono-repo and multi-repo** — Confirm every independent Git root receives a stable opaque `repoRef`, while a common parent directory does not. Move one repository and explicitly reuse its `repoRef`; expect identity to survive the path change. 8. **Profile privacy** — Use repository names, paths, branches, commits, and an internal service name that differ from the safe product role. Capture the PUT body and confirm none appears remotely. The local state must retain them. 9. **Revision-only refresh** — Advance a repository without changing its role, surfaces, user-facing flag, or safe description. Expect a sync without new user approval. Change a material field and expect approval to be required. -10. **Watch one Round** — Pull a valid `DQ-...` code. Confirm `Next action: answer` while `open_for_agent` has open questions, `wait` for `ready_to_create` / `needs_attention`, and `stop` for `creating` / `consumed` / `cancelled`. A later pull may add `established_context` plus new open questions; the candidate must cover only the new open IDs. Do not treat `ready_to_create` as finished. +10. **Watch one Round** — Pull a valid `DQ-...` code. Confirm `Next action: answer` while `open_for_agent` has open questions, and `wait` for `ready_to_create` / `needs_attention`. A pre-create Round that reaches `creating` or `consumed` also answers `wait`: it hands its code to the follow-up Rounds of the TRD it just created, so the watch continues on the same DQ. Expect `stop` only for `cancelled`, or for a pre-create Round bound to no session. A later pull may add `established_context` plus new open questions; the candidate must cover only the new open IDs. Do not treat `ready_to_create` as finished. 11. **Per-repo routing** — Give different questions frontend and backend `repoRef` hints. Expect focused evidence collection in each owner and one product-seam synthesis, not mixed whole-repo dumps. 12. **Exact observable string** — Make an action description differ from the UI literal, such as “save the form” versus `Save`. Expect the finding and anchor to use the verified literal only. 13. **Existence versus absence** — Ask whether a validation exists. Positive evidence may establish existence. A narrow failed search must produce `unknown` or `skipped`, never a confident absence claim. diff --git a/plugins/doable-code-context/scripts/doable-code-context.mjs b/plugins/doable-code-context/scripts/doable-code-context.mjs index 1fe44f6..ef702a9 100644 --- a/plugins/doable-code-context/scripts/doable-code-context.mjs +++ b/plugins/doable-code-context/scripts/doable-code-context.mjs @@ -1222,6 +1222,42 @@ function validateFindingJourneys(findings) { } } +function normalizeJourneys(value, label, state, findings) { + assert(Array.isArray(value), `${label} must be an array`); + const ordered = new Set(findings.filter((finding) => finding.journey_ref).map((finding) => finding.journey_ref)); + const seen = new Set(); + return value.map((journey, index) => { + assert(journey && typeof journey === "object" && !Array.isArray(journey), `${label}[${index}] must be an object`); + const journeyRef = string(journey.journeyRef, `${label}[${index}].journeyRef`, { max: 42 }); + assert(JOURNEY_REF_RE.test(journeyRef), `${label}[${index}].journeyRef is invalid`); + // A declaration names findings; it never introduces a journey of its own. + assert(ordered.has(journeyRef), `${label}[${index}].journeyRef ${journeyRef} has no ordered findings`); + assert(!seen.has(journeyRef), `${label} declares ${journeyRef} more than once`); + seen.add(journeyRef); + const name = string(journey.name, `${label}[${index}].name`, { max: 200 }); + const goal = string(journey.goal, `${label}[${index}].goal`, { max: 500 }); + assertNoLocalProvenance(name, `${label}[${index}].name`, state); + assertNoLocalProvenance(goal, `${label}[${index}].goal`, state); + let actor; + if (journey.actor !== undefined && journey.actor !== null) { + actor = string(journey.actor, `${label}[${index}].actor`, { max: 120 }); + assertNoLocalProvenance(actor, `${label}[${index}].actor`, state); + } + const alternatePaths = (journey.alternatePaths || []).map((path, pathIndex) => { + const text = string(path, `${label}[${index}].alternatePaths[${pathIndex}]`, { max: 300 }); + assertNoLocalProvenance(text, `${label}[${index}].alternatePaths[${pathIndex}]`, state); + return text; + }); + return { + journey_ref: journeyRef, + name, + goal, + ...(actor ? { actor } : {}), + alternate_paths: alternatePaths, + }; + }); +} + function normalizeConflicts(value, label, state, findings) { assert(Array.isArray(value), `${label} must be an array`); const findingsByRef = new Map(findings.map((finding) => [finding.finding_ref, finding])); @@ -1370,6 +1406,10 @@ function buildSubmission(statePath, candidatePath) { ]; validateFindingJourneys(allFindings); const conflicts = normalizeConflicts(candidate.conflicts || [], "conflicts", privacyState, allFindings); + const journeys = normalizeJourneys(candidate.journeys || [], "journeys", privacyState, allFindings); + const journeyMapSkipReason = candidate.journeyMapSkipReason + ? string(candidate.journeyMapSkipReason, "journeyMapSkipReason", { max: 500 }) + : undefined; const payload = { round_revision: roundRevision, workspace_id: state.workspace.serverId, @@ -1377,6 +1417,8 @@ function buildSubmission(statePath, candidatePath) { agent_observations: agentObservations, conflicts, evidence_references: evidenceReferences, + ...(journeys.length ? { journeys } : {}), + ...(journeyMapSkipReason ? { journey_map_skip_reason: journeyMapSkipReason } : {}), }; assertRemotePayloadSafe(payload, privacyState); return { state, frozenRound, payload, payloadDigest: sha256(stableJson(payload)) }; diff --git a/plugins/doable-code-context/skills/doable-answer-questions/SKILL.md b/plugins/doable-code-context/skills/doable-answer-questions/SKILL.md index d273e02..a306487 100644 --- a/plugins/doable-code-context/skills/doable-answer-questions/SKILL.md +++ b/plugins/doable-code-context/skills/doable-answer-questions/SKILL.md @@ -62,6 +62,7 @@ node /scripts/doable-code-context.mjs ... - When two grounded code, human-authority, artifact, or runtime findings anywhere in the submission clearly contradict each other, give them stable `findingRef` values and add one explicit top-level `conflicts` relation. Do not mark ordinary truth-plane differences, complementary facts, or uncertain inferences as conflicts. - Bind every material claim to local evidence IDs or exact human clarification. Do not submit chain of thought. - Mark executable order only when inspected evidence establishes it. When two or more transitions form one tester journey in a definite order, add the optional `journeyRef`, `step`, and `role` fields defined in the answer contract. The annotation only groups existing findings; it never adds a claim. Put inseparable facts at the same step and omit all three fields when order is not established. Never order an `unknown` or `inference` finding. + - Name every journey you ordered. When the Round's `journey_map` asks for one, add a `journeys` entry per `journeyRef` with its `name`, `goal`, `actor`, and any `alternatePaths`, as defined in the answer contract. Ordering says which findings belong together; only this says what the journey is, and without it Doable titles the flow with one raw finding sentence. Declare nothing for a `journeyRef` you did not order, and when a map would add nothing, send an empty `journeys` with a `journeyMapSkipReason` instead of guessing one. 9. Ask the customer only when missing authority or a normative decision materially affects a required answer. Collect every such question first, ask one concise batched round, and preserve each exact question and verbatim answer. Do not ask for facts the code or supplied artifacts establish. A `human_clarification` finding's statement must be the exact submitted answer; place any interpretation in a separate inference finding. 10. Handle new discoveries without widening the round: @@ -71,7 +72,7 @@ node /scripts/doable-code-context.mjs ... The agent cannot create a new required question, defer a question, or waive scope; those remain platform-user actions. 11. Use `answered` only when at least one grounded finding addresses the question. Use `skipped` with a bounded reason when the workspace cannot answer it. Never send `deferred` or `waived` from the coding agent. Use only the contract truth-plane values `implemented_behavior`, `desired_behavior`, `artifact_observation`, `inference`, and `unknown`; do not invent adjacent confidence or evidence labels. -12. Before transport validation, review each confirmed finding against its first observable anchor: a reader seeing only that statement and compact quote must not infer an unrelated behavior. Split mixed validation families, conditional success branches with different outcomes, independent fixtures, or neighboring controls when the quote supports only one part. Delete operation-availability findings that still lack an observable result; do not retain them as an inventory. For a base answer, run the coverage check: for every capability with a submitted create or entry finding, confirm that the local ledger contains an explicit `included`, `out-of-scope`, or `ask-user` decision for its sibling lifecycle operations and configuration dimensions. An undecided sibling is a coverage defect; decide it from the ledger without rescanning. For supplemental answers, review coverage only against the propositions named by the open questions and do not add sibling coverage. Reuse the existing evidence and do not rescan merely to satisfy this review. Immediately before the first transport validation of this answer batch, report `phase: preparing_answers` with that same actual Round ID and revision. Do not report this repeatedly for each validation retry. Run `validate-submission`, repair all diagnostics without scanning unrelated code, then run `build-submission --output `. Submit the exact generated `submission` with Doable MCP `submit_code_context_round`; save the MCP response privately and run `record-submission --payload --response `. The helper strips local provenance, validates the privacy boundary, and checks that the frozen revision and payload were not mutated. MCP owns the remote idempotent submission. If submit reports that the open question set changed, re-pull, `record-round`, and answer only the new open IDs. After a successful submit, immediately pull again and follow `Next action`. Do not wait for the user to paste another prompt. +12. Before transport validation, review each confirmed finding against its first observable anchor: a reader seeing only that statement and compact quote must not infer an unrelated behavior. Split mixed validation families, conditional success branches with different outcomes, independent fixtures, or neighboring controls when the quote supports only one part. Delete operation-availability findings that still lack an observable result; do not retain them as an inventory. Where a finding's result is the call the product made rather than what the surface shows, restate it as the visible outcome. For a base answer, run the coverage check: for every capability with a submitted create or entry finding, confirm that the local ledger contains an explicit `included`, `out-of-scope`, or `ask-user` decision for its sibling lifecycle operations and configuration dimensions. An undecided sibling is a coverage defect; decide it from the ledger without rescanning. For supplemental answers, review coverage only against the propositions named by the open questions and do not add sibling coverage. Reuse the existing evidence and do not rescan merely to satisfy this review. Immediately before the first transport validation of this answer batch, report `phase: preparing_answers` with that same actual Round ID and revision. Do not report this repeatedly for each validation retry. Run `validate-submission`, repair all diagnostics without scanning unrelated code, then run `build-submission --output `. Submit the exact generated `submission` with Doable MCP `submit_code_context_round`; save the MCP response privately and run `record-submission --payload --response `. The helper strips local provenance, validates the privacy boundary, and checks that the frozen revision and payload were not mutated. MCP owns the remote idempotent submission. If submit reports that the open question set changed, re-pull, `record-round`, and answer only the new open IDs. After a successful submit, immediately pull again and follow `Next action`. Do not wait for the user to paste another prompt. ## Scope and safety @@ -82,4 +83,4 @@ node /scripts/doable-code-context.mjs ... ## Completion -For pre-create, when `Next action` is `stop`, report the round code, why watching ended, how many question batches were answered or skipped, any nonblocking observations, and how many evidence-backed journeys and distinct steps were declared. For follow-up, keep polling across applied or cancelled Rounds until the user stops the coding-agent task. If the host interrupts the turn and the user later says continue or resume without repeating the prompt, treat that as continuation of the most recent active follow-up connection: recover its original code from the conversation or private request history and immediately pull it again. Do not print the full safe payload, local evidence ledger, or hidden reasoning. +For pre-create, when `Next action` is `stop`, report the round code, why watching ended, how many question batches were answered or skipped, any nonblocking observations, and how many evidence-backed journeys and distinct steps were declared, and whether a journey map was sent or declined. For follow-up, keep polling across applied or cancelled Rounds until the user stops the coding-agent task. If the host interrupts the turn and the user later says continue or resume without repeating the prompt, treat that as continuation of the most recent active follow-up connection: recover its original code from the conversation or private request history and immediately pull it again. Do not print the full safe payload, local evidence ledger, or hidden reasoning. diff --git a/plugins/doable-code-context/skills/doable-answer-questions/references/answer-contract.md b/plugins/doable-code-context/skills/doable-answer-questions/references/answer-contract.md index c7f23f1..e0efec0 100644 --- a/plugins/doable-code-context/skills/doable-answer-questions/references/answer-contract.md +++ b/plugins/doable-code-context/skills/doable-answer-questions/references/answer-contract.md @@ -45,6 +45,16 @@ } ], "agentObservations": [], + "journeys": [ + { + "journeyRef": "j_create_promotion", + "name": "Create a promotion", + "goal": "Turn a drafted promotion into a live one buyers can redeem.", + "actor": "Store staff", + "alternatePaths": ["Submitting with an expired date range is rejected"] + } + ], + "journeyMapSkipReason": null, "conflicts": [ { "leftFindingRef": "f_savecode01", @@ -83,7 +93,7 @@ - `truthPlane`: `implemented_behavior`, `desired_behavior`, `artifact_observation`, `inference`, or `unknown`. - `sourceType`: `code`, `human_clarification`, `artifact`, `runtime`, or `inference`. - `statement`: one independently citable product proposition or one causally coherent state transition. Split unrelated lifecycle operations, validation families, outcomes, roles, and fixture facts. A statement that lists more than two independent operations or joins independent actions without one shared observable result is invalid; summary findings have no exception. -- Before submitting an executable transition, establish its entry or trigger, action or required input, and observable result. A capability inventory may route further inspection, but it is not itself an executable flow. If one of those elements remains material and unproven, narrow the confirmed finding and preserve the missing proposition as `unknown` instead of inventing a generic action or outcome. +- Before submitting an executable transition, establish its entry or trigger, action or required input, and observable result. Where the behavior has a user-visible surface, the observable result is what that surface shows. A request, mutation, or operation name is the mechanism, not the result: it says how the product did the thing, and a tester watching the screen cannot confirm it. Keep such a name as an anchor when it is genuinely the exposed contract, and still state the result the user can see. A capability inventory may route further inspection, but it is not itself an executable flow. If one of those elements remains material and unproven, narrow the confirmed finding and preserve the missing proposition as `unknown` instead of inventing a generic action or outcome. - Include a remote finding only when it changes test scope, fixture/setup data, an observable oracle, or a material environment boundary. Internal model fields, event taxonomies, webhook payloads, generated clients, and implementation helpers stay in the private ledger. A user-reachable operation that changes persistent state, or a configuration dimension that changes validation, fixtures, or outcomes, is not an internal inventory item: evaluate it on its merits and submit it as an atomic finding when it passes the selection gate. Delete executable-operation findings that still lack a grounded result or state change. - Represent inspected code as `implemented_behavior`; represent an observed runtime as `artifact_observation` with `sourceType: runtime`. Runtime reachability alone is not feature behavior. - Do not encode test strategy as implemented product behavior. Keep factual preconditions separate from derived fixture naming, isolation, or cleanup advice; use `inference` for a material evidence-backed recommendation or keep it local. @@ -91,6 +101,8 @@ - `evidenceRefIds`: IDs from the local evidence ledger. Do not put paths or symbols here. - `sourceFingerprint` is generated by the helper from the private evidence locator or exact matching clarification. Do not author or upload a local fingerprint field yourself. - `journeyRef`, `step`, and `role` are optional ordering metadata. Provide all three or none. `journeyRef` matches `j_[a-z0-9_]+`; `step` starts at 1 and is consecutive within a journey; `role` is `entry`, `precondition`, `action`, `outcome`, or `failure`. A journey needs at least two distinct steps and an entry or action. These fields add no claim: the finding must remain complete without them. Use the same step when evidence does not establish an order between two facts, omit ordering when the sequence is unknown, and never annotate an `unknown` or `inference` finding. +- `journeys` names the journeys your findings already ordered. Order lives on the findings; this is the only place the journey's own wording lives, so without it Doable titles the flow with one raw finding sentence and takes its goal from another. Declare one entry per `journeyRef` you used: `name` and `goal` in product terms, `actor` in the same role wording you used in the findings, and `alternatePaths` for branches that change the outcome and must stay attached to this journey rather than becoming journeys of their own. A `journeyRef` with no ordered findings is rejected. Leaving `journeys` empty is valid and keeps today's behavior. +- `journeyMapSkipReason` is how you decline when the round asked for a map and one would add nothing — a request that already arrives with its flows described, for example. Say why. An empty map with a reason is a better answer than a guessed one. - `humanClarifications`: exact `{ "question": "...", "answer": "..." }` pairs. Preserve the user's wording except mandatory secret or personal-data redaction. - For a `human_clarification` finding, `statement` must exactly equal one submitted clarification answer. Put interpretation in a separate `inference` finding. - Human clarification is authoritative only for the desired behavior or product decision that the diff --git a/tests/doable-code-context-helper.test.mjs b/tests/doable-code-context-helper.test.mjs index 0d0e4e4..792bdf7 100644 --- a/tests/doable-code-context-helper.test.mjs +++ b/tests/doable-code-context-helper.test.mjs @@ -350,6 +350,47 @@ test("connected helper preserves the local/private boundary and retries idempote runHelper(["validate-submission", "--state", statePath, "--candidate", submissionPath], environment), /cannot carry executable order/i, ); + // A declaration names journeys the findings already ordered. It must not be able to + // introduce one, or the journey map could assert structure no evidence supports. + const declaredJourneys = structuredClone(orderedJourney); + declaredJourneys.journeys = [ + { + journeyRef: "j_submit_promotion", + name: "Submit a promotion", + goal: "Turn a drafted promotion into one buyers can redeem.", + actor: "Store staff", + alternatePaths: ["Submitting an expired date range is rejected"], + }, + ]; + writeFileSync(submissionPath, `${JSON.stringify(declaredJourneys, null, 2)}\n`); + await runHelper(["validate-submission", "--state", statePath, "--candidate", submissionPath], environment); + + const unbackedJourney = structuredClone(declaredJourneys); + unbackedJourney.journeys[0].journeyRef = "j_nothing_orders_this"; + writeFileSync(submissionPath, `${JSON.stringify(unbackedJourney, null, 2)}\n`); + await assert.rejects( + runHelper(["validate-submission", "--state", statePath, "--candidate", submissionPath], environment), + /has no ordered findings/i, + ); + + const duplicateJourney = structuredClone(declaredJourneys); + duplicateJourney.journeys.push(structuredClone(declaredJourneys.journeys[0])); + writeFileSync(submissionPath, `${JSON.stringify(duplicateJourney, null, 2)}\n`); + await assert.rejects( + runHelper(["validate-submission", "--state", statePath, "--candidate", submissionPath], environment), + /more than once/i, + ); + + // The journey's own wording crosses the boundary like any other prose, so it gets the + // same provenance check rather than riding along unexamined. + const leakyJourney = structuredClone(declaredJourneys); + leakyJourney.journeys[0].goal = "Turn a draft into a live promotion in private-admin-repository."; + writeFileSync(submissionPath, `${JSON.stringify(leakyJourney, null, 2)}\n`); + await assert.rejects( + runHelper(["validate-submission", "--state", statePath, "--candidate", submissionPath], environment), + /journeys\[0\]\.goal exposes local repository provenance/i, + ); + writeFileSync(submissionPath, `${JSON.stringify(submission, null, 2)}\n`); const invalidStatus = structuredClone(submission);