diff --git a/.changeset/tidy-tasks-verify.md b/.changeset/tidy-tasks-verify.md new file mode 100644 index 0000000000..783dd473f6 --- /dev/null +++ b/.changeset/tidy-tasks-verify.md @@ -0,0 +1,5 @@ +--- +"@fission-ai/openspec": patch +--- + +Require generated tasks to state how their completion can be verified. diff --git a/schemas/spec-driven/schema.yaml b/schemas/spec-driven/schema.yaml index ae4d9eb336..d38c6a357a 100644 --- a/schemas/spec-driven/schema.yaml +++ b/schemas/spec-driven/schema.yaml @@ -182,22 +182,26 @@ artifacts: - Each task MUST be a checkbox: `- [ ] X.Y Task description` - Tasks should be small enough to complete in one session - Order tasks by dependency (what must be done first?) + - Each task MUST state how to verify completion (a test, command, + observable behavior, or delivered artifact). Put the verification in + that task's checkbox description. Use a separate verification task only + when it checks broader integration or system behavior that spans + multiple implementation tasks. Example: ``` ## 1. Setup - - [ ] 1.1 Create new module structure - - [ ] 1.2 Add dependencies to package.json + - [ ] 1.1 Create new module structure and verify expected files are present + - [ ] 1.2 Add dependencies to package.json and verify package installation succeeds ## 2. Core Implementation - - [ ] 2.1 Implement data export function - - [ ] 2.2 Add CSV formatting utilities + - [ ] 2.1 Implement data export function and verify the export test passes + - [ ] 2.2 Add CSV formatting utilities and verify unit tests cover quoting and delimiters ``` Reference specs for what needs to be built, design for how to build it. - Each task should be verifiable - you know when it's done. requires: - specs - design diff --git a/skills/openspec-onboard/SKILL.md b/skills/openspec-onboard/SKILL.md index 5619596ea1..fb3f13bec7 100644 --- a/skills/openspec-onboard/SKILL.md +++ b/skills/openspec-onboard/SKILL.md @@ -373,12 +373,12 @@ Here are the implementation tasks: ## 1. [Category or file] -- [ ] 1.1 [Specific task] -- [ ] 1.2 [Specific task] +- [ ] 1.1 [Specific task] — verify: [test, command, observable behavior, or delivered artifact] +- [ ] 1.2 [Specific task] — verify: [test, command, observable behavior, or delivered artifact] -## 2. Verify +## 2. Integration Verification -- [ ] 2.1 [Verification step] +- [ ] 2.1 Verify [broader integration or system behavior] with [end-to-end test or observable result] --- diff --git a/src/core/templates/workflows/onboard.ts b/src/core/templates/workflows/onboard.ts index 743c71ff8d..414c6e18b5 100644 --- a/src/core/templates/workflows/onboard.ts +++ b/src/core/templates/workflows/onboard.ts @@ -383,12 +383,12 @@ Here are the implementation tasks: ## 1. [Category or file] -- [ ] 1.1 [Specific task] -- [ ] 1.2 [Specific task] +- [ ] 1.1 [Specific task] — verify: [test, command, observable behavior, or delivered artifact] +- [ ] 1.2 [Specific task] — verify: [test, command, observable behavior, or delivered artifact] -## 2. Verify +## 2. Integration Verification -- [ ] 2.1 [Verification step] +- [ ] 2.1 Verify [broader integration or system behavior] with [end-to-end test or observable result] --- diff --git a/test/core/templates/propose.test.ts b/test/core/templates/propose.test.ts index aeff3089af..0ac2e6201f 100644 --- a/test/core/templates/propose.test.ts +++ b/test/core/templates/propose.test.ts @@ -61,6 +61,34 @@ describe('propose preamble', () => { }); }); +describe('default task guidance', () => { + it('requires a concrete verification method in each task (#345)', () => { + const tasks = defaultSchema.artifacts.find(artifact => artifact.id === 'tasks'); + expect(tasks).toBeDefined(); + expect(tasks!.instruction).toContain('Each task MUST state how to verify completion'); + expect(tasks!.instruction).toMatch( + /a test, command,\s+observable behavior, or delivered artifact/ + ); + expect(tasks!.instruction).toMatch( + /Put the verification in\s+that task's checkbox description/ + ); + expect(tasks!.instruction).toMatch( + /Use a separate verification task only\s+when it checks broader integration or system behavior that spans\s+multiple implementation tasks/ + ); + + const example = tasks!.instruction.match(/```\s*([\s\S]*?)```/)?.[1]; + expect(example).toBeDefined(); + const numberedTasks = example!.split('\n').filter(line => /^- \[ \] \d+\.\d+ /.test(line)); + expect(numberedTasks).toHaveLength(4); + expect(numberedTasks.every(line => /\bverify\b/i.test(line))).toBe(true); + expect(numberedTasks[0]).toContain('expected files are present'); + expect(numberedTasks[1]).toContain('package installation succeeds'); + expect(numberedTasks[2]).toContain('export test passes'); + expect(numberedTasks[3]).toContain('unit tests cover quoting and delimiters'); + expect(example).not.toMatch(/^- \[ \] \d+\.\d+ (?:verify|run (?:the )?verification)\b/im); + }); +}); + describe('propose implementation boundary', () => { it('makes the planning-only boundary prominent (#232, #258, #262)', () => { for (const [label, body] of proposeBodies) { diff --git a/test/core/templates/skill-templates-parity.test.ts b/test/core/templates/skill-templates-parity.test.ts index decad7b0b5..fc66975478 100644 --- a/test/core/templates/skill-templates-parity.test.ts +++ b/test/core/templates/skill-templates-parity.test.ts @@ -44,7 +44,7 @@ const EXPECTED_FUNCTION_HASHES: Record = { getApplyChangeSkillTemplate: 'd1e7d5ceb85193c0964057dbb88e9651526754bd33f84020e2440ff0621d5dbb', getFfChangeSkillTemplate: '5501740e7ec36ab23ab8c3a0d6dd0655a5e2f35433c7b90e82904fef5e7a326a', getSyncSpecsSkillTemplate: 'b099e2ff31859c9b10d928066e662524f9aad9ecf2be12fceacb732d718c4146', - getOnboardSkillTemplate: '29b1d825179cff92fbc7b790694c1baef138575ea3de56848715e27d7e367946', + getOnboardSkillTemplate: '3a836faae463d88c289a1c129cb7ee556a563b7e53e1a52a4711ff152a3b51f7', getOpsxExploreCommandTemplate: 'd2f70d11588f902c15c1e5ce9908cc4124c6b82fe78dc766ac5c3599c9e2a6f1', getOpsxNewCommandTemplate: 'f2d30e569798a4c92ba932859d6ba4e0ad10e18feccbade1cfee0957597b3463', getOpsxContinueCommandTemplate: 'e50e50266efa1b8e64ff9b6274ee8254f0a240d6adc1b862d126e2f1c9d3a559', @@ -55,7 +55,7 @@ const EXPECTED_FUNCTION_HASHES: Record = { getOpsxSyncCommandTemplate: '0d2427efb79986e8fff3f96bd075a739c80d45eb29159fae717e950030da8202', getVerifyChangeSkillTemplate: '223b7ffd99299a7d430e13092b9a0a3421b39f0d3217232f46c39d79b5f619ff', getOpsxArchiveCommandTemplate: '9f973c819b11620985b03322945f0e0a92a02a2ef455b94e74482f5e6292ac5d', - getOpsxOnboardCommandTemplate: '7e251da66e2fdf539a09326463ee3ed0d01fe665ecb1d8f36f941fed00a01891', + getOpsxOnboardCommandTemplate: 'ee99aa99252c602720fbb8c63fb3ac438a5bd4e952fd961ddf1ae956cbfc2c8f', getOpsxBulkArchiveCommandTemplate: '9fa8cdebe2f5667ebfc37bdc023396762c59d5b038c771dac2d8fd2c19e2627b', getOpsxVerifyCommandTemplate: '1efcf7eff0671f48e9d9420f50865c563dd3079ee60f8c380bb7a90dd0102696', getOpsxProposeSkillTemplate: '24623c066f97e34b957d448d1f9a9e8b8a13da3dfce45d45671f6226a2534848', @@ -75,7 +75,7 @@ const EXPECTED_GENERATED_SKILL_CONTENT_HASHES: Record = { 'openspec-archive-change': '7c65053d674ba4e1e20e2bf73ba7e5a7f94baef2eaa9b33cee48d4cadea51b7a', 'openspec-bulk-archive-change': '2039b9ecf6e64339dffe0e16272507a386d9fe326f419ff758315aa736fdd96c', 'openspec-verify-change': 'af9be013dcbe8c6d8f6d9ab10c893fbd03f4c62933c384d82f63894dd0ceb84f', - 'openspec-onboard': 'd53403b4910ab64307862ccf97e70bd8f7174ee44508088fb239c880f0939331', + 'openspec-onboard': 'f6f59476acaf5e4d65dbb180da4cef62432612f3cecf207d471a951295e2003a', 'openspec-propose': '25d08ed4f031770cea219604167d76bca9f3e89fe0c2f545263674482c6f13f0', 'openspec-update-change': '586547406aca94422dfeb3ffedce6c01049429b743f57ce829baa79ebc714d51', }; @@ -382,6 +382,40 @@ describe('skill templates split parity', () => { } }); + it('keeps onboarding task examples aligned with concrete verification guidance (#345)', () => { + const variants: Array<[string, string]> = [ + ['onboard skill', generateSkillContent(getOnboardSkillTemplate(), 'PARITY-BASELINE')], + ['onboard command', getOpsxOnboardCommandTemplate().content], + ]; + + for (const [label, content] of variants) { + const taskBlock = content.match( + /Here are the implementation tasks:([\s\S]*?)Each checkbox becomes a unit of work/ + )?.[1]; + expect(taskBlock, label).toBeDefined(); + const checkboxes = taskBlock! + .split('\n') + .filter(line => /^- \[ \] \d+\.\d+ /.test(line)); + expect(checkboxes, label).toHaveLength(3); + expect( + checkboxes.every( + line => + line.endsWith( + '[Specific task] — verify: [test, command, observable behavior, or delivered artifact]' + ) || / Verify .+ with \[.+\]$/.test(line) + ), + label + ).toBe(true); + expect(content, label).toContain( + '[Specific task] — verify: [test, command, observable behavior, or delivered artifact]' + ); + expect(content, label).toContain( + 'Verify [broader integration or system behavior] with [end-to-end test or observable result]' + ); + expect(content, label).not.toContain('[Verification step]'); + } + }); + it('generates no workspace-planning residue in any workflow template (4.1)', () => { const allSkills: Array<[string, () => SkillTemplate]> = [ ['openspec-apply-change', getApplyChangeSkillTemplate],