From dd1f2c90d16e38fbd4514c274a8f868e35f54788 Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Thu, 5 Nov 2026 10:49:31 -0800 Subject: [PATCH 1/3] docs: clarify AI check limits and remove internal license note in README Closes github/evals-internal#514 Closes github/evals-internal#515 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 4 +--- tests/example.test.ts | 9 +++++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cdba43f..544e1ec 100644 --- a/README.md +++ b/README.md @@ -305,6 +305,7 @@ Lint success is `validated`, not an eval pass. A fully evidenced empty run with exit zero warns and reports `not-run`: nonblocking but **not a pass**. Ambiguous empty discovery with nonzero exit or missing JSONL remains an error. Ungraded trials and weighted thresholds retain Vally's native semantics. +A passing check doesn't guarantee the AI will get it right every time. | Output | Meaning | | --- | --- | @@ -437,9 +438,6 @@ Bundled dependencies are listed in [`dist/bundled-dependencies.json`](dist/bundled-dependencies.json), with available full terms in [`dist/THIRD-PARTY-NOTICES.txt`](dist/THIRD-PARTY-NOTICES.txt) and retained inline notices in [`dist/index.cjs.LEGAL.txt`](dist/index.cjs.LEGAL.txt). -Entries marked `missing-license-text` still need authoritative terms and -release-owner clearance before public distribution; packaging checks do not -grant that clearance. Use Node 24 and the pinned pnpm version. Run `pnpm check`, rebuild and commit `dist/index.cjs`, and follow [CONTRIBUTING.md](CONTRIBUTING.md). Security reports diff --git a/tests/example.test.ts b/tests/example.test.ts index 2aa32ed..7634b67 100644 --- a/tests/example.test.ts +++ b/tests/example.test.ts @@ -40,6 +40,15 @@ describe('telemetry workflow contracts', () => { }); }); +describe('documentation content contracts', () => { + it('states in Outcomes and evidence that passing checks do not guarantee AI correctness and omits the internal license-clearance note', async () => { + const readme = await readFile('README.md', 'utf8'); + expect(readme).toContain("A passing check doesn't guarantee the AI will get it right every time."); + expect(readme).not.toContain('Entries marked `missing-license-text` still need authoritative terms'); + expect(readme).not.toContain('release-owner clearance before public distribution'); + }); +}); + describe('real Copilot example contract', () => { it('documents separate jobs with one automatic reporter and no artifact wiring', async () => { const readme = await readFile('README.md', 'utf8'); From 05f8d8dbb7004bf2badfb641865007004a5b4262 Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Thu, 5 Nov 2026 10:53:35 -0800 Subject: [PATCH 2/3] test: scope Outcomes and evidence README assertion to its section Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/example.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/example.test.ts b/tests/example.test.ts index 7634b67..0c48b5a 100644 --- a/tests/example.test.ts +++ b/tests/example.test.ts @@ -43,7 +43,8 @@ describe('telemetry workflow contracts', () => { describe('documentation content contracts', () => { it('states in Outcomes and evidence that passing checks do not guarantee AI correctness and omits the internal license-clearance note', async () => { const readme = await readFile('README.md', 'utf8'); - expect(readme).toContain("A passing check doesn't guarantee the AI will get it right every time."); + const outcomesMatch = /^## Outcomes and evidence\n([\s\S]*?)(?=\n## |\s*$)/m.exec(readme); + expect(outcomesMatch?.[1]).toContain("A passing check doesn't guarantee the AI will get it right every time."); expect(readme).not.toContain('Entries marked `missing-license-text` still need authoritative terms'); expect(readme).not.toContain('release-owner clearance before public distribution'); }); From 7078847acf7fe6e195692b17d77fcad97597825d Mon Sep 17 00:00:00 2001 From: Jon Gallant <2163001+jongio@users.noreply.github.com> Date: Thu, 5 Nov 2026 10:54:23 -0800 Subject: [PATCH 3/3] fix(test): anchor Outcomes and evidence section regex to end-of-string Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- tests/example.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/example.test.ts b/tests/example.test.ts index 0c48b5a..5301f3c 100644 --- a/tests/example.test.ts +++ b/tests/example.test.ts @@ -43,7 +43,7 @@ describe('telemetry workflow contracts', () => { describe('documentation content contracts', () => { it('states in Outcomes and evidence that passing checks do not guarantee AI correctness and omits the internal license-clearance note', async () => { const readme = await readFile('README.md', 'utf8'); - const outcomesMatch = /^## Outcomes and evidence\n([\s\S]*?)(?=\n## |\s*$)/m.exec(readme); + const outcomesMatch = /\n## Outcomes and evidence\n([\s\S]*?)(?=\n## |$)/.exec(readme); expect(outcomesMatch?.[1]).toContain("A passing check doesn't guarantee the AI will get it right every time."); expect(readme).not.toContain('Entries marked `missing-license-text` still need authoritative terms'); expect(readme).not.toContain('release-owner clearance before public distribution');