From 3aa642e217926696e5d8f8a7e0ebcfe87473cd0b Mon Sep 17 00:00:00 2001 From: Ugur Cekmez Date: Sun, 21 Jun 2026 19:07:11 +0300 Subject: [PATCH] fix(compliance-cli): require earned checks for a conformance medal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The conformance label was awarded whenever `failed === 0`, so a run with zero passing checks or only skipped checks still printed "Full EEP Compliant" — a near-empty run could claim conformance it never verified. Now a medal requires no failures, at least one passing check, and zero skipped checks. Runs with skips (but no failures) report " EEP: incomplete (N skipped, M passed)"; runs that verified nothing report "Not EEP Compliant (no checks verified)". Behavior is identical across the TypeScript and Python CLIs. Exit codes are unchanged (still keyed on failures) to avoid breaking existing CI. Signed-off-by: Ugur Cekmez Co-Authored-By: Claude Opus 4.8 (1M context) --- .../compliance-cli/src/helpers.test.ts | 25 ++++++++++++++++ .../@eep-dev/compliance-cli/src/helpers.ts | 27 +++++++++++++---- .../eep_compliance_cli/helpers.py | 30 +++++++++++++------ .../tests/test_helpers.py | 21 +++++++++++++ 4 files changed, 89 insertions(+), 14 deletions(-) diff --git a/packages/@eep-dev/compliance-cli/src/helpers.test.ts b/packages/@eep-dev/compliance-cli/src/helpers.test.ts index dc2b18b..9c852db 100644 --- a/packages/@eep-dev/compliance-cli/src/helpers.test.ts +++ b/packages/@eep-dev/compliance-cli/src/helpers.test.ts @@ -105,6 +105,31 @@ describe('@eep-dev/compliance-cli helpers', () => { runner.fail('t1', 'err'); expect(runner.conformanceLabel('core')).toBe('❌ Not EEP Compliant (1 failure)'); }); + + it('should NOT award a medal when no checks ran (empty run)', () => { + const runner = createTestRunner(); + expect(runner.conformanceLabel('full')).toBe('❌ Not EEP Compliant (no checks verified)'); + }); + + it('should NOT award a medal when every check skipped', () => { + const runner = createTestRunner(); + runner.skip('t1', 'n/a'); + runner.skip('t2', 'n/a'); + expect(runner.conformanceLabel('full')).toBe('❌ Not EEP Compliant (no checks verified)'); + }); + + it('should report "incomplete" when some checks skipped (no failures)', () => { + const runner = createTestRunner(); + runner.pass('t1'); + runner.skip('t2', 'n/a'); + expect(runner.conformanceLabel('full')).toBe('⚠️ Full EEP: incomplete (1 skipped, 1 passed)'); + }); + + it('should fall back to a generic compliant label for an unknown level', () => { + const runner = createTestRunner(); + runner.pass('t1'); + expect(runner.conformanceLabel('enterprise')).toBe('✅ Enterprise EEP Compliant'); + }); }); // ── validateArgs ──────────────────────────────────────────────── diff --git a/packages/@eep-dev/compliance-cli/src/helpers.ts b/packages/@eep-dev/compliance-cli/src/helpers.ts index 5e7d6d1..7f7b5b9 100644 --- a/packages/@eep-dev/compliance-cli/src/helpers.ts +++ b/packages/@eep-dev/compliance-cli/src/helpers.ts @@ -40,11 +40,28 @@ export function createTestRunner() { } function conformanceLabel(level: string): string { - const { failed } = summary(); - if (failed === 0 && level === 'core') return '🥉 Core EEP Compliant'; - if (failed === 0 && level === 'standard') return '🥈 Standard EEP Compliant'; - if (failed === 0 && level === 'full') return '🏆 Full EEP Compliant'; - return `❌ Not EEP Compliant (${failed} failure${failed !== 1 ? 's' : ''})`; + const { passed, failed, skipped } = summary(); + const lvl = level.charAt(0).toUpperCase() + level.slice(1); + if (failed > 0) { + return `❌ Not EEP Compliant (${failed} failure${failed !== 1 ? 's' : ''})`; + } + // A conformance claim must be *earned*, not granted by omission. A run + // with zero passing checks, or with skipped checks, has not actually + // verified the level — so it must not receive a compliance medal even + // though `failed === 0`. (Otherwise a near-empty or fully-skipped run + // would print "Full EEP Compliant".) + if (passed === 0) { + return '❌ Not EEP Compliant (no checks verified)'; + } + if (skipped > 0) { + return `⚠️ ${lvl} EEP: incomplete (${skipped} skipped, ${passed} passed)`; + } + const labels: Record = { + core: '🥉 Core EEP Compliant', + standard: '🥈 Standard EEP Compliant', + full: '🏆 Full EEP Compliant', + }; + return labels[level] ?? `✅ ${lvl} EEP Compliant`; } return { pass, fail, skip, results, summary, conformanceLabel }; diff --git a/packages/eep-compliance-cli-python/eep_compliance_cli/helpers.py b/packages/eep-compliance-cli-python/eep_compliance_cli/helpers.py index ed1bd5e..9e994ee 100644 --- a/packages/eep-compliance-cli-python/eep_compliance_cli/helpers.py +++ b/packages/eep-compliance-cli-python/eep_compliance_cli/helpers.py @@ -43,15 +43,27 @@ def summary(self) -> Dict[str, int]: def conformance_label(self, level: str) -> str: s = self.summary() - if s["failed"] == 0: - labels = { - "core": "🥉 Core EEP Compliant", - "standard": "🥈 Standard EEP Compliant", - "full": "🏆 Full EEP Compliant", - } - return labels.get(level, f"✅ {level.title()} EEP Compliant") - count = s["failed"] - return f"❌ Not EEP Compliant ({count} failure{'s' if count != 1 else ''})" + if s["failed"] > 0: + count = s["failed"] + return f"❌ Not EEP Compliant ({count} failure{'s' if count != 1 else ''})" + # A conformance claim must be *earned*, not granted by omission. A run + # with zero passing checks, or with skipped checks, has not actually + # verified the level — so it must not receive a compliance medal even + # though ``failed == 0``. (Otherwise a near-empty or fully-skipped run + # would print "Full EEP Compliant".) + if s["passed"] == 0: + return "❌ Not EEP Compliant (no checks verified)" + if s["skipped"] > 0: + return ( + f"⚠️ {level.title()} EEP: incomplete " + f"({s['skipped']} skipped, {s['passed']} passed)" + ) + labels = { + "core": "🥉 Core EEP Compliant", + "standard": "🥈 Standard EEP Compliant", + "full": "🏆 Full EEP Compliant", + } + return labels.get(level, f"✅ {level.title()} EEP Compliant") def normalize_target(url: str) -> str: diff --git a/packages/eep-compliance-cli-python/tests/test_helpers.py b/packages/eep-compliance-cli-python/tests/test_helpers.py index 119c9f4..d30c0a1 100644 --- a/packages/eep-compliance-cli-python/tests/test_helpers.py +++ b/packages/eep-compliance-cli-python/tests/test_helpers.py @@ -37,6 +37,27 @@ def test_conformance_label_failure(self): assert "Not EEP Compliant" in label assert "1 failure" in label + def test_conformance_label_empty_run_not_compliant(self): + r = TestRunner() + assert r.conformance_label("full") == "❌ Not EEP Compliant (no checks verified)" + + def test_conformance_label_all_skipped_not_compliant(self): + r = TestRunner() + r.skip("A", "n/a") + r.skip("B", "n/a") + assert r.conformance_label("full") == "❌ Not EEP Compliant (no checks verified)" + + def test_conformance_label_partial_skips_incomplete(self): + r = TestRunner() + r.pass_("A") + r.skip("B", "n/a") + assert r.conformance_label("full") == "⚠️ Full EEP: incomplete (1 skipped, 1 passed)" + + def test_conformance_label_unknown_level_fallback(self): + r = TestRunner() + r.pass_("A") + assert r.conformance_label("enterprise") == "✅ Enterprise EEP Compliant" + class TestNormalizeTarget: def test_strips_trailing_slash(self):