diff --git a/docs/exit-codes.md b/docs/exit-codes.md new file mode 100644 index 0000000..9a3b3d5 --- /dev/null +++ b/docs/exit-codes.md @@ -0,0 +1,42 @@ +# Exit codes + +`tooltrace-bench` treats its exit codes as a public contract: CI gates and onboarding +scripts branch on them, so an existing code never changes meaning -- new ones +are appended. + +Defined in [`tooltrace/core/exceptions.py`](../tooltrace/core/exceptions.py). + +| Code | Name | Meaning | +|---|---|---| +| `0` | `(success)` | No exception was raised. | +| `1` | `ToolTraceError` | Base class for all ToolTrace Bench errors. | +| `2` | `TaskValidationError` | A task definition failed schema or semantic validation. | +| `3` | `SandboxError` | Sandbox creation, enforcement or cleanup failed. | +| `4` | `PolicyViolation` | An agent action violated task policy (tool allowlist, boundary, network). | +| `5` | `AgentError` | An agent adapter failed to initialize or run. | +| `6` | `BundleError` | A result bundle is missing, corrupt or fails checksum verification. | +| `7` | `ComparisonError` | Two runs cannot be compared (incompatible versions/protocols). | +| `8` | `RegressionThresholdError` | A regression check failed its configured thresholds. | +| `9` | `SecretScanError` | Likely secrets were detected in content destined for publication. | + +## If you use more than one of these tools + +These four projects are independent and their exit codes are **not** a shared +vocabulary. Only `0` means the same thing in all of them (success). Every other +code differs, and two collisions are worth knowing before you write a wrapper: + +| Code | api-verity-lab | devrepro-doctor | tooltrace-bench | local-ai-hardware-bench | +|---|---|---|---|---| +| 1 | findings detected | **ready, with warnings** | error | validation error | +| 2 | usage error | **machine blocked** | task validation error | usage error | + +The dangerous one is `1`. In devrepro-doctor it means *the machine is usable*; +in the other three it means something went wrong. A wrapper that treats any +non-zero status as failure will block on a DevRepro run that reported success. + +The second is `2`: an operator mistake in two of them, and devrepro-doctor's +most important verdict -- the machine cannot build this project -- in the third. + +These are not being unified. A shared exit-code library would couple four +independent release cycles, and one of these projects deliberately ships with +no dependencies at all. Knowing the difference is cheaper than removing it. diff --git a/tests/test_exit_codes_documented.py b/tests/test_exit_codes_documented.py new file mode 100644 index 0000000..cad80fc --- /dev/null +++ b/tests/test_exit_codes_documented.py @@ -0,0 +1,76 @@ +"""The documented exit codes must match the ones the code defines. + +Every project in this family calls its exit codes a public contract -- CI +gates and onboarding scripts branch on them, so a code that changes meaning +breaks something silently, somewhere else. A *document* that disagrees with the +code is worse than no document, because wrappers get written from the document. + +Checking this is cheap, so it is checked. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +_ROOT = Path(__file__).resolve().parent.parent +_DOC = _ROOT / "docs" / "exit-codes.md" + + +def _documented() -> dict[int, str]: + rows = re.findall( + r"^\|\s*`(\d+)`\s*\|\s*`([^`]+)`\s*\|", _DOC.read_text(encoding="utf-8"), re.M + ) + return {int(code): name for code, name in rows} + + +def _defined() -> dict[int, str]: + import ast + + src = (_ROOT / "tooltrace/core/exceptions.py").read_text(encoding="utf-8") + out: dict[int, str] = {0: "(success)"} + for node in ast.parse(src).body: + if not isinstance(node, ast.ClassDef): + continue + for stmt in node.body: + if ( + isinstance(stmt, ast.Assign) + and getattr(stmt.targets[0], "id", "") == "exit_code" + and isinstance(stmt.value, ast.Constant) + ): + out[stmt.value.value] = node.name + return out + + +def test_every_defined_code_is_documented() -> None: + documented, defined = _documented(), _defined() + missing = sorted(set(defined) - set(documented)) + assert not missing, f"exit codes defined in code but absent from docs/exit-codes.md: {missing}" + + +def test_no_documented_code_is_invented() -> None: + documented, defined = _documented(), _defined() + extra = sorted(set(documented) - set(defined)) + assert not extra, f"docs/exit-codes.md documents codes the code does not define: {extra}" + + +def test_each_code_is_documented_with_its_real_name() -> None: + documented, defined = _documented(), _defined() + wrong = { + code: (documented[code], defined[code]) + for code in sorted(set(documented) & set(defined)) + if documented[code] != defined[code] + } + assert not wrong, f"docs name codes differently from the code: {wrong}" + + +def test_the_cross_project_warning_is_present() -> None: + """The collision is the reason this document exists. + + Only `0` means the same thing across the four sibling projects. In + devrepro-doctor `1` means the machine is usable; elsewhere it means + failure. A wrapper treating non-zero as failure blocks a successful run. + """ + text = _DOC.read_text(encoding="utf-8") + assert "If you use more than one of these tools" in text + assert "devrepro-doctor" in text