Fail the check only from a chosen severity

docs/usage.md

@@ -31,7 +31,8 @@ coverage: 2/2 files, 12/12 functions
 ```
 
 The exit status is 0 when nothing was found, 1 when findings were reported and 2 on a
-usage or analysis error, so the command can gate a pipeline as it is.
+usage or analysis error, so the command can gate a pipeline as it is. `--fail-on high`
+keeps reporting every finding but fails only from that severity up.
 
 ## Command line
 
@@ -52,6 +53,7 @@ coretrace-python-analyzer [--check | --emit-ir [--ssa]] [options] [path]
 | `--advisories FILE` | Read a local advisory file in addition to `advisories.json` at the root. Repeatable. |
 | `--policy FILE` | Apply this dependency policy instead of `coretrace-policy.toml` at the root. |
 | `--import-advisories SRC OUT` | Convert an OSV dump into the local advisory file `OUT`, then exit. |
+| `--fail-on SEVERITY` | Exit with status 1 only when a finding of this severity or above was reported: `info`, `low`, `medium`, `high` or `critical`. Default: any finding. |
 | `--baseline PATH` | The accepted findings: written on the first run, then only findings not recorded there fail the check. |
 | `--emit-ir` | Print the intermediate representation of `path` instead of checking it. |
 | `--ssa` | With `--emit-ir`, print the static single assignment form. |
@@ -310,7 +312,7 @@ Actions:
 
 ```yaml
 - run: pip install coretrace-python-analyzer
-- run: coretrace-python-analyzer --check src/ --format sarif > coretrace.sarif
+- run: coretrace-python-analyzer --check src/ --fail-on high --format sarif > coretrace.sarif
   continue-on-error: true
 - uses: github/codeql-action/upload-sarif@v3
   with:

src/coretrace_python/cli.py

@@ -10,6 +10,7 @@
 from coretrace_python.cache import ProjectCache
 from coretrace_python.cfg import CFGError
 from coretrace_python.dependency import dump_advisories, import_osv, read_osv, render_sbom
+from coretrace_python.findings import Severity
 from coretrace_python.findings.baseline import Baseline, BaselineError
 from coretrace_python.frontend import HIRBuildError, ParseError, build_hir
 from coretrace_python.ir.lowering import LoweringError, lower_module
@@ -130,6 +131,14 @@ def build_parser() -> argparse.ArgumentParser:
         help="with --check, the accepted findings: written on the first run, then only "
         "findings not recorded there fail the check",
     )
+    parser.add_argument(
+        "--fail-on",
+        choices=[severity.value for severity in Severity],
+        default=None,
+        metavar="SEVERITY",
+        help="with --check, exit with status 1 only when a finding of this severity or "
+        "above was reported (info, low, medium, high, critical); default: any finding",
+    )
     parser.add_argument(
         "--import-advisories",
         nargs=2,
@@ -180,6 +189,9 @@ def main(argv: list[str] | None = None) -> int:
         print("error: --policy only applies to --check on a directory", file=sys.stderr)
         return EXIT_ERROR
 
+    if args.fail_on is not None and not args.check:
+        print("error: --fail-on only applies to --check", file=sys.stderr)
+        return EXIT_ERROR
     if args.baseline is not None and not args.check:
         print("error: --baseline only applies to --check", file=sys.stderr)
         return EXIT_ERROR
@@ -229,7 +241,9 @@ def main(argv: list[str] | None = None) -> int:
                     findings, baselined = (), findings
             report = engine.report(findings, coverage, root, suppressed, baselined)
             print(render(args.format or "text", report), end="")
-            return EXIT_FINDINGS if findings else EXIT_CLEAN
+            threshold = Severity(args.fail_on).rank if args.fail_on is not None else 0
+            failing = any(finding.severity.rank >= threshold for finding in findings)
+            return EXIT_FINDINGS if failing else EXIT_CLEAN
     except _ANALYSIS_ERRORS as error:
         print(f"error: {error}", file=sys.stderr)
         return EXIT_ERROR

src/coretrace_python/findings/model.py

@@ -24,6 +24,15 @@ class Severity(Enum):
     HIGH = "high"
     CRITICAL = "critical"
 
+    @property
+    def rank(self) -> int:
+        """Position in the order of severities, ``INFO`` lowest, ``CRITICAL`` highest."""
+
+        return _SEVERITY_ORDER.index(self)
+
+
+_SEVERITY_ORDER = (Severity.INFO, Severity.LOW, Severity.MEDIUM, Severity.HIGH, Severity.CRITICAL)
+
 
 class Confidence(Enum):
     LOW = "low"

tests/test_fail_on.py

@@ -0,0 +1,77 @@
+"""Acceptance tests for the severity threshold (issue #68).
+
+``--fail-on SEVERITY`` makes the exit status 1 only when a finding of that severity or
+above was reported; every finding is still printed. Without it, any finding fails the
+check as before. Suppressed and baselined findings never count.
+
+Expected to remain red until ``Severity`` is ordered and ``--fail-on`` exists.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from coretrace_python.cli import main
+from coretrace_python.findings import Severity
+
+MISSING = None if hasattr(Severity, "rank") else "Severity has no rank"
+
+
+@pytest.fixture(autouse=True)
+def require_threshold() -> None:
+    if MISSING is not None:
+        pytest.fail(f"the severity threshold is not implemented yet: {MISSING}")
+
+
+HIGH_AND_MEDIUM = "import hashlib\n\ndef run(code):\n    eval(code)\n    hashlib.md5(code)\n"
+
+
+def write(tmp_path: Path, text: str) -> Path:
+    source = tmp_path / "app.py"
+    source.write_text(text, encoding="utf-8")
+    return source
+
+
+def test_severities_are_ordered() -> None:
+    ranks = [s.rank for s in (Severity.INFO, Severity.LOW, Severity.MEDIUM, Severity.HIGH, Severity.CRITICAL)]
+
+    assert ranks == sorted(ranks) and len(set(ranks)) == 5
+
+
+def test_findings_below_the_threshold_are_reported_but_do_not_fail(tmp_path: Path, capsys) -> None:  # type: ignore[no-untyped-def]
+    source = write(tmp_path, HIGH_AND_MEDIUM)
+
+    assert main(["--check", str(source), "--fail-on", "critical"]) == 0
+    out = capsys.readouterr().out
+    assert "dangerous-eval" in out and "weak-crypto" in out and "2 findings\n" in out
+
+    assert main(["--check", str(source), "--fail-on", "high"]) == 1
+    assert main(["--check", str(source), "--fail-on", "medium"]) == 1
+    assert main(["--check", str(source)]) == 1
+
+
+def test_a_clean_file_passes_whatever_the_threshold(tmp_path: Path) -> None:
+    source = write(tmp_path, "def ok():\n    return 1\n")
+
+    assert main(["--check", str(source), "--fail-on", "info"]) == 0
+
+
+def test_suppressed_and_baselined_findings_never_count(tmp_path: Path, capsys) -> None:  # type: ignore[no-untyped-def]
+    source = write(tmp_path, "def run(code):\n    eval(code)  # coretrace: ignore\n")
+    assert main(["--check", str(source), "--fail-on", "low"]) == 0
+
+    source = write(tmp_path, HIGH_AND_MEDIUM)
+    baseline = tmp_path / "baseline.json"
+    main(["--check", str(source), "--baseline", str(baseline)])
+    assert main(["--check", str(source), "--baseline", str(baseline), "--fail-on", "low"]) == 0
+
+
+def test_fail_on_requires_a_check_and_a_known_severity(tmp_path: Path, capsys) -> None:  # type: ignore[no-untyped-def]
+    source = write(tmp_path, "def ok():\n    return 1\n")
+
+    assert main(["--emit-ir", "--fail-on", "high", str(source)]) == 2
+    assert "--fail-on" in capsys.readouterr().err
+    with pytest.raises(SystemExit):
+        main(["--check", str(source), "--fail-on", "severe"])