From 1f6a01f74f55b57472d2f883466431888957f5fc Mon Sep 17 00:00:00 2001 From: Hugo Date: Sat, 5 Sep 2026 04:14:00 +0200 Subject: [PATCH 1/2] test: define the severity threshold --- tests/test_fail_on.py | 77 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 tests/test_fail_on.py diff --git a/tests/test_fail_on.py b/tests/test_fail_on.py new file mode 100644 index 0000000..7528836 --- /dev/null +++ b/tests/test_fail_on.py @@ -0,0 +1,77 @@ +"""Acceptance tests for the severity threshold (issue #68). + +``--fail-on SEVERITY`` makes the exit status 1 only when a finding of that severity or +above was reported; every finding is still printed. Without it, any finding fails the +check as before. Suppressed and baselined findings never count. + +Expected to remain red until ``Severity`` is ordered and ``--fail-on`` exists. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from coretrace_python.cli import main +from coretrace_python.findings import Severity + +MISSING = None if hasattr(Severity, "rank") else "Severity has no rank" + + +@pytest.fixture(autouse=True) +def require_threshold() -> None: + if MISSING is not None: + pytest.fail(f"the severity threshold is not implemented yet: {MISSING}") + + +HIGH_AND_MEDIUM = "import hashlib\n\ndef run(code):\n eval(code)\n hashlib.md5(code)\n" + + +def write(tmp_path: Path, text: str) -> Path: + source = tmp_path / "app.py" + source.write_text(text, encoding="utf-8") + return source + + +def test_severities_are_ordered() -> None: + ranks = [s.rank for s in (Severity.INFO, Severity.LOW, Severity.MEDIUM, Severity.HIGH, Severity.CRITICAL)] + + assert ranks == sorted(ranks) and len(set(ranks)) == 5 + + +def test_findings_below_the_threshold_are_reported_but_do_not_fail(tmp_path: Path, capsys) -> None: # type: ignore[no-untyped-def] + source = write(tmp_path, HIGH_AND_MEDIUM) + + assert main(["--check", str(source), "--fail-on", "critical"]) == 0 + out = capsys.readouterr().out + assert "dangerous-eval" in out and "weak-crypto" in out and "2 findings\n" in out + + assert main(["--check", str(source), "--fail-on", "high"]) == 1 + assert main(["--check", str(source), "--fail-on", "medium"]) == 1 + assert main(["--check", str(source)]) == 1 + + +def test_a_clean_file_passes_whatever_the_threshold(tmp_path: Path) -> None: + source = write(tmp_path, "def ok():\n return 1\n") + + assert main(["--check", str(source), "--fail-on", "info"]) == 0 + + +def test_suppressed_and_baselined_findings_never_count(tmp_path: Path, capsys) -> None: # type: ignore[no-untyped-def] + source = write(tmp_path, "def run(code):\n eval(code) # coretrace: ignore\n") + assert main(["--check", str(source), "--fail-on", "low"]) == 0 + + source = write(tmp_path, HIGH_AND_MEDIUM) + baseline = tmp_path / "baseline.json" + main(["--check", str(source), "--baseline", str(baseline)]) + assert main(["--check", str(source), "--baseline", str(baseline), "--fail-on", "low"]) == 0 + + +def test_fail_on_requires_a_check_and_a_known_severity(tmp_path: Path, capsys) -> None: # type: ignore[no-untyped-def] + source = write(tmp_path, "def ok():\n return 1\n") + + assert main(["--emit-ir", "--fail-on", "high", str(source)]) == 2 + assert "--fail-on" in capsys.readouterr().err + with pytest.raises(SystemExit): + main(["--check", str(source), "--fail-on", "severe"]) From e75d8dfa4b9ef9a12232d929c991961a3054b42a Mon Sep 17 00:00:00 2001 From: Hugo Date: Sat, 5 Sep 2026 04:14:00 +0200 Subject: [PATCH 2/2] feat: fail the check only from a chosen severity --- docs/usage.md | 6 ++++-- src/coretrace_python/cli.py | 16 +++++++++++++++- src/coretrace_python/findings/model.py | 9 +++++++++ 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index b561dd6..38e0326 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -31,7 +31,8 @@ coverage: 2/2 files, 12/12 functions ``` The exit status is 0 when nothing was found, 1 when findings were reported and 2 on a -usage or analysis error, so the command can gate a pipeline as it is. +usage or analysis error, so the command can gate a pipeline as it is. `--fail-on high` +keeps reporting every finding but fails only from that severity up. ## Command line @@ -52,6 +53,7 @@ coretrace-python-analyzer [--check | --emit-ir [--ssa]] [options] [path] | `--advisories FILE` | Read a local advisory file in addition to `advisories.json` at the root. Repeatable. | | `--policy FILE` | Apply this dependency policy instead of `coretrace-policy.toml` at the root. | | `--import-advisories SRC OUT` | Convert an OSV dump into the local advisory file `OUT`, then exit. | +| `--fail-on SEVERITY` | Exit with status 1 only when a finding of this severity or above was reported: `info`, `low`, `medium`, `high` or `critical`. Default: any finding. | | `--baseline PATH` | The accepted findings: written on the first run, then only findings not recorded there fail the check. | | `--emit-ir` | Print the intermediate representation of `path` instead of checking it. | | `--ssa` | With `--emit-ir`, print the static single assignment form. | @@ -310,7 +312,7 @@ Actions: ```yaml - run: pip install coretrace-python-analyzer -- run: coretrace-python-analyzer --check src/ --format sarif > coretrace.sarif +- run: coretrace-python-analyzer --check src/ --fail-on high --format sarif > coretrace.sarif continue-on-error: true - uses: github/codeql-action/upload-sarif@v3 with: diff --git a/src/coretrace_python/cli.py b/src/coretrace_python/cli.py index bcddce6..d3aefa3 100644 --- a/src/coretrace_python/cli.py +++ b/src/coretrace_python/cli.py @@ -10,6 +10,7 @@ from coretrace_python.cache import ProjectCache from coretrace_python.cfg import CFGError from coretrace_python.dependency import dump_advisories, import_osv, read_osv, render_sbom +from coretrace_python.findings import Severity from coretrace_python.findings.baseline import Baseline, BaselineError from coretrace_python.frontend import HIRBuildError, ParseError, build_hir from coretrace_python.ir.lowering import LoweringError, lower_module @@ -130,6 +131,14 @@ def build_parser() -> argparse.ArgumentParser: help="with --check, the accepted findings: written on the first run, then only " "findings not recorded there fail the check", ) + parser.add_argument( + "--fail-on", + choices=[severity.value for severity in Severity], + default=None, + metavar="SEVERITY", + help="with --check, exit with status 1 only when a finding of this severity or " + "above was reported (info, low, medium, high, critical); default: any finding", + ) parser.add_argument( "--import-advisories", nargs=2, @@ -180,6 +189,9 @@ def main(argv: list[str] | None = None) -> int: print("error: --policy only applies to --check on a directory", file=sys.stderr) return EXIT_ERROR + if args.fail_on is not None and not args.check: + print("error: --fail-on only applies to --check", file=sys.stderr) + return EXIT_ERROR if args.baseline is not None and not args.check: print("error: --baseline only applies to --check", file=sys.stderr) return EXIT_ERROR @@ -229,7 +241,9 @@ def main(argv: list[str] | None = None) -> int: findings, baselined = (), findings report = engine.report(findings, coverage, root, suppressed, baselined) print(render(args.format or "text", report), end="") - return EXIT_FINDINGS if findings else EXIT_CLEAN + threshold = Severity(args.fail_on).rank if args.fail_on is not None else 0 + failing = any(finding.severity.rank >= threshold for finding in findings) + return EXIT_FINDINGS if failing else EXIT_CLEAN except _ANALYSIS_ERRORS as error: print(f"error: {error}", file=sys.stderr) return EXIT_ERROR diff --git a/src/coretrace_python/findings/model.py b/src/coretrace_python/findings/model.py index c3a7d27..2b2ca19 100644 --- a/src/coretrace_python/findings/model.py +++ b/src/coretrace_python/findings/model.py @@ -24,6 +24,15 @@ class Severity(Enum): HIGH = "high" CRITICAL = "critical" + @property + def rank(self) -> int: + """Position in the order of severities, ``INFO`` lowest, ``CRITICAL`` highest.""" + + return _SEVERITY_ORDER.index(self) + + +_SEVERITY_ORDER = (Severity.INFO, Severity.LOW, Severity.MEDIUM, Severity.HIGH, Severity.CRITICAL) + class Confidence(Enum): LOW = "low"