From 0cca629c21d227bde773e876a634b032efa7505d Mon Sep 17 00:00:00 2001 From: Hugo Date: Sat, 5 Sep 2026 13:33:48 +0200 Subject: [PATCH 1/2] test: define the confidence of credentials in tests and examples --- tests/test_credential_context.py | 88 ++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 tests/test_credential_context.py diff --git a/tests/test_credential_context.py b/tests/test_credential_context.py new file mode 100644 index 0000000..a215440 --- /dev/null +++ b/tests/test_credential_context.py @@ -0,0 +1,88 @@ +"""Acceptance tests for credentials in test code and example files (issue #71). + +On real projects most ``hardcoded-credential`` findings sit in test fixtures (37 of 39 in +healthchecks) or in ``.env.example`` files: a password there is a fixture or a template, +not a leak, but a real one can still slip in. Such findings are reported at low +confidence with the context in their metadata; provider-format secrets keep their +confidence wherever they are, since a real key in a test is still a key. + +Expected to remain red until ``coretrace_python.plugins.secrets.credential_context`` exists. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from coretrace_python import engine +from coretrace_python.findings import Confidence, Finding +from coretrace_python.plugins import secrets +from coretrace_python.source import SourceManager + +MISSING = None if hasattr(secrets, "credential_context") else "credential_context is missing" + + +@pytest.fixture(autouse=True) +def require_context() -> None: + if MISSING is not None: + pytest.fail(f"credential context is not implemented yet: {MISSING}") + + +REPO = Path(__file__).resolve().parent.parent +PLUGINS = REPO / "src" / "coretrace_python" / "bundled" + +PASSWORD = "password = 'hunter2-not-a-placeholder'\n" +AWS = "key = 'AKIAIOSFODNN7EXAMPLE'\n" + + +def check(name: str, text: str) -> tuple[Finding, ...]: + return engine.check(SourceManager().add_source(name, text), [PLUGINS]) + + +def test_paths_are_classified() -> None: + assert secrets.credential_context("app/tests/test_login.py") == "test" + assert secrets.credential_context("app/test_views.py") == "test" + assert secrets.credential_context("app/views_test.py") == "test" + assert secrets.credential_context("conftest.py") == "test" + assert secrets.credential_context("fixtures/users.json") == "test" + assert secrets.credential_context("docker/.env.example") == "example" + assert secrets.credential_context("config.yaml.sample") == "example" + assert secrets.credential_context("settings.template.toml") is None + assert secrets.credential_context("app/views.py") is None + assert secrets.credential_context("latest_results.py") is None + + +def test_credentials_in_test_code_are_low_confidence() -> None: + (finding,) = check("app/tests/test_login.py", PASSWORD) + + assert finding.rule_id == "hardcoded-credential" + assert finding.confidence is Confidence.LOW + assert finding.metadata["context"] == "test" + + +def test_credentials_in_production_code_keep_medium_confidence() -> None: + (finding,) = check("app/views.py", PASSWORD) + + assert finding.confidence is Confidence.MEDIUM + assert "context" not in finding.metadata + + +def test_provider_secrets_keep_their_confidence_in_tests() -> None: + (finding,) = check("app/tests/test_s3.py", AWS) + + assert finding.rule_id == "hardcoded-secret" + assert finding.confidence is Confidence.HIGH + + +def test_example_configuration_files_are_low_confidence(tmp_path: Path) -> None: + (tmp_path / "app.py").write_text("", encoding="utf-8") + (tmp_path / ".env.example").write_text("DB_PASSWORD=hunter2-not-a-placeholder\n", encoding="utf-8") + (tmp_path / "docker-compose.yml").write_text("services:\n db:\n environment:\n POSTGRES_PASSWORD: hunter2-not-a-placeholder\n", encoding="utf-8") + + findings = sorted(engine.analyze_project(tmp_path, [PLUGINS]).findings, key=lambda f: str(f.span.source_id)) + + assert [(Path(str(f.span.source_id)).name, f.confidence) for f in findings] == [ + (".env.example", Confidence.LOW), + ("docker-compose.yml", Confidence.MEDIUM), + ] From 7ab35ca240f10ed3ad650fb20bd6c984ea5a7ec5 Mon Sep 17 00:00:00 2001 From: Hugo Date: Sat, 5 Sep 2026 13:33:48 +0200 Subject: [PATCH 2/2] fix: report credentials in test code and template files at low confidence --- docs/usage.md | 2 +- src/coretrace_python/plugins/secrets.py | 27 +++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index c1b43a5..45367e6 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -129,7 +129,7 @@ sources and in configuration files. | Rule | Confidence | Trigger | |---|---|---| | `hardcoded-secret` | high | A provider-specific format: AWS, GitHub, Slack, Stripe, Google, private keys, JWTs, SendGrid, Twilio. | -| `hardcoded-credential` | medium | A credential-like name (`password`, `token`, `api_key`, `app.config['SECRET_KEY']`, …) bound to a real value. Placeholders are excluded. | +| `hardcoded-credential` | medium | A credential-like name (`password`, `token`, `api_key`, `app.config['SECRET_KEY']`, …) bound to a real value. Placeholders are excluded. Low confidence in test code, fixtures and template files such as `.env.example`, with the context in the metadata. | | `high-entropy-string` | low | An opaque high-entropy token. Hex digests (MD5, SHA-1, SHA-256, SHA-512 lengths, or a name such as `hash`, `checksum`, `commit`), subresource-integrity hashes (`sha512-…`) and character-set constants are not secrets unless a credential name says so. | ### Dependencies diff --git a/src/coretrace_python/plugins/secrets.py b/src/coretrace_python/plugins/secrets.py index 3cad3e1..f885f92 100644 --- a/src/coretrace_python/plugins/secrets.py +++ b/src/coretrace_python/plugins/secrets.py @@ -59,6 +59,23 @@ def looks_like_digest(value: str, name: str | None) -> bool: return name is not None and any(word in name.lower() for word in _DIGEST_WORDS) +_TEST_DIRECTORIES = frozenset({"test", "tests", "testing", "fixtures", "__tests__"}) +_TEST_FILE = re.compile(r"^(test_.*|.*_test|conftest)\.py$") +_EXAMPLE_SUFFIXES = (".example", ".sample", ".dist", ".tpl") + + +def credential_context(path: str) -> str | None: + """``"test"`` for test code and fixtures, ``"example"`` for template configuration + files (``.env.example``), ``None`` for anything else.""" + + parts = Path(path).parts + if any(part in _TEST_DIRECTORIES for part in parts[:-1]) or _TEST_FILE.match(parts[-1]): + return "test" + if parts[-1].endswith(_EXAMPLE_SUFFIXES): + return "example" + return None + + def is_alphabet(value: str, name: str | None) -> bool: """A constant that spells out a character set is high-entropy by construction: its name says so, or every character in it is distinct, which no random token is.""" @@ -321,14 +338,20 @@ def judge(self, value: str, name: str | None, span: SourceSpan, function: str | {"provider": pattern.provider, "name": name or "", "length": str(len(value))}, ) if name is not None and self.is_credential_name(name) and not is_placeholder(value): + # A password in a test fixture or a template file is rarely a leak; a name + # is a hint, so the finding stays, at low confidence. + context = credential_context(str(span.source_id)) + metadata = {"name": name, "length": str(len(value))} + if context is not None: + metadata["context"] = context return Finding( "hardcoded-credential", f"Hardcoded credential {where}: {redacted(value)}", Severity.HIGH, - Confidence.MEDIUM, + Confidence.LOW if context is not None else Confidence.MEDIUM, span, function, - {"name": name, "length": str(len(value))}, + metadata, ) entropy = self.entropy_of(value) if entropy is not None and not looks_like_digest(value, name) and not is_alphabet(value, name):