From a3324cdede13063740054f874b34413c03947504 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 4 Sep 2026 03:01:14 +0200 Subject: [PATCH 1/2] test: define secret detection and SBOM behavior --- tests/test_detectors.py | 1 + tests/test_secrets_sbom.py | 342 +++++++++++++++++++++++++++++++++++++ 2 files changed, 343 insertions(+) create mode 100644 tests/test_secrets_sbom.py diff --git a/tests/test_detectors.py b/tests/test_detectors.py index 3f7bc45..220bb91 100644 --- a/tests/test_detectors.py +++ b/tests/test_detectors.py @@ -167,6 +167,7 @@ def test_shipped_plugins_load_with_their_manifests() -> None: "django-models", "fastapi-models", "flask-models", + "hardcoded-secrets", "http-client-models", "path-traversal", "python-stdlib-models", diff --git a/tests/test_secrets_sbom.py b/tests/test_secrets_sbom.py new file mode 100644 index 0000000..7c641c6 --- /dev/null +++ b/tests/test_secrets_sbom.py @@ -0,0 +1,342 @@ +"""Acceptance tests for secret detection and SBOM generation (``docs/architecture.md`` +§25 plugins/secrets, §26; roadmap issue #37). + +Secrets are string literals of the PyHIR: a ``SecretDetector`` base walks every literal +with the name it is bound to (assignment target, keyword, dictionary key) and reports at +most one finding per literal, provider patterns first, then credential-like names, then +entropy alone. Messages and metadata never contain the secret itself. + +The SBOM is a CycloneDX document rendered from the dependency graph and the advisories +that affect it; ``--sbom PATH`` writes it during a directory check. + +Expected to remain red until ``coretrace_python.plugins.secrets``, +``coretrace_python.dependency.sbom``, the ``hardcoded-secrets`` plugin and ``--sbom`` exist. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from coretrace_python import engine +from coretrace_python.cli import main +from coretrace_python.dependency import Advisory, parse_dependencies +from coretrace_python.findings import Confidence, Finding, Severity +from coretrace_python.frontend import build_hir +from coretrace_python.plugins import run_plugins +from coretrace_python.semantic.symbols import SymbolId +from coretrace_python.source import SourceManager + +try: + from coretrace_python.dependency.sbom import render_sbom + from coretrace_python.plugins.secrets import ( + SecretDetector, + SecretPattern, + literals, + shannon_entropy, + ) +except ImportError as error: # pragma: no cover - red until secrets and SBOM land + MISSING = error +else: + MISSING = None + + +@pytest.fixture(autouse=True) +def require_secrets() -> None: + if MISSING is not None: + pytest.fail(f"secrets and SBOM are not implemented yet: {MISSING}") + + +REPO = Path(__file__).resolve().parent.parent +PLUGINS = REPO / "plugins" + +AWS_KEY = "AKIAIOSFODNN7EXAMPLE" +GITHUB_TOKEN = "ghp_" + "a1B2c3D4e5F6g7H8i9J0k1L2m3N4o5P6q7R8" +RANDOM_BASE64 = "Qm9vdHN0cmFwcGluZyBhIHJhbmRvbSBrZXkgMjA0OA==" +RANDOM_HEX = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08" + + +def check(text: str) -> tuple[Finding, ...]: + return engine.check(SourceManager().add_source("settings.py", text), [PLUGINS]) + + +def rules(findings: tuple[Finding, ...]) -> list[str]: + return [f.rule_id for f in findings] + + +# --------------------------------------------------------------------------- base + + +def test_shannon_entropy_measures_bits_per_character() -> None: + assert shannon_entropy("") == 0 + assert shannon_entropy("aaaa") == 0 + assert shannon_entropy("abcd") == 2 + assert shannon_entropy(RANDOM_HEX) > 3.5 + assert shannon_entropy(RANDOM_BASE64) > 4.5 + assert shannon_entropy("the quick brown fox jumps over the lazy dog") < 4.5 + + +def test_literals_carry_the_name_they_are_bound_to() -> None: + module = build_hir( + SourceManager().add_source( + "m.py", + "PASSWORD = 'a'\n" + "db.connect(host='h', password='b')\n" + "config = {'api_key': 'c', 'debug': True}\n" + "def f():\n token = 'd'\n return 'e'\n" + "class C:\n secret = 'f'\n" + "x: str = 'g'\n" + "y: int\n", + ) + ) + + found = [(value, name, span.start_line, function) for value, name, span, function in literals(module)] + + assert found == [ + ("a", "PASSWORD", 1, None), + ("h", "host", 2, None), + ("b", "password", 2, None), + ("c", "api_key", 3, None), + ("d", "token", 5, "f"), + ("e", None, 6, "f"), + ("f", "secret", 8, None), + ("g", "x", 9, None), + ] + + +def test_a_detector_reports_one_finding_per_literal_provider_first() -> None: + class Detector(SecretDetector): + name = "d" + patterns = (SecretPattern("acme", r"acme_[a-z0-9]{16}"),) + credential_names = ("secret",) + + module = build_hir( + SourceManager().add_source("m.py", "secret = 'acme_0123456789abcdef'\nother = 'acme_fedcba9876543210'\n") + ) + findings = run_plugins(engine.build_manager(module), [Detector()]) + + assert [(f.rule_id, f.metadata["provider"], f.span.start_line) for f in findings] == [ + ("hardcoded-secret", "acme", 1), + ("hardcoded-secret", "acme", 2), + ] + + +# --------------------------------------------------------------------------- shipped plugin + + +def test_shipped_secret_plugin_loads() -> None: + from coretrace_python.plugins import discover_plugins + + module = build_hir(SourceManager().add_source("empty.py", "")) + loaded = {p.manifest.name: p.manifest for p in discover_plugins(PLUGINS, engine.build_manager(module))} + + assert loaded["hardcoded-secrets"].provides == ("secret.provider-patterns", "secret.credentials", "secret.entropy") + assert loaded["hardcoded-secrets"].requires == () + + +def test_provider_keys_are_high_confidence_and_never_echoed() -> None: + (finding,) = check(f"AWS_ACCESS_KEY_ID = '{AWS_KEY}'\n") + + assert finding.rule_id == "hardcoded-secret" + assert finding.severity is Severity.HIGH + assert finding.confidence is Confidence.HIGH + assert finding.metadata["provider"] == "aws" + assert finding.metadata["name"] == "AWS_ACCESS_KEY_ID" + assert finding.span.start_line == 1 + assert AWS_KEY not in finding.message + assert all(AWS_KEY not in value for value in finding.metadata.values()) + assert "AKIA" in finding.message and "20 characters" in finding.message + + +@pytest.mark.parametrize( + ("provider", "literal"), + [ + ("aws", AWS_KEY), + ("github", GITHUB_TOKEN), + ("github", "github_pat_" + "A" * 22 + "_" + "b" * 59), + ("slack", "xoxb-" + "1234567890-1234567890123-" + "AbCdEfGhIjKlMnOpQrStUvWx"), + ("stripe", "sk_live_" + "4eC39HqLyjWDarjtT1zdp7dc"), + ("google", "AIza" + "SyA1b2C3d4E5f6G7h8I9j0K1l2M3n4O5p6Q"), + ("private-key", "-----BEGIN RSA PRIVATE KEY-----\\nMIIE\\n-----END RSA PRIVATE KEY-----"), + ("jwt", "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0NTY3ODkwIn0.dozjgNryP4J3jVmNHl0w5N_XgL0n3I9PlFUP0THsR8U"), + ("sendgrid", "SG." + "a" * 22 + "." + "b" * 43), + ("twilio", "SK" + "0123456789abcdef" * 2), + ], +) +def test_provider_patterns(provider: str, literal: str) -> None: + (finding,) = check(f"value = '{literal}'\n") + assert (finding.rule_id, finding.metadata["provider"]) == ("hardcoded-secret", provider) + + +def test_credential_names_with_a_real_value_are_medium_confidence() -> None: + findings = check( + "password = 'hunter2'\n" + "db.connect(host='localhost', passwd='s3cr3t!')\n" + "settings = {'api_key': 'k-1234', 'timeout': '30'}\n" + "SECRET_KEY = 'django-insecure-abc'\n" + ) + + assert rules(findings) == ["hardcoded-credential"] * 4 + assert [f.confidence for f in findings] == [Confidence.MEDIUM] * 4 + assert [f.metadata["name"] for f in findings] == ["password", "passwd", "api_key", "SECRET_KEY"] + assert "hunter2" not in findings[0].message + + +def test_placeholders_and_indirections_are_not_credentials() -> None: + assert check( + "password = ''\n" + "token = 'changeme'\n" + "api_key = ''\n" + "secret = '${DB_SECRET}'\n" + "passwd = os.environ['PASSWD']\n" + "password = input()\n" + "PASSWORD_FIELD = 'password'\n" + "key = 'xxxxxxxx'\n" + "secret_name = 'db-secret'\n" + ) == () + + +def test_high_entropy_tokens_without_context_are_low_confidence() -> None: + (finding,) = check(f"blob = '{RANDOM_BASE64}'\n") + + assert finding.rule_id == "high-entropy-string" + assert finding.confidence is Confidence.LOW + assert finding.severity is Severity.MEDIUM + assert finding.metadata["entropy"].startswith("4.") + assert RANDOM_BASE64 not in finding.message + + +def test_hex_digests_count_as_high_entropy() -> None: + (finding,) = check(f"digest = '{RANDOM_HEX}'\n") + assert finding.rule_id == "high-entropy-string" + + +def test_prose_urls_and_short_strings_are_not_secrets() -> None: + assert check( + "MESSAGE = 'The quick brown fox jumps over the lazy dog and keeps running'\n" + "URL = 'https://example.com/api/v1/resources/0123456789abcdef'\n" + "PATH = '/usr/local/lib/python3.12/site-packages/x'\n" + "SHORT = 'abc123'\n" + "ident = 'f47ac10b-58cc-4372-a567-0e02b2c3d479'\n" + ) == () + + +def test_secrets_are_found_inside_functions_and_classes() -> None: + findings = check( + f"def connect():\n return client(token='{GITHUB_TOKEN}')\n\n" + "class Settings:\n password = 'hunter2'\n" + ) + + assert [(f.rule_id, f.span.start_line, f.function) for f in findings] == [ + ("hardcoded-secret", 2, "connect"), + ("hardcoded-credential", 5, None), + ] + + +def test_secret_findings_render_with_a_redacted_message(capsys) -> None: # type: ignore[no-untyped-def] + path = Path(__file__).parent / "_secret_sample.py" + path.write_text(f"token = '{GITHUB_TOKEN}'\n", encoding="utf-8") + try: + code = main(["--check", str(path), "--plugins", str(PLUGINS), "--format", "json"]) + output = capsys.readouterr().out + finally: + path.unlink() + + assert code == 1 + assert "hardcoded-secret" in output + assert GITHUB_TOKEN not in output + + +# --------------------------------------------------------------------------- SBOM + + +ADVISORY = Advisory("CVE-2020-1747", "pyyaml", "<5.4", "unsafe load", Severity.CRITICAL, (SymbolId("python.yaml.load"),)) + + +def graph_for(text: str, name: str = "requirements.txt"): # type: ignore[no-untyped-def] + return parse_dependencies(SourceManager().add_source(name, text)) + + +def test_sbom_is_cyclonedx_with_one_component_per_requirement() -> None: + document = json.loads(render_sbom(graph_for("pyyaml==5.3.1\nrequests>=2.20\nFlask\n"), (), "coretrace", "0.1.0")) + + assert document["bomFormat"] == "CycloneDX" + assert document["specVersion"] == "1.5" + assert document["metadata"]["tools"]["components"] == [{"type": "application", "name": "coretrace", "version": "0.1.0"}] + assert [c["name"] for c in document["components"]] == ["flask", "pyyaml", "requests"] + pyyaml = document["components"][1] + assert pyyaml == { + "type": "library", + "bom-ref": "pkg:pypi/pyyaml@5.3.1", + "name": "pyyaml", + "version": "5.3.1", + "purl": "pkg:pypi/pyyaml@5.3.1", + "properties": [{"name": "coretrace:specifier", "value": "==5.3.1"}], + } + assert "version" not in document["components"][2] + assert document["components"][2]["purl"] == "pkg:pypi/requests" + assert document["vulnerabilities"] == [] + + +def test_sbom_lists_advisories_affecting_the_requirements() -> None: + document = json.loads(render_sbom(graph_for("pyyaml==5.3.1\nrequests==2.31.0\n"), (ADVISORY,), "coretrace", "0.1.0")) + + assert document["vulnerabilities"] == [ + { + "id": "CVE-2020-1747", + "description": "unsafe load", + "ratings": [{"severity": "critical"}], + "affects": [{"ref": "pkg:pypi/pyyaml@5.3.1"}], + } + ] + + +def test_sbom_marks_optional_requirements_and_is_stable() -> None: + text = '[project]\nname = "x"\ndependencies = ["a==1.0"]\n[project.optional-dependencies]\ndev = ["b>=2"]\n' + first = render_sbom(graph_for(text, "pyproject.toml"), (), "coretrace", "0.1.0") + second = render_sbom(graph_for(text, "pyproject.toml"), (), "coretrace", "0.1.0") + + assert first == second + (a, b) = json.loads(first)["components"] + assert a["properties"] == [{"name": "coretrace:specifier", "value": "==1.0"}] + assert b["properties"] == [ + {"name": "coretrace:specifier", "value": ">=2"}, + {"name": "coretrace:optional", "value": "true"}, + ] + + +def test_project_analysis_exposes_the_advisories_it_used(tmp_path: Path) -> None: + (tmp_path / "requirements.txt").write_text("pyyaml==5.3.1\n", encoding="utf-8") + (tmp_path / "app.py").write_text("x = 1\n", encoding="utf-8") + + analysis = engine.analyze_project(tmp_path, [PLUGINS]) + + assert any(a.id == "CVE-2020-1747" for a in analysis.advisories) + assert engine.analyze_project(tmp_path).advisories == () + + +def test_check_writes_the_sbom_next_to_the_report(tmp_path: Path, capsys) -> None: # type: ignore[no-untyped-def] + root = tmp_path / "src" + root.mkdir() + (root / "requirements.txt").write_text("pyyaml==5.3.1\n", encoding="utf-8") + (root / "app.py").write_text("import yaml\n\ndef load(t):\n return yaml.load(t)\n", encoding="utf-8") + sbom = tmp_path / "bom.json" + + code = main(["--check", str(root), "--plugins", str(PLUGINS), "--sbom", str(sbom)]) + output = capsys.readouterr().out + + assert code == 1 + assert "vulnerable-dependency" in output + document = json.loads(sbom.read_text(encoding="utf-8")) + assert [c["purl"] for c in document["components"]] == ["pkg:pypi/pyyaml@5.3.1"] + assert [v["id"] for v in document["vulnerabilities"]] == ["CVE-2020-1747"] + + +def test_sbom_option_requires_a_directory_check(tmp_path: Path, capsys) -> None: # type: ignore[no-untyped-def] + source = tmp_path / "x.py" + source.write_text("", encoding="utf-8") + + assert main(["--check", "--sbom", str(tmp_path / "bom.json"), str(source)]) == 2 + assert "--sbom" in capsys.readouterr().err From 1a792e3a3db60ae1bad9df9547687de8fb9ce2f6 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 4 Sep 2026 03:01:14 +0200 Subject: [PATCH 2/2] feat: add secret detection base, hardcoded-secrets plugin and CycloneDX SBOM --- README.md | 11 ++ .../hardcoded_secrets/hardcoded_secrets.py | 36 ++++ plugins/secrets/hardcoded_secrets/plugin.toml | 9 + src/coretrace_python/cli.py | 23 ++- src/coretrace_python/dependency/__init__.py | 2 + src/coretrace_python/dependency/sbom.py | 65 +++++++ src/coretrace_python/engine.py | 5 +- src/coretrace_python/frontend/ast_adapter.py | 5 + src/coretrace_python/plugins/__init__.py | 10 + src/coretrace_python/plugins/secrets.py | 176 ++++++++++++++++++ 10 files changed, 338 insertions(+), 4 deletions(-) create mode 100644 plugins/secrets/hardcoded_secrets/hardcoded_secrets.py create mode 100644 plugins/secrets/hardcoded_secrets/plugin.toml create mode 100644 src/coretrace_python/dependency/sbom.py create mode 100644 src/coretrace_python/plugins/secrets.py diff --git a/README.md b/README.md index c820f8c..b008581 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,17 @@ attacker-controlled data reaches such a call, the engine correlates the four fac one critical `exploitable-vulnerability` finding, judged like any other flow. The shipped advisory database is a small offline sample; a live OSV feed is future work. +`--sbom PATH` writes a CycloneDX 1.5 bill of materials of the dependency graph, one +component per requirement with its package URL, and the advisories affecting them as +vulnerabilities. The shipped `secrets/` plugin scans every string literal of the Python +sources for provider-specific secret formats (AWS, GitHub, Slack, Stripe, Google, private +keys, JWTs, SendGrid, Twilio), for credential-like names bound to a real value +(`password`, `token`, `api_key` and the like, placeholders excluded) and for opaque +high-entropy tokens; one finding per literal, at high, medium and low confidence +respectively, with a redacted preview and never the secret itself. Other plugins add +providers by subclassing `SecretDetector` with their own patterns. Configuration files +are not scanned. + `--cache DIR` keeps the results of a directory check on disk, one JSON entry per module. The entry is keyed by the module's source, the engine and plugin versions, the plugin code, the security models, the advisories, the dependency graph and the keys of the diff --git a/plugins/secrets/hardcoded_secrets/hardcoded_secrets.py b/plugins/secrets/hardcoded_secrets/hardcoded_secrets.py new file mode 100644 index 0000000..4d9e064 --- /dev/null +++ b/plugins/secrets/hardcoded_secrets/hardcoded_secrets.py @@ -0,0 +1,36 @@ +"""Hardcoded secrets: provider formats, credential-like names, high-entropy tokens.""" + +from __future__ import annotations + +from typing import ClassVar + +from coretrace_python.plugins.secrets import SecretDetector, SecretPattern + + +class HardcodedSecrets(SecretDetector): + name: ClassVar[str] = "hardcoded-secrets" + patterns: ClassVar[tuple[SecretPattern, ...]] = ( + SecretPattern("aws", r"\b(AKIA|ASIA)[0-9A-Z]{16}\b"), + SecretPattern("github", r"\bgh[pousr]_[A-Za-z0-9]{36,}\b"), + SecretPattern("github", r"\bgithub_pat_[A-Za-z0-9]{22}_[A-Za-z0-9]{59}\b"), + SecretPattern("slack", r"\bxox[abpr]-[0-9]{10,}-[0-9A-Za-z-]{10,}"), + SecretPattern("stripe", r"\b[sr]k_(live|test)_[0-9A-Za-z]{24,}\b"), + SecretPattern("google", r"\bAIza[0-9A-Za-z_-]{35}\b"), + SecretPattern("private-key", r"-----BEGIN [A-Z ]*PRIVATE KEY-----"), + SecretPattern("jwt", r"\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b"), + SecretPattern("sendgrid", r"\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b"), + SecretPattern("twilio", r"\bSK[0-9a-fA-F]{32}\b"), + ) + credential_names: ClassVar[tuple[str, ...]] = ( + "password", + "passwd", + "pwd", + "secret", + "token", + "api_key", + "apikey", + "api-key", + "private_key", + "access_key", + "credential", + ) diff --git a/plugins/secrets/hardcoded_secrets/plugin.toml b/plugins/secrets/hardcoded_secrets/plugin.toml new file mode 100644 index 0000000..397866f --- /dev/null +++ b/plugins/secrets/hardcoded_secrets/plugin.toml @@ -0,0 +1,9 @@ +name = "hardcoded-secrets" +version = "1.0.0" +plugin_api = ">=1,<2" +requires = [] +provides = ["secret.provider-patterns", "secret.credentials", "secret.entropy"] + +[entrypoint] +module = "hardcoded_secrets" +class = "HardcodedSecrets" diff --git a/src/coretrace_python/cli.py b/src/coretrace_python/cli.py index cc1466e..7d5a2f7 100644 --- a/src/coretrace_python/cli.py +++ b/src/coretrace_python/cli.py @@ -4,10 +4,11 @@ import sys from pathlib import Path -from coretrace_python import engine +from coretrace_python import __version__, engine from coretrace_python.analysis import AnalysisError from coretrace_python.cache import ProjectCache from coretrace_python.cfg import CFGError +from coretrace_python.dependency import render_sbom from coretrace_python.frontend import HIRBuildError, ParseError, build_hir from coretrace_python.ir.lowering import LoweringError, lower_module from coretrace_python.ir.printer import format_module @@ -86,6 +87,13 @@ def build_parser() -> argparse.ArgumentParser: metavar="N", help="with --check on a directory, analyse independent modules in N processes", ) + parser.add_argument( + "--sbom", + type=Path, + default=None, + metavar="PATH", + help="with --check on a directory, write a CycloneDX bill of materials to PATH", + ) return parser @@ -106,6 +114,9 @@ def main(argv: list[str] | None = None) -> int: if args.jobs is not None and args.jobs < 1: print("error: --jobs must be at least 1", file=sys.stderr) return EXIT_ERROR + if args.sbom is not None and not (args.check and args.path.is_dir()): + print("error: --sbom only applies to --check on a directory", file=sys.stderr) + return EXIT_ERROR if args.path.is_dir() and args.emit_ir: print(f"error: {args.path} is a directory; --emit-ir needs a file", file=sys.stderr) @@ -118,9 +129,15 @@ def main(argv: list[str] | None = None) -> int: if args.check: if args.path.is_dir(): cache = None if args.cache is None else ProjectCache(args.cache) - findings = engine.analyze_project( + analysis = engine.analyze_project( args.path, args.plugins, cache=cache, jobs=args.jobs or 1 - ).findings + ) + findings = analysis.findings + if args.sbom is not None: + args.sbom.write_text( + render_sbom(analysis.dependencies, analysis.advisories, engine.TOOL_NAME, __version__), + encoding="utf-8", + ) else: findings = engine.check(SourceManager().load_file(args.path), args.plugins) print(render(args.format or "text", engine.report(findings)), end="") diff --git a/src/coretrace_python/dependency/__init__.py b/src/coretrace_python/dependency/__init__.py index 28308b8..32b9252 100644 --- a/src/coretrace_python/dependency/__init__.py +++ b/src/coretrace_python/dependency/__init__.py @@ -10,6 +10,7 @@ normalize, parse_dependencies, ) +from coretrace_python.dependency.sbom import render_sbom __all__ = [ "DEPENDENCY_FILES", @@ -20,4 +21,5 @@ "Version", "normalize", "parse_dependencies", + "render_sbom", ] diff --git a/src/coretrace_python/dependency/sbom.py b/src/coretrace_python/dependency/sbom.py new file mode 100644 index 0000000..65b0900 --- /dev/null +++ b/src/coretrace_python/dependency/sbom.py @@ -0,0 +1,65 @@ +"""CycloneDX software bill of materials from the dependency graph (architecture §26).""" + +from __future__ import annotations + +import json +from collections.abc import Iterable + +from coretrace_python.dependency.graph import Advisory, DependencyGraph, Requirement + +SPEC_VERSION = "1.5" + + +def purl(requirement: Requirement) -> str: + base = f"pkg:pypi/{requirement.name}" + if requirement.pinned is None: + return base + return f"{base}@{'.'.join(str(part) for part in requirement.pinned.parts)}" + + +def _component(requirement: Requirement) -> dict[str, object]: + reference = purl(requirement) + component: dict[str, object] = {"type": "library", "bom-ref": reference, "name": requirement.name} + if requirement.pinned is not None: + component["version"] = ".".join(str(part) for part in requirement.pinned.parts) + component["purl"] = reference + properties: list[dict[str, str]] = [] + if requirement.specifier: + properties.append({"name": "coretrace:specifier", "value": requirement.specifier}) + if requirement.optional: + properties.append({"name": "coretrace:optional", "value": "true"}) + if properties: + component["properties"] = properties + return component + + +def render_sbom( + dependencies: DependencyGraph, advisories: Iterable[Advisory], tool_name: str, tool_version: str +) -> str: + """A CycloneDX JSON document: one component per requirement, and the advisories + affecting them as vulnerabilities. Deterministic for a given graph.""" + + requirements = dependencies.requirements + vulnerabilities: list[dict[str, object]] = [] + for advisory in sorted(advisories, key=lambda a: a.id): + affected = [purl(r) for r in requirements if advisory.affects(r)] + if affected: + vulnerabilities.append( + { + "id": advisory.id, + "description": advisory.summary, + "ratings": [{"severity": advisory.severity.value}], + "affects": [{"ref": reference} for reference in affected], + } + ) + document = { + "bomFormat": "CycloneDX", + "specVersion": SPEC_VERSION, + "version": 1, + "metadata": { + "tools": {"components": [{"type": "application", "name": tool_name, "version": tool_version}]} + }, + "components": [_component(requirement) for requirement in requirements], + "vulnerabilities": vulnerabilities, + } + return json.dumps(document, indent=2) + "\n" diff --git a/src/coretrace_python/engine.py b/src/coretrace_python/engine.py index 415a7fa..e94b4e3 100644 --- a/src/coretrace_python/engine.py +++ b/src/coretrace_python/engine.py @@ -161,6 +161,7 @@ class ProjectAnalysis: dependencies: DependencyGraph = field(default_factory=DependencyGraph) keys: Mapping[str, str] = field(default_factory=_no_keys) reused: tuple[str, ...] = () + advisories: tuple[Advisory, ...] = () def build_manager( @@ -331,7 +332,9 @@ def analyze_project( for plugin in all_plugins: if isinstance(plugin, ProjectPlugin): findings.extend(plugin.analyze_project(context)) - return ProjectAnalysis(graph, index, tuple(findings), dependencies, MappingProxyType(keys), reused) + return ProjectAnalysis( + graph, index, tuple(findings), dependencies, MappingProxyType(keys), reused, advisories + ) def _seed(results: Mapping[str, CachedModule], graph: ModuleGraph, component: frozenset[str]) -> SummaryIndex: diff --git a/src/coretrace_python/frontend/ast_adapter.py b/src/coretrace_python/frontend/ast_adapter.py index 8a5479a..9a35223 100644 --- a/src/coretrace_python/frontend/ast_adapter.py +++ b/src/coretrace_python/frontend/ast_adapter.py @@ -181,6 +181,11 @@ def statement(self, node: ast.stmt) -> nodes.Statement: if len(node.targets) != 1: self.fail(node, "chained assignment is not supported yet") return nodes.Assign(self.target(node.targets[0]), self.expression(node.value), span) + if isinstance(node, ast.AnnAssign): + # The annotation never affects behaviour; a bare declaration does nothing. + if node.value is None: + return nodes.Pass(span) + return nodes.Assign(self.target(node.target), self.expression(node.value), span) if isinstance(node, ast.AugAssign): operator = _BINARY_OPERATORS.get(type(node.op)) if operator is None: diff --git a/src/coretrace_python/plugins/__init__.py b/src/coretrace_python/plugins/__init__.py index d1c49e4..e62ac1c 100644 --- a/src/coretrace_python/plugins/__init__.py +++ b/src/coretrace_python/plugins/__init__.py @@ -24,6 +24,12 @@ load_manifest, ) from coretrace_python.plugins.registry import PluginRegistry +from coretrace_python.plugins.secrets import ( + SecretDetector, + SecretPattern, + literals, + shannon_entropy, +) __all__ = [ "PLUGIN_API_VERSION", @@ -38,11 +44,15 @@ "PluginRegistry", "ProjectContext", "ProjectPlugin", + "SecretDetector", + "SecretPattern", "SymbolCallDetector", "TaintDetector", "VersionRange", "discover_plugins", + "literals", "load_manifest", "load_plugin", "run_plugins", + "shannon_entropy", ] diff --git a/src/coretrace_python/plugins/secrets.py b/src/coretrace_python/plugins/secrets.py new file mode 100644 index 0000000..193036a --- /dev/null +++ b/src/coretrace_python/plugins/secrets.py @@ -0,0 +1,176 @@ +"""Secret detection base (architecture §25 plugins/secrets). + +Secrets are string literals of the PyHIR. ``literals`` walks every string literal with +the name it is bound to (assignment target, keyword argument, dictionary key) and the +enclosing function; ``SecretDetector`` reports at most one finding per literal: a +provider pattern first (``hardcoded-secret``), then a credential-like name with a real +value (``hardcoded-credential``), then a high-entropy token on its own +(``high-entropy-string``). Messages and metadata carry a redacted preview, never the +secret. Only Python sources are scanned; configuration files are not. +""" + +from __future__ import annotations + +import math +import re +from collections import Counter +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from typing import ClassVar + +from coretrace_python.findings import Confidence, Finding, Severity +from coretrace_python.hir import nodes +from coretrace_python.hir.visitors import Node, children +from coretrace_python.plugins.api import Plugin, PluginContext +from coretrace_python.source import SourceSpan + +Literal = tuple[str, str | None, SourceSpan, str | None] + +_HEX = re.compile(r"^[0-9a-fA-F]+$") +_TOKEN = re.compile(r"^[A-Za-z0-9+/=_-]+$") +_PLACEHOLDER = re.compile(r"^(<.*>|\$\{.*\}|\{\{.*\}\}|%.*%|\.{3,}|(.)\2*)$") +_PLACEHOLDER_WORDS = frozenset( + {"", "changeme", "change_me", "password", "passwd", "secret", "token", "example", "xxx", "todo", "none", "null"} +) +_NOT_CREDENTIAL_SUFFIXES = ("_name", "_field", "_file", "_path", "_url", "_id", "_env", "_var", "_header", "_param") + + +def shannon_entropy(text: str) -> float: + """Bits of information per character of ``text``.""" + + if not text: + return 0.0 + counts = Counter(text) + length = len(text) + return -sum(count / length * math.log2(count / length) for count in counts.values()) + + +def literals(module: nodes.Module) -> Iterator[Literal]: + """Every string literal of the module as ``(value, bound name, span, function)``.""" + + for statement in module.body: + yield from _walk(statement, None, None) + + +def _walk(node: Node, name: str | None, function: str | None) -> Iterator[Literal]: + if isinstance(node, nodes.Constant): + if isinstance(node.value, str): + yield node.value, name, node.span, function + return + if isinstance(node, nodes.Function): + for child in children(node): + yield from _walk(child, None, node.name) + return + if isinstance(node, nodes.Class): + for child in children(node): + yield from _walk(child, None, None) + return + if isinstance(node, nodes.Assign): + target = node.target.identifier if isinstance(node.target, nodes.Name) else None + yield from _walk(node.value, target, function) + return + if isinstance(node, nodes.Keyword): + yield from _walk(node.value, node.name, function) + return + if isinstance(node, nodes.Dict): + for key, value in node.items: + # A constant key names the value; it is not a value itself. + if isinstance(key, nodes.Constant): + bound = key.value if isinstance(key.value, str) else None + else: + bound = None + yield from _walk(key, None, function) + yield from _walk(value, bound, function) + return + for child in children(node): + yield from _walk(child, None, function) + + +@dataclass(frozen=True) +class SecretPattern: + """A provider-specific secret format.""" + + provider: str + regex: str + + def matches(self, text: str) -> bool: + return re.search(self.regex, text) is not None + + +def redacted(value: str) -> str: + preview = value[:4] if len(value) > 8 else value[:1] + return f"{preview}… ({len(value)} characters)" + + +def is_placeholder(value: str) -> bool: + lowered = value.strip().lower() + return len(lowered) < 4 or lowered in _PLACEHOLDER_WORDS or _PLACEHOLDER.match(lowered) is not None + + +class SecretDetector(Plugin): + """Report hardcoded secrets among the module's string literals.""" + + patterns: ClassVar[tuple[SecretPattern, ...]] = () + credential_names: ClassVar[tuple[str, ...]] = () + entropy_threshold: ClassVar[float] = 4.5 + hex_entropy_threshold: ClassVar[float] = 3.5 + minimum_length: ClassVar[int] = 32 + + def analyze(self, ctx: PluginContext) -> Sequence[Finding]: + findings: list[Finding] = [] + for value, name, span, function in literals(ctx.module): + finding = self.judge(value, name, span, function) + if finding is not None: + findings.append(finding) + return findings + + def judge(self, value: str, name: str | None, span: SourceSpan, function: str | None) -> Finding | None: + where = f"in {name}" if name is not None else "in a string literal" + for pattern in self.patterns: + if pattern.matches(value): + return Finding( + "hardcoded-secret", + f"Hardcoded {pattern.provider} secret {where}: {redacted(value)}", + Severity.HIGH, + Confidence.HIGH, + span, + function, + {"provider": pattern.provider, "name": name or "", "length": str(len(value))}, + ) + if name is not None and self.is_credential_name(name) and not is_placeholder(value): + return Finding( + "hardcoded-credential", + f"Hardcoded credential {where}: {redacted(value)}", + Severity.HIGH, + Confidence.MEDIUM, + span, + function, + {"name": name, "length": str(len(value))}, + ) + entropy = self.entropy_of(value) + if entropy is not None: + return Finding( + "high-entropy-string", + f"High-entropy string {where}: {redacted(value)}", + Severity.MEDIUM, + Confidence.LOW, + span, + function, + {"name": name or "", "length": str(len(value)), "entropy": f"{entropy:.2f}"}, + ) + return None + + def is_credential_name(self, name: str) -> bool: + lowered = name.lower() + if lowered.endswith(_NOT_CREDENTIAL_SUFFIXES): + return False + return any(word in lowered for word in self.credential_names) + + def entropy_of(self, value: str) -> float | None: + """The entropy of ``value`` when it looks like an opaque token, else ``None``.""" + + if len(value) < self.minimum_length or _TOKEN.match(value) is None: + return None + entropy = shannon_entropy(value) + threshold = self.hex_entropy_threshold if _HEX.match(value) else self.entropy_threshold + return entropy if entropy >= threshold else None