From a18ffb134cfb83c94d1b8621b5f29770711f6e88 Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 4 Sep 2026 18:47:55 +0200 Subject: [PATCH 1/3] test: define the non-regression suite on real repositories --- tests/test_regression.py | 122 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 122 insertions(+) create mode 100644 tests/test_regression.py diff --git a/tests/test_regression.py b/tests/test_regression.py new file mode 100644 index 0000000..1c2b088 --- /dev/null +++ b/tests/test_regression.py @@ -0,0 +1,122 @@ +"""Non-regression suite on real repositories. + +Every repository listed in ``tests/regression/repositories.toml`` is checked out at its +pinned commit, analysed with the bundled plugins and compared with the snapshot recorded +in ``tests/regression/expected/.json``: the findings (file, line, rule, function) +and the coverage. A change in either is a regression, or an intended change to record +with ``CORETRACE_REGRESSION_UPDATE=1``. Checkouts are kept under +``CORETRACE_REGRESSION_ROOT`` (default ``.regression/`` at the repository root). + +The repository tests carry the ``regression`` marker, deselected by default and run by +the dedicated CI job with ``pytest -m regression``. + +Expected to remain red until the manifest, the snapshots and the ``regression`` marker exist. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import tomllib +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from coretrace_python import engine + +REPO = Path(__file__).resolve().parent.parent +REGRESSION = REPO / "tests" / "regression" +MANIFEST = REGRESSION / "repositories.toml" +EXPECTED = REGRESSION / "expected" + + +@dataclass(frozen=True) +class Repository: + name: str + url: str | None = None + commit: str | None = None + path: str | None = None + + +def repositories() -> tuple[Repository, ...]: + if not MANIFEST.is_file(): + return () + manifest = tomllib.loads(MANIFEST.read_text(encoding="utf-8")) + return tuple(Repository(**entry) for entry in manifest["repository"]) + + +REPOSITORIES = repositories() + + +def checkout(repository: Repository) -> Path: + """The repository at its pinned commit, cloned on first use.""" + + if repository.path is not None: + return REGRESSION / repository.path + assert repository.url is not None and repository.commit is not None + root = Path(os.environ.get("CORETRACE_REGRESSION_ROOT", REPO / ".regression")) + target = root / repository.name + marker = target / ".coretrace-commit" + if marker.is_file() and marker.read_text(encoding="utf-8").strip() == repository.commit: + return target + shutil.rmtree(target, ignore_errors=True) + target.mkdir(parents=True) + for command in ( + ["git", "init", "-q"], + ["git", "fetch", "-q", "--depth", "1", repository.url, repository.commit], + ["git", "checkout", "-q", "FETCH_HEAD"], + ): + subprocess.run(command, cwd=target, check=True) + marker.write_text(repository.commit + "\n", encoding="utf-8") + return target + + +def snapshot(path: Path) -> dict[str, object]: + analysis = engine.analyze_project(path, [engine.BUNDLED_PLUGINS]) + findings = sorted( + ( + { + "file": Path(str(f.span.source_id)).resolve().relative_to(path.resolve()).as_posix(), + "line": f.span.start_line, + "rule": f.rule_id, + "function": f.function, + } + for f in analysis.findings + ), + key=lambda f: (f["file"], f["line"], f["rule"], f["function"] or ""), + ) + return { + "coverage": { + "files": analysis.coverage.files, + "files_analysed": analysis.coverage.files_analysed, + "functions": analysis.coverage.functions, + "functions_analysed": analysis.coverage.functions_analysed, + }, + "findings": findings, + } + + +def test_every_repository_has_a_snapshot_and_vice_versa() -> None: + assert MANIFEST.is_file(), f"{MANIFEST} is missing" + names = {repository.name for repository in REPOSITORIES} + assert names == {path.stem for path in EXPECTED.glob("*.json")} + assert len(names) >= 16 + assert all( + (r.path is None) != (r.url is None and r.commit is None) for r in REPOSITORIES + ) + assert all(len(r.commit) == 40 for r in REPOSITORIES if r.commit is not None) + + +@pytest.mark.regression +@pytest.mark.parametrize("repository", REPOSITORIES, ids=lambda r: r.name) +def test_repository_analysis_matches_its_snapshot(repository: Repository) -> None: + observed = snapshot(checkout(repository)) + expected_path = EXPECTED / f"{repository.name}.json" + if os.environ.get("CORETRACE_REGRESSION_UPDATE"): + expected_path.write_text(json.dumps(observed, indent=2) + "\n", encoding="utf-8") + expected = json.loads(expected_path.read_text(encoding="utf-8")) + assert observed["coverage"] == expected["coverage"] + assert observed["findings"] == expected["findings"] From 0ec8ce1937b46351dc2aba705ea09e199f301c7c Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 4 Sep 2026 18:47:56 +0200 Subject: [PATCH 2/3] feat: add the non-regression suite on pinned repositories to CI --- .github/workflows/ci.yml | 14 +++ .gitignore | 1 + README.md | 10 +++ pyproject.toml | 5 +- tests/regression/expected/Chatbot.json | 22 +++++ tests/regression/expected/DjangoGoat.json | 58 ++++++++++++ .../expected/Flask-Image-Editing-Website.json | 52 +++++++++++ tests/regression/expected/InsecureGram.json | 64 +++++++++++++ .../expected/MiniBookApiServer.json | 64 +++++++++++++ tests/regression/expected/NPL-IA.json | 16 ++++ tests/regression/expected/Pylinkit.json | 22 +++++ .../expected/Spam-Detection-Project.json | 16 ++++ tests/regression/expected/VideoSubtitles.json | 9 ++ .../expected/Vulnerable_fastapi.json | 46 ++++++++++ tests/regression/expected/my_gdb.json | 9 ++ .../expected/network-topologer.json | 9 ++ .../expected/tag-collector-server.json | 16 ++++ .../uploadbox-cloud-storage-solution.json | 52 +++++++++++ .../regression/expected/vulnerable-flask.json | 64 +++++++++++++ tests/regression/expected/yt-fts.json | 28 ++++++ .../fixtures/vulnerable-flask/README.md | 40 +++++++++ .../fixtures/vulnerable-flask/app.py | 90 +++++++++++++++++++ .../vulnerable-flask/requirements.txt | 2 + .../vulnerable-flask/vulnerable_services.py | 33 +++++++ tests/regression/repositories.toml | 83 +++++++++++++++++ 25 files changed, 824 insertions(+), 1 deletion(-) create mode 100644 tests/regression/expected/Chatbot.json create mode 100644 tests/regression/expected/DjangoGoat.json create mode 100644 tests/regression/expected/Flask-Image-Editing-Website.json create mode 100644 tests/regression/expected/InsecureGram.json create mode 100644 tests/regression/expected/MiniBookApiServer.json create mode 100644 tests/regression/expected/NPL-IA.json create mode 100644 tests/regression/expected/Pylinkit.json create mode 100644 tests/regression/expected/Spam-Detection-Project.json create mode 100644 tests/regression/expected/VideoSubtitles.json create mode 100644 tests/regression/expected/Vulnerable_fastapi.json create mode 100644 tests/regression/expected/my_gdb.json create mode 100644 tests/regression/expected/network-topologer.json create mode 100644 tests/regression/expected/tag-collector-server.json create mode 100644 tests/regression/expected/uploadbox-cloud-storage-solution.json create mode 100644 tests/regression/expected/vulnerable-flask.json create mode 100644 tests/regression/expected/yt-fts.json create mode 100644 tests/regression/fixtures/vulnerable-flask/README.md create mode 100644 tests/regression/fixtures/vulnerable-flask/app.py create mode 100644 tests/regression/fixtures/vulnerable-flask/requirements.txt create mode 100644 tests/regression/fixtures/vulnerable-flask/vulnerable_services.py create mode 100644 tests/regression/repositories.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da437e0..3bd97d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,20 @@ jobs: - run: python -m ruff check . - run: python -m pytest + regression: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: actions/cache@v4 + with: + path: .regression + key: regression-${{ hashFiles('tests/regression/repositories.toml') }} + - run: python -m pip install -e ".[dev]" + - run: python -m pytest -m regression + package: runs-on: ubuntu-latest steps: diff --git a/.gitignore b/.gitignore index a4dfd36..b19b131 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ build/ dist/ .DS_Store +.regression/ diff --git a/README.md b/README.md index 6e864c0..fa0ccd4 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,16 @@ python -m pytest python -m ruff check . ``` +The non-regression suite analyses the public repositories pinned in +[`tests/regression/repositories.toml`](tests/regression/repositories.toml) and compares +findings and coverage with the snapshots in `tests/regression/expected/`. It clones on +first use, needs the network and runs in its own CI job: + +```bash +python -m pytest -m regression +CORETRACE_REGRESSION_UPDATE=1 python -m pytest -m regression # record an intended change +``` + ## Install ```bash diff --git a/pyproject.toml b/pyproject.toml index 74eab44..d8a4185 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,7 +46,10 @@ coretrace-python-analyzer = "coretrace_python.cli:main" packages = ["src/coretrace_python"] [tool.pytest.ini_options] -addopts = "-q" +addopts = "-q -m 'not regression'" +markers = [ + "regression: analyses the pinned public repositories of tests/regression (network, slow)", +] testpaths = ["tests"] [tool.ruff] diff --git a/tests/regression/expected/Chatbot.json b/tests/regression/expected/Chatbot.json new file mode 100644 index 0000000..7f13a3c --- /dev/null +++ b/tests/regression/expected/Chatbot.json @@ -0,0 +1,22 @@ +{ + "coverage": { + "files": 16, + "files_analysed": 16, + "functions": 176, + "functions_analysed": 176 + }, + "findings": [ + { + "file": "chatbot/__init__.py", + "line": 224, + "rule": "missing-timeout", + "function": "train" + }, + { + "file": "setup.py", + "line": 48, + "rule": "command-injection", + "function": "install_ollama" + } + ] +} diff --git a/tests/regression/expected/DjangoGoat.json b/tests/regression/expected/DjangoGoat.json new file mode 100644 index 0000000..c6e1460 --- /dev/null +++ b/tests/regression/expected/DjangoGoat.json @@ -0,0 +1,58 @@ +{ + "coverage": { + "files": 29, + "files_analysed": 29, + "functions": 39, + "functions_analysed": 39 + }, + "findings": [ + { + "file": "authentication/views.py", + "line": 33, + "rule": "open-redirect", + "function": "sign_up" + }, + { + "file": "authentication/views.py", + "line": 57, + "rule": "sql-injection", + "function": "log_in" + }, + { + "file": "authentication/views.py", + "line": 82, + "rule": "open-redirect", + "function": "profile_update" + }, + { + "file": "djangogoat/settings.py", + "line": 24, + "rule": "hardcoded-credential", + "function": null + }, + { + "file": "features/environment.py", + "line": 196, + "rule": "hardcoded-credential", + "function": "after_all" + }, + { + "file": "notes/views.py", + "line": 86, + "rule": "open-redirect", + "function": "conversation" + }, + { + "file": "requirements_app.txt", + "line": 2, + "rule": "vulnerable-dependency", + "function": null + }, + { + "file": "requirements_app.txt", + "line": 3, + "rule": "vulnerable-dependency", + "function": null + } + ] +} diff --git a/tests/regression/expected/Flask-Image-Editing-Website.json b/tests/regression/expected/Flask-Image-Editing-Website.json new file mode 100644 index 0000000..71ce7eb --- /dev/null +++ b/tests/regression/expected/Flask-Image-Editing-Website.json @@ -0,0 +1,52 @@ +{ + "coverage": { + "files": 1, + "files_analysed": 1, + "functions": 4, + "functions_analysed": 4 + }, + "findings": [ + { + "file": "main.py", + "line": 15, + "rule": "hardcoded-credential", + "function": null + }, + { + "file": "main.py", + "line": 29, + "rule": "reachable-vulnerability", + "function": "processImage" + }, + { + "file": "main.py", + "line": 79, + "rule": "exploitable-vulnerability", + "function": "edit" + }, + { + "file": "main.py", + "line": 91, + "rule": "debug-enabled", + "function": null + }, + { + "file": "requirements.txt", + "line": 9, + "rule": "vulnerable-dependency", + "function": null + }, + { + "file": "requirements.txt", + "line": 10, + "rule": "vulnerable-dependency", + "function": null + }, + { + "file": "requirements.txt", + "line": 10, + "rule": "vulnerable-dependency", + "function": null + } + ] +} diff --git a/tests/regression/expected/InsecureGram.json b/tests/regression/expected/InsecureGram.json new file mode 100644 index 0000000..987f25b --- /dev/null +++ b/tests/regression/expected/InsecureGram.json @@ -0,0 +1,64 @@ +{ + "coverage": { + "files": 15, + "files_analysed": 15, + "functions": 15, + "functions_analysed": 15 + }, + "findings": [ + { + "file": "app/admin.py", + "line": 45, + "rule": "insecure-deserialization", + "function": "insecure_deserialize" + }, + { + "file": "app/auth.py", + "line": 40, + "rule": "hardcoded-credential", + "function": "login" + }, + { + "file": "app/command.py", + "line": 12, + "rule": "command-injection", + "function": "execute" + }, + { + "file": "app/config.py", + "line": 4, + "rule": "hardcoded-credential", + "function": null + }, + { + "file": "app/config.py", + "line": 6, + "rule": "hardcoded-credential", + "function": null + }, + { + "file": "app/redirect.py", + "line": 11, + "rule": "open-redirect", + "function": "redirect" + }, + { + "file": "app/ssrf.py", + "line": 12, + "rule": "ssrf", + "function": "proxy" + }, + { + "file": "app/upload.py", + "line": 36, + "rule": "path-traversal", + "function": "upload" + }, + { + "file": "app/users.py", + "line": 36, + "rule": "sql-injection", + "function": "search_users" + } + ] +} diff --git a/tests/regression/expected/MiniBookApiServer.json b/tests/regression/expected/MiniBookApiServer.json new file mode 100644 index 0000000..28b8033 --- /dev/null +++ b/tests/regression/expected/MiniBookApiServer.json @@ -0,0 +1,64 @@ +{ + "coverage": { + "files": 6, + "files_analysed": 6, + "functions": 40, + "functions_analysed": 40 + }, + "findings": [ + { + "file": "app.py", + "line": 69, + "rule": "path-traversal", + "function": "get_img" + }, + { + "file": "app.py", + "line": 100, + "rule": "exploitable-vulnerability", + "function": "new_book" + }, + { + "file": "config-example.json", + "line": 3, + "rule": "hardcoded-credential", + "function": null + }, + { + "file": "config-example.json", + "line": 4, + "rule": "hardcoded-credential", + "function": null + }, + { + "file": "docs/insomnia/insomnia_config_file.yaml", + "line": 71, + "rule": "high-entropy-string", + "function": null + }, + { + "file": "docs/insomnia/insomnia_config_file.yaml", + "line": 231, + "rule": "hardcoded-credential", + "function": null + }, + { + "file": "docs/insomnia/insomnia_config_file.yaml", + "line": 232, + "rule": "hardcoded-credential", + "function": null + }, + { + "file": "requirements.txt", + "line": 10, + "rule": "vulnerable-dependency", + "function": null + }, + { + "file": "res.py", + "line": 29, + "rule": "reachable-vulnerability", + "function": "store_pic_from_base64" + } + ] +} diff --git a/tests/regression/expected/NPL-IA.json b/tests/regression/expected/NPL-IA.json new file mode 100644 index 0000000..bf4243d --- /dev/null +++ b/tests/regression/expected/NPL-IA.json @@ -0,0 +1,16 @@ +{ + "coverage": { + "files": 1, + "files_analysed": 1, + "functions": 9, + "functions_analysed": 9 + }, + "findings": [ + { + "file": "model.json", + "line": 1, + "rule": "hardcoded-credential", + "function": null + } + ] +} diff --git a/tests/regression/expected/Pylinkit.json b/tests/regression/expected/Pylinkit.json new file mode 100644 index 0000000..5f5dde3 --- /dev/null +++ b/tests/regression/expected/Pylinkit.json @@ -0,0 +1,22 @@ +{ + "coverage": { + "files": 11, + "files_analysed": 11, + "functions": 213, + "functions_analysed": 213 + }, + "findings": [ + { + "file": "pylinkit/gui.py", + "line": 525, + "rule": "hardcoded-secret", + "function": "_sync_pass_predict" + }, + { + "file": "pylinkit/gui.py", + "line": 526, + "rule": "missing-timeout", + "function": "_sync_pass_predict" + } + ] +} diff --git a/tests/regression/expected/Spam-Detection-Project.json b/tests/regression/expected/Spam-Detection-Project.json new file mode 100644 index 0000000..d459c9a --- /dev/null +++ b/tests/regression/expected/Spam-Detection-Project.json @@ -0,0 +1,16 @@ +{ + "coverage": { + "files": 1, + "files_analysed": 1, + "functions": 2, + "functions_analysed": 2 + }, + "findings": [ + { + "file": "app.py", + "line": 50, + "rule": "debug-enabled", + "function": null + } + ] +} diff --git a/tests/regression/expected/VideoSubtitles.json b/tests/regression/expected/VideoSubtitles.json new file mode 100644 index 0000000..9e7a974 --- /dev/null +++ b/tests/regression/expected/VideoSubtitles.json @@ -0,0 +1,9 @@ +{ + "coverage": { + "files": 3, + "files_analysed": 3, + "functions": 13, + "functions_analysed": 13 + }, + "findings": [] +} diff --git a/tests/regression/expected/Vulnerable_fastapi.json b/tests/regression/expected/Vulnerable_fastapi.json new file mode 100644 index 0000000..7119f4d --- /dev/null +++ b/tests/regression/expected/Vulnerable_fastapi.json @@ -0,0 +1,46 @@ +{ + "coverage": { + "files": 20, + "files_analysed": 20, + "functions": 27, + "functions_analysed": 27 + }, + "findings": [ + { + "file": "app/routers/auth.py", + "line": 37, + "rule": "sql-injection", + "function": "login" + }, + { + "file": "app/routers/files.py", + "line": 27, + "rule": "path-traversal", + "function": "download" + }, + { + "file": "exploit_scripts/csrf_change_email.py", + "line": 22, + "rule": "hardcoded-credential", + "function": null + }, + { + "file": "exploit_scripts/idor_orders.py", + "line": 12, + "rule": "hardcoded-credential", + "function": null + }, + { + "file": "exploit_scripts/sqli_login_bypass.py", + "line": 24, + "rule": "hardcoded-credential", + "function": null + }, + { + "file": "exploit_scripts/stored_xss_profile.py", + "line": 19, + "rule": "hardcoded-credential", + "function": null + } + ] +} diff --git a/tests/regression/expected/my_gdb.json b/tests/regression/expected/my_gdb.json new file mode 100644 index 0000000..99682c6 --- /dev/null +++ b/tests/regression/expected/my_gdb.json @@ -0,0 +1,9 @@ +{ + "coverage": { + "files": 1, + "files_analysed": 1, + "functions": 30, + "functions_analysed": 30 + }, + "findings": [] +} diff --git a/tests/regression/expected/network-topologer.json b/tests/regression/expected/network-topologer.json new file mode 100644 index 0000000..258f80b --- /dev/null +++ b/tests/regression/expected/network-topologer.json @@ -0,0 +1,9 @@ +{ + "coverage": { + "files": 6, + "files_analysed": 6, + "functions": 20, + "functions_analysed": 20 + }, + "findings": [] +} diff --git a/tests/regression/expected/tag-collector-server.json b/tests/regression/expected/tag-collector-server.json new file mode 100644 index 0000000..b68da80 --- /dev/null +++ b/tests/regression/expected/tag-collector-server.json @@ -0,0 +1,16 @@ +{ + "coverage": { + "files": 5, + "files_analysed": 5, + "functions": 68, + "functions_analysed": 68 + }, + "findings": [ + { + "file": "MQTT-client/influx_writer.py", + "line": 17, + "rule": "hardcoded-credential", + "function": "get_influx_client" + } + ] +} diff --git a/tests/regression/expected/uploadbox-cloud-storage-solution.json b/tests/regression/expected/uploadbox-cloud-storage-solution.json new file mode 100644 index 0000000..88ce378 --- /dev/null +++ b/tests/regression/expected/uploadbox-cloud-storage-solution.json @@ -0,0 +1,52 @@ +{ + "coverage": { + "files": 2, + "files_analysed": 2, + "functions": 17, + "functions_analysed": 17 + }, + "findings": [ + { + "file": "app.py", + "line": 10, + "rule": "hardcoded-credential", + "function": null + }, + { + "file": "app.py", + "line": 139, + "rule": "path-traversal", + "function": "upload_file" + }, + { + "file": "app.py", + "line": 150, + "rule": "path-traversal", + "function": "download_file" + }, + { + "file": "app.py", + "line": 161, + "rule": "path-traversal", + "function": "delete_file" + }, + { + "file": "db.py", + "line": 24, + "rule": "plaintext-credential-storage", + "function": "register_user" + }, + { + "file": "db.py", + "line": 31, + "rule": "plaintext-credential-storage", + "function": "login_user" + }, + { + "file": "requirements.txt", + "line": 3, + "rule": "vulnerable-dependency", + "function": null + } + ] +} diff --git a/tests/regression/expected/vulnerable-flask.json b/tests/regression/expected/vulnerable-flask.json new file mode 100644 index 0000000..d51bc93 --- /dev/null +++ b/tests/regression/expected/vulnerable-flask.json @@ -0,0 +1,64 @@ +{ + "coverage": { + "files": 2, + "files_analysed": 2, + "functions": 12, + "functions_analysed": 12 + }, + "findings": [ + { + "file": "app.py", + "line": 20, + "rule": "hardcoded-secret", + "function": null + }, + { + "file": "app.py", + "line": 21, + "rule": "hardcoded-secret", + "function": null + }, + { + "file": "app.py", + "line": 44, + "rule": "command-injection", + "function": "command_injection" + }, + { + "file": "app.py", + "line": 50, + "rule": "sql-injection", + "function": "sql_injection" + }, + { + "file": "app.py", + "line": 55, + "rule": "path-traversal", + "function": "path_traversal" + }, + { + "file": "app.py", + "line": 60, + "rule": "ssrf", + "function": "server_side_request_forgery" + }, + { + "file": "app.py", + "line": 66, + "rule": "xss", + "function": "cross_site_scripting" + }, + { + "file": "app.py", + "line": 71, + "rule": "dangerous-eval", + "function": "code_injection" + }, + { + "file": "app.py", + "line": 78, + "rule": "weak-crypto", + "function": "weak_cryptography" + } + ] +} diff --git a/tests/regression/expected/yt-fts.json b/tests/regression/expected/yt-fts.json new file mode 100644 index 0000000..d7408a2 --- /dev/null +++ b/tests/regression/expected/yt-fts.json @@ -0,0 +1,28 @@ +{ + "coverage": { + "files": 20, + "files_analysed": 19, + "functions": 141, + "functions_analysed": 141 + }, + "findings": [ + { + "file": "src/yt_fts/llm/get_embeddings.py", + "line": 1, + "rule": "syntax-error", + "function": null + }, + { + "file": "tests/test_export.py", + "line": 48, + "rule": "high-entropy-string", + "function": "test_export_search" + }, + { + "file": "tests/testing_utils.py", + "line": 17, + "rule": "missing-timeout", + "function": "fetch_and_unzip_test_db" + } + ] +} diff --git a/tests/regression/fixtures/vulnerable-flask/README.md b/tests/regression/fixtures/vulnerable-flask/README.md new file mode 100644 index 0000000..91e1a23 --- /dev/null +++ b/tests/regression/fixtures/vulnerable-flask/README.md @@ -0,0 +1,40 @@ +# Intentionally vulnerable Flask server + +This project is a local fixture for exercising CoreTrace. It deliberately contains +security vulnerabilities and must never be deployed or exposed to a network. + +## Analyze it + +Run this command from the root of `coretrace-python-analyzer`: + +```bash +coretrace-python-analyzer --check tests/regression/fixtures/vulnerable-flask +``` + +The project is expected to exercise these rules: + +- `command-injection`, through a helper in another module; +- `sql-injection`, also through that helper module; +- `path-traversal`; +- `ssrf`; +- `xss`; +- `dangerous-eval`; +- `weak-crypto`; +- `hardcoded-secret`, using a synthetic key that is not a real credential. + +`/safe-render` is a negative control: it applies `markupsafe.escape`, so the analyzer +should not report XSS for that route. + +## Run it locally (optional) + +Running the server is not required to analyze it. If you do run it, keep it local: + +```bash +cd tests-project/vulnerable-flask +python -m venv .venv +.venv/bin/python -m pip install -r requirements.txt +.venv/bin/python app.py +``` + +The application explicitly listens only on `127.0.0.1` with debug mode disabled. The +routes themselves remain intentionally unsafe. diff --git a/tests/regression/fixtures/vulnerable-flask/app.py b/tests/regression/fixtures/vulnerable-flask/app.py new file mode 100644 index 0000000..f6202e0 --- /dev/null +++ b/tests/regression/fixtures/vulnerable-flask/app.py @@ -0,0 +1,90 @@ +"""Intentionally vulnerable Flask application for CoreTrace demonstrations. + +This application is a local test fixture. Do not deploy it or expose it to a network. +""" + +from __future__ import annotations + +import hashlib + +import requests +from flask import Flask, Response, request, send_file +from markupsafe import escape +from vulnerable_services import find_user, initialize_database, run_command + +app = Flask(__name__) + +NOIR = "Noah" + +# Synthetic and unusable, but shaped like an AWS access key so the secret detector sees it. +DEMO_AWS_ACCESS_KEY = "AKIA0000000000000000" +DEMO_AWS_ACCESS_KEY2 = "AKIA0000000000000000" + + +@app.route("/") +def index() -> dict[str, object]: + return { + "warning": "Intentionally vulnerable test application", + "routes": ( + "/command?cmd=...", + "/users?name=...", + "/file?path=...", + "/fetch?url=...", + "/render?html=...", + "/calculate?expression=...", + "/digest?value=...", + "/safe-render?text=...", + ), + } + + +@app.route("/command") +def command_injection() -> dict[str, int]: + command = request.args["cmd"] + return {"exit_code": run_command(command)} + + +@app.route("/users") +def sql_injection() -> dict[str, object]: + username = request.args["name"] + return {"rows": find_user(username)} + + +@app.route("/file") +def path_traversal() -> Response: + return send_file(request.args["path"]) + + +@app.route("/fetch") +def server_side_request_forgery() -> dict[str, int]: + response = requests.get(request.args["url"], timeout=2) + return {"status": response.status_code} + + +@app.route("/render") +def cross_site_scripting() -> Response: + return Response(request.args["html"], mimetype="text/html") + + +@app.route("/calculate") +def code_injection() -> dict[str, str]: + result = eval(request.args["expression"]) + return {"result": repr(result)} + + +@app.route("/digest") +def weak_cryptography() -> dict[str, str]: + value = request.args["value"] + return {"digest": hashlib.md5(value.encode()).hexdigest()} + + +@app.route("/safe-render") +def safe_render_control() -> Response: + """Negative control: escaping should prevent an XSS finding for this route.""" + + return Response(escape(request.args["text"]), mimetype="text/html") + + +if __name__ == "__main__": + initialize_database() + app.run(host="127.0.0.1", port=5000, debug=False) diff --git a/tests/regression/fixtures/vulnerable-flask/requirements.txt b/tests/regression/fixtures/vulnerable-flask/requirements.txt new file mode 100644 index 0000000..99578c4 --- /dev/null +++ b/tests/regression/fixtures/vulnerable-flask/requirements.txt @@ -0,0 +1,2 @@ +Flask>=3.1,<4 +requests>=2.32,<3 diff --git a/tests/regression/fixtures/vulnerable-flask/vulnerable_services.py b/tests/regression/fixtures/vulnerable-flask/vulnerable_services.py new file mode 100644 index 0000000..d455e39 --- /dev/null +++ b/tests/regression/fixtures/vulnerable-flask/vulnerable_services.py @@ -0,0 +1,33 @@ +"""Intentionally unsafe service functions called by the Flask routes.""" + +from __future__ import annotations + +import os +import sqlite3 + +DATABASE = "demo.db" + + +def initialize_database() -> None: + connection = sqlite3.connect(DATABASE) + connection.execute( + "CREATE TABLE IF NOT EXISTS users (id INTEGER PRIMARY KEY, username TEXT UNIQUE)" + ) + connection.execute("INSERT OR IGNORE INTO users(username) VALUES ('alice')") + connection.commit() + connection.close() + + +def run_command(command: str) -> int: + # Intentionally unsafe: the command comes directly from an HTTP query parameter. + return os.system(command) + + +def find_user(username: str) -> list[tuple[object, ...]]: + connection = sqlite3.connect(DATABASE) + cursor = connection.cursor() + # Intentionally unsafe: string concatenation allows SQL injection. + query = "SELECT id, username FROM users WHERE username = '" + username + "'" + rows = cursor.execute(query).fetchall() + connection.close() + return rows diff --git a/tests/regression/repositories.toml b/tests/regression/repositories.toml new file mode 100644 index 0000000..db84293 --- /dev/null +++ b/tests/regression/repositories.toml @@ -0,0 +1,83 @@ +# Public repositories analysed by the non-regression suite, pinned by commit. Each one is +# checked out at that commit, analysed with the bundled plugins and compared with the +# snapshot in expected/.json. See tests/test_regression.py. + +[[repository]] +name = "Chatbot" +url = "https://github.com/ahmadfaizalbh/Chatbot.git" +commit = "6e78f7c071172fdf8f5d8880aa7903415b0e7463" + +[[repository]] +name = "DjangoGoat" +url = "https://github.com/red-and-black/DjangoGoat.git" +commit = "b5123eba8c002adb4f5513197e7e69a724385d29" + +[[repository]] +name = "Flask-Image-Editing-Website" +url = "https://github.com/vickytilotia/Flask-Image-Editing-Website.git" +commit = "9e7d0f46e778cbe6901643a1555be4b3ff77f4db" + +[[repository]] +name = "InsecureGram" +url = "https://github.com/Osama2012/InsecureGram.git" +commit = "e61786fa2d8ddc2ed364809d52f566f459c29b19" + +[[repository]] +name = "MiniBookApiServer" +url = "https://github.com/ViniciusFM/MiniBookApiServer.git" +commit = "e72af68bf4b0a0799cc39ffaa58b5caea5f647bf" + +[[repository]] +name = "my_gdb" +url = "https://github.com/thedreamteam240/my_gdb.git" +commit = "d158faf012a602df36698bd8d289ddb3eb3bc691" + +[[repository]] +name = "network-topologer" +url = "https://github.com/clement-antoine-xavier/network-topologer.git" +commit = "9b1a1fb62e33affba24090f62a1bd387e10a4b78" + +[[repository]] +name = "NPL-IA" +url = "https://github.com/SizzleUnrlsd/NPL-IA.git" +commit = "eb85aecb7f9bd44c1d1ad35af5871bd13e5f3a8d" + +[[repository]] +name = "Pylinkit" +url = "https://github.com/SizzleUnrlsd/Pylinkit.git" +commit = "aa5b5737c1de745bc77e6a8dd76c0f33edacb106" + +[[repository]] +name = "Spam-Detection-Project" +url = "https://github.com/Vatshayan/Spam-Detection-Project.git" +commit = "7defac6bb13616e525bc34a65dbcf2f21e53693f" + +[[repository]] +name = "tag-collector-server" +url = "https://github.com/arribada/tag-collector-server.git" +commit = "78f72776eb96e4511e8d327689cbd9c3bcf521c3" + +[[repository]] +name = "uploadbox-cloud-storage-solution" +url = "https://github.com/lakshman-chaudhary/uploadbox-cloud-storage-solution.git" +commit = "6263b58d4295e71f705b2f6d2691732a6d734fe7" + +[[repository]] +name = "VideoSubtitles" +url = "https://github.com/SizzleUnrlsd/VideoSubtitles.git" +commit = "37243e8631c14f4aa00e0c0014de471e7845255c" + +[[repository]] +name = "Vulnerable_fastapi" +url = "https://github.com/Thalin19/Vulnerable_fastapi.git" +commit = "0028a3dc3d5646f4f6a098374f59e5dbe6a41303" + +[[repository]] +name = "yt-fts" +url = "https://github.com/NotJoeMartinez/yt-fts.git" +commit = "7fbc0088f30672b6fbfe2ea7351fa42bafe1f886" + +# Intentionally vulnerable Flask fixture kept in the tree. +[[repository]] +name = "vulnerable-flask" +path = "fixtures/vulnerable-flask" From 25d8584faeab6cf49ac9c749f4024ea2d66889ad Mon Sep 17 00:00:00 2001 From: Hugo Date: Fri, 4 Sep 2026 18:51:10 +0200 Subject: [PATCH 3/3] fix: keep only public repositories in the regression suite --- tests/regression/expected/Pylinkit.json | 22 ------------------- tests/regression/expected/VideoSubtitles.json | 9 -------- .../expected/tag-collector-server.json | 16 -------------- tests/regression/repositories.toml | 18 ++------------- tests/test_regression.py | 2 +- 5 files changed, 3 insertions(+), 64 deletions(-) delete mode 100644 tests/regression/expected/Pylinkit.json delete mode 100644 tests/regression/expected/VideoSubtitles.json delete mode 100644 tests/regression/expected/tag-collector-server.json diff --git a/tests/regression/expected/Pylinkit.json b/tests/regression/expected/Pylinkit.json deleted file mode 100644 index 5f5dde3..0000000 --- a/tests/regression/expected/Pylinkit.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "coverage": { - "files": 11, - "files_analysed": 11, - "functions": 213, - "functions_analysed": 213 - }, - "findings": [ - { - "file": "pylinkit/gui.py", - "line": 525, - "rule": "hardcoded-secret", - "function": "_sync_pass_predict" - }, - { - "file": "pylinkit/gui.py", - "line": 526, - "rule": "missing-timeout", - "function": "_sync_pass_predict" - } - ] -} diff --git a/tests/regression/expected/VideoSubtitles.json b/tests/regression/expected/VideoSubtitles.json deleted file mode 100644 index 9e7a974..0000000 --- a/tests/regression/expected/VideoSubtitles.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "coverage": { - "files": 3, - "files_analysed": 3, - "functions": 13, - "functions_analysed": 13 - }, - "findings": [] -} diff --git a/tests/regression/expected/tag-collector-server.json b/tests/regression/expected/tag-collector-server.json deleted file mode 100644 index b68da80..0000000 --- a/tests/regression/expected/tag-collector-server.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "coverage": { - "files": 5, - "files_analysed": 5, - "functions": 68, - "functions_analysed": 68 - }, - "findings": [ - { - "file": "MQTT-client/influx_writer.py", - "line": 17, - "rule": "hardcoded-credential", - "function": "get_influx_client" - } - ] -} diff --git a/tests/regression/repositories.toml b/tests/regression/repositories.toml index db84293..34a8ca5 100644 --- a/tests/regression/repositories.toml +++ b/tests/regression/repositories.toml @@ -1,4 +1,5 @@ -# Public repositories analysed by the non-regression suite, pinned by commit. Each one is +# Public repositories analysed by the non-regression suite, pinned by commit. Private +# repositories cannot be listed: the CI job clones without credentials. Each one is # checked out at that commit, analysed with the bundled plugins and compared with the # snapshot in expected/.json. See tests/test_regression.py. @@ -42,31 +43,16 @@ name = "NPL-IA" url = "https://github.com/SizzleUnrlsd/NPL-IA.git" commit = "eb85aecb7f9bd44c1d1ad35af5871bd13e5f3a8d" -[[repository]] -name = "Pylinkit" -url = "https://github.com/SizzleUnrlsd/Pylinkit.git" -commit = "aa5b5737c1de745bc77e6a8dd76c0f33edacb106" - [[repository]] name = "Spam-Detection-Project" url = "https://github.com/Vatshayan/Spam-Detection-Project.git" commit = "7defac6bb13616e525bc34a65dbcf2f21e53693f" -[[repository]] -name = "tag-collector-server" -url = "https://github.com/arribada/tag-collector-server.git" -commit = "78f72776eb96e4511e8d327689cbd9c3bcf521c3" - [[repository]] name = "uploadbox-cloud-storage-solution" url = "https://github.com/lakshman-chaudhary/uploadbox-cloud-storage-solution.git" commit = "6263b58d4295e71f705b2f6d2691732a6d734fe7" -[[repository]] -name = "VideoSubtitles" -url = "https://github.com/SizzleUnrlsd/VideoSubtitles.git" -commit = "37243e8631c14f4aa00e0c0014de471e7845255c" - [[repository]] name = "Vulnerable_fastapi" url = "https://github.com/Thalin19/Vulnerable_fastapi.git" diff --git a/tests/test_regression.py b/tests/test_regression.py index 1c2b088..ad9062d 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -103,7 +103,7 @@ def test_every_repository_has_a_snapshot_and_vice_versa() -> None: assert MANIFEST.is_file(), f"{MANIFEST} is missing" names = {repository.name for repository in REPOSITORIES} assert names == {path.stem for path in EXPECTED.glob("*.json")} - assert len(names) >= 16 + assert len(names) >= 13 assert all( (r.path is None) != (r.url is None and r.commit is None) for r in REPOSITORIES )