Add the non-regression suite on pinned repositories to CI

.github/workflows/ci.yml

@@ -20,6 +20,20 @@ jobs:
       - run: python -m ruff check .
       - run: python -m pytest
 
+  regression:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - uses: actions/cache@v4
+        with:
+          path: .regression
+          key: regression-${{ hashFiles('tests/regression/repositories.toml') }}
+      - run: python -m pip install -e ".[dev]"
+      - run: python -m pytest -m regression
+
   package:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -8,3 +8,4 @@ build/
 dist/
 
 .DS_Store
+.regression/

README.md

@@ -24,6 +24,16 @@ python -m pytest
 python -m ruff check .
 ```
 
+The non-regression suite analyses the public repositories pinned in
+[`tests/regression/repositories.toml`](tests/regression/repositories.toml) and compares
+findings and coverage with the snapshots in `tests/regression/expected/`. It clones on
+first use, needs the network and runs in its own CI job:
+
+```bash
+python -m pytest -m regression
+CORETRACE_REGRESSION_UPDATE=1 python -m pytest -m regression   # record an intended change
+```
+
 ## Install
 
 ```bash

pyproject.toml

@@ -46,7 +46,10 @@ coretrace-python-analyzer = "coretrace_python.cli:main"
 packages = ["src/coretrace_python"]
 
 [tool.pytest.ini_options]
-addopts = "-q"
+addopts = "-q -m 'not regression'"
+markers = [
+  "regression: analyses the pinned public repositories of tests/regression (network, slow)",
+]
 testpaths = ["tests"]
 
 [tool.ruff]

tests/regression/expected/Chatbot.json

@@ -0,0 +1,22 @@
+{
+  "coverage": {
+    "files": 16,
+    "files_analysed": 16,
+    "functions": 176,
+    "functions_analysed": 176
+  },
+  "findings": [
+    {
+      "file": "chatbot/__init__.py",
+      "line": 224,
+      "rule": "missing-timeout",
+      "function": "train"
+    },
+    {
+      "file": "setup.py",
+      "line": 48,
+      "rule": "command-injection",
+      "function": "install_ollama"
+    }
+  ]
+}

tests/regression/expected/DjangoGoat.json

@@ -0,0 +1,58 @@
+{
+  "coverage": {
+    "files": 29,
+    "files_analysed": 29,
+    "functions": 39,
+    "functions_analysed": 39
+  },
+  "findings": [
+    {
+      "file": "authentication/views.py",
+      "line": 33,
+      "rule": "open-redirect",
+      "function": "sign_up"
+    },
+    {
+      "file": "authentication/views.py",
+      "line": 57,
+      "rule": "sql-injection",
+      "function": "log_in"
+    },
+    {
+      "file": "authentication/views.py",
+      "line": 82,
+      "rule": "open-redirect",
+      "function": "profile_update"
+    },
+    {
+      "file": "djangogoat/settings.py",
+      "line": 24,
+      "rule": "hardcoded-credential",
+      "function": null
+    },
+    {
+      "file": "features/environment.py",
+      "line": 196,
+      "rule": "hardcoded-credential",
+      "function": "after_all"
+    },
+    {
+      "file": "notes/views.py",
+      "line": 86,
+      "rule": "open-redirect",
+      "function": "conversation"
+    },
+    {
+      "file": "requirements_app.txt",
+      "line": 2,
+      "rule": "vulnerable-dependency",
+      "function": null
+    },
+    {
+      "file": "requirements_app.txt",
+      "line": 3,
+      "rule": "vulnerable-dependency",
+      "function": null
+    }
+  ]
+}

tests/regression/expected/Flask-Image-Editing-Website.json

@@ -0,0 +1,52 @@
+{
+  "coverage": {
+    "files": 1,
+    "files_analysed": 1,
+    "functions": 4,
+    "functions_analysed": 4
+  },
+  "findings": [
+    {
+      "file": "main.py",
+      "line": 15,
+      "rule": "hardcoded-credential",
+      "function": null
+    },
+    {
+      "file": "main.py",
+      "line": 29,
+      "rule": "reachable-vulnerability",
+      "function": "processImage"
+    },
+    {
+      "file": "main.py",
+      "line": 79,
+      "rule": "exploitable-vulnerability",
+      "function": "edit"
+    },
+    {
+      "file": "main.py",
+      "line": 91,
+      "rule": "debug-enabled",
+      "function": null
+    },
+    {
+      "file": "requirements.txt",
+      "line": 9,
+      "rule": "vulnerable-dependency",
+      "function": null
+    },
+    {
+      "file": "requirements.txt",
+      "line": 10,
+      "rule": "vulnerable-dependency",
+      "function": null
+    },
+    {
+      "file": "requirements.txt",
+      "line": 10,
+      "rule": "vulnerable-dependency",
+      "function": null
+    }
+  ]
+}

tests/regression/expected/InsecureGram.json

@@ -0,0 +1,64 @@
+{
+  "coverage": {
+    "files": 15,
+    "files_analysed": 15,
+    "functions": 15,
+    "functions_analysed": 15
+  },
+  "findings": [
+    {
+      "file": "app/admin.py",
+      "line": 45,
+      "rule": "insecure-deserialization",
+      "function": "insecure_deserialize"
+    },
+    {
+      "file": "app/auth.py",
+      "line": 40,
+      "rule": "hardcoded-credential",
+      "function": "login"
+    },
+    {
+      "file": "app/command.py",
+      "line": 12,
+      "rule": "command-injection",
+      "function": "execute"
+    },
+    {
+      "file": "app/config.py",
+      "line": 4,
+      "rule": "hardcoded-credential",
+      "function": null
+    },
+    {
+      "file": "app/config.py",
+      "line": 6,
+      "rule": "hardcoded-credential",
+      "function": null
+    },
+    {
+      "file": "app/redirect.py",
+      "line": 11,
+      "rule": "open-redirect",
+      "function": "redirect"
+    },
+    {
+      "file": "app/ssrf.py",
+      "line": 12,
+      "rule": "ssrf",
+      "function": "proxy"
+    },
+    {
+      "file": "app/upload.py",
+      "line": 36,
+      "rule": "path-traversal",
+      "function": "upload"
+    },
+    {
+      "file": "app/users.py",
+      "line": 36,
+      "rule": "sql-injection",
+      "function": "search_users"
+    }
+  ]
+}

tests/regression/expected/MiniBookApiServer.json

@@ -0,0 +1,64 @@
+{
+  "coverage": {
+    "files": 6,
+    "files_analysed": 6,
+    "functions": 40,
+    "functions_analysed": 40
+  },
+  "findings": [
+    {
+      "file": "app.py",
+      "line": 69,
+      "rule": "path-traversal",
+      "function": "get_img"
+    },
+    {
+      "file": "app.py",
+      "line": 100,
+      "rule": "exploitable-vulnerability",
+      "function": "new_book"
+    },
+    {
+      "file": "config-example.json",
+      "line": 3,
+      "rule": "hardcoded-credential",
+      "function": null
+    },
+    {
+      "file": "config-example.json",
+      "line": 4,
+      "rule": "hardcoded-credential",
+      "function": null
+    },
+    {
+      "file": "docs/insomnia/insomnia_config_file.yaml",
+      "line": 71,
+      "rule": "high-entropy-string",
+      "function": null
+    },
+    {
+      "file": "docs/insomnia/insomnia_config_file.yaml",
+      "line": 231,
+      "rule": "hardcoded-credential",
+      "function": null
+    },
+    {
+      "file": "docs/insomnia/insomnia_config_file.yaml",
+      "line": 232,
+      "rule": "hardcoded-credential",
+      "function": null
+    },
+    {
+      "file": "requirements.txt",
+      "line": 10,
+      "rule": "vulnerable-dependency",
+      "function": null
+    },
+    {
+      "file": "res.py",
+      "line": 29,
+      "rule": "reachable-vulnerability",
+      "function": "store_pic_from_base64"
+    }
+  ]
+}

tests/regression/expected/NPL-IA.json

@@ -0,0 +1,16 @@
+{
+  "coverage": {
+    "files": 1,
+    "files_analysed": 1,
+    "functions": 9,
+    "functions_analysed": 9
+  },
+  "findings": [
+    {
+      "file": "model.json",
+      "line": 1,
+      "rule": "hardcoded-credential",
+      "function": null
+    }
+  ]
+}

tests/regression/expected/Spam-Detection-Project.json

@@ -0,0 +1,16 @@
+{
+  "coverage": {
+    "files": 1,
+    "files_analysed": 1,
+    "functions": 2,
+    "functions_analysed": 2
+  },
+  "findings": [
+    {
+      "file": "app.py",
+      "line": 50,
+      "rule": "debug-enabled",
+      "function": null
+    }
+  ]
+}