From ada3864c4a346143a8730662c02e43141b93b971 Mon Sep 17 00:00:00 2001 From: Samran Asif Date: Tue, 8 Sep 2026 00:30:19 +0500 Subject: [PATCH 1/4] docs: give every repo a reporting channel that actually exists Two of the four Code of Conduct files pointed at GitHub features that are not real: - devrepro-doctor: report "via GitHub private message". GitHub has no private messaging. - local-ai-hardware-bench: report by "opening a private issue tagged `conduct`". GitHub has private *vulnerability reports*; it has no private issues. api-verity-lab gave a profile URL rather than a contact channel. Only tooltrace-bench named an address that works. Someone reporting harassment is the worst possible person to hand a dead end, so all four now carry the same wording: the maintainer's noreply address plus GitHub's real report-abuse form. Worse, and found while checking the above: all four SECURITY.md files direct reporters to GitHub's private vulnerability reporting, and it was **disabled on all four repositories**. Every documented security-disclosure path in this family of projects led to a page the reporter could not use. It is enabled now (a repository setting, so not visible in this diff). The disclosure SLAs also disagreed for the same solo maintainer -- 72h, 72h+7d, 7d, and 7d+30d. Standardized on 7 days to acknowledge and 30 to update, the most conservative of the four, and said plainly why: promising 72 hours when nobody is on call is a promise, not a policy. tests/test_contact_channels_exist.py pins this. It asserts the working address and the report-abuse form are present, that SECURITY.md still names private vulnerability reporting and still warns against public issues, and -- the point of the exercise -- that no document mentions "private message", "private issue" or "report-user functionality" again. Verified by reintroducing the devrepro-doctor wording and watching it fail. --- CODE_OF_CONDUCT.md | 8 ++- SECURITY.md | 6 +- tests/unit/test_contact_channels_exist.py | 74 +++++++++++++++++++++++ 3 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 tests/unit/test_contact_channels_exist.py diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index 46818cb..6d7e853 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -51,7 +51,13 @@ representing the project in public spaces. ## Enforcement Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported to the lead maintainer at https://github.com/webdevsamran. +reported to the lead maintainer at **webdevsamran@users.noreply.github.com**, or +through GitHub's report-abuse form at +. All complaints will be reviewed and +investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the diff --git a/SECURITY.md b/SECURITY.md index e04db83..1e4e92a 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -14,7 +14,11 @@ Use GitHub's private vulnerability reporting: https://github.com/webdevsamran/api-verity-lab/security/advisories/new Include: affected component, reproduction steps, impact assessment, -and any suggested fix. You will receive an acknowledgment within 72 hours. +and any suggested fix. + +You will receive an acknowledgment within 7 days and a status update within 30 +days. This project has a single maintainer; those are the windows that can +actually be met, rather than a shorter number that sounds better. ## Security design principles of this project diff --git a/tests/unit/test_contact_channels_exist.py b/tests/unit/test_contact_channels_exist.py new file mode 100644 index 0000000..f74f1c5 --- /dev/null +++ b/tests/unit/test_contact_channels_exist.py @@ -0,0 +1,74 @@ +"""The documented ways to reach a maintainer must be things that exist. + +Two of the four Code of Conduct files in this family of projects told people to +use GitHub features that are not real: "GitHub private message" (there is no +such feature) and "opening a private issue tagged `conduct`" (GitHub has +private *vulnerability reports*, not private issues). A third gave a profile +URL instead of a contact channel. + +Someone reporting harassment is the worst possible person to hand a dead end, +so the channels are pinned here. This is a documentation test on purpose: the +defect was never in the code. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +_ROOT = Path(__file__).resolve().parent.parent.parent +_CONTACT = "webdevsamran@users.noreply.github.com" + +#: Mechanisms these files have claimed that GitHub does not provide. +_NONEXISTENT = ( + r"private message", + r"private issue", + r"report-user function", +) + + +def _read(name: str) -> str: + return (_ROOT / name).read_text(encoding="utf-8") + + +def test_the_code_of_conduct_names_a_channel_that_exists() -> None: + text = _read("CODE_OF_CONDUCT.md") + assert _CONTACT in text, "CODE_OF_CONDUCT.md names no working contact address" + assert "https://github.com/contact/report-abuse" in text, ( + "CODE_OF_CONDUCT.md dropped GitHub's report-abuse form" + ) + + +def test_no_document_invents_a_github_feature() -> None: + offenders = [] + for name in ("CODE_OF_CONDUCT.md", "SECURITY.md", "CONTRIBUTING.md"): + path = _ROOT / name + if not path.exists(): + continue + body = path.read_text(encoding="utf-8") + for pattern in _NONEXISTENT: + if re.search(pattern, body, re.IGNORECASE): + offenders.append(f"{name}: {pattern!r}") + assert not offenders, ( + "these documents point at GitHub features that do not exist: " + ", ".join(offenders) + ) + + +def test_security_reporting_points_at_private_vulnerability_reporting() -> None: + """The channel has to be enabled on the repository, not just written down. + + It was documented in all four of these projects and enabled in none, so a + reporter following the instructions reached a page they could not use. + Enabling it is a repository setting, which a test cannot assert -- what it + can assert is that the document keeps naming the real mechanism rather than + drifting back to an invented one. + """ + text = _read("SECURITY.md") + assert re.search(r"security/advisories/new|GitHub Security Advisories", text), ( + "SECURITY.md no longer points at GitHub's private vulnerability reporting" + ) + # api-verity-lab writes "public GitHub issue", the others "public issue"; + # the first version of this assertion matched only the latter. + assert re.search(r"public (github )?issue", text, re.IGNORECASE), ( + "SECURITY.md dropped the do-not-file-publicly warning" + ) From 2f6928f644f8756f1c09d5914e7cdbb90239d63d Mon Sep 17 00:00:00 2001 From: Samran Asif Date: Tue, 8 Sep 2026 00:45:05 +0500 Subject: [PATCH 2/4] fix(server): stop two endpoints echoing exception text back over HTTP CodeQL reported two `py/stack-trace-exposure` alerts (medium). Both are real, though for different reasons. `/readyz` is unauthenticated and returned `str(exc)` from whatever the sqlite3 probe raised. A sqlite3 message carries the database path and often schema detail, handed to anyone who can reach the port. A readiness probe needs one bit; the detail goes to the server's own log, where an operator can already see it. The job-enqueue handler returned `str(exc)` from `QueueFull`, whose message this codebase writes itself, so nothing untrusted was leaking today. The habit is still the thing worth removing: the next exception to reach that handler may not be one we wrote. It now returns a fixed message plus `max_active_jobs` as a field, which is also easier for a client to act on than parsing prose. tests/integration/test_selfhosted_server.py gains a test that makes `/readyz` fail with a message containing a database path and asserts the response body contains neither it nor the error text. --- apiverity/server/api.py | 27 ++++++++++++++++++--- tests/integration/test_selfhosted_server.py | 27 +++++++++++++++++++++ 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/apiverity/server/api.py b/apiverity/server/api.py index 76b2aaa..830c561 100644 --- a/apiverity/server/api.py +++ b/apiverity/server/api.py @@ -115,8 +115,14 @@ def readyz() -> Any: try: store.conn.execute("SELECT 1").fetchone() return jsonify({"status": "ready"}) - except Exception as exc: - return jsonify({"status": "not-ready", "error": str(exc)}), 503 + except Exception: + # `/readyz` is unauthenticated, and a sqlite3 exception string + # carries the database path and often schema detail. A readiness + # probe needs one bit; the detail belongs in the server's own log, + # where the operator can already see it. (CodeQL + # py/stack-trace-exposure, and it was right.) + app.logger.exception("readiness check failed") + return jsonify({"status": "not-ready"}), 503 @app.get("/metrics") def metrics() -> Any: @@ -323,9 +329,22 @@ def enqueue_job() -> Any: environment=body.get("environment"), idempotency_key=body.get("idempotency_key"), ) - except QueueFull as exc: + except QueueFull: + # QueueFull's message names the org id and the configured limit. + # Both are the caller's own, but echoing an exception's text back + # over HTTP is the habit worth not having: the next exception to + # reach this handler may not be one this codebase wrote. The limit + # is returned as a field instead, which is also easier to act on. _METRICS["jobs_rejected_total"] += 1 - return jsonify({"error": str(exc)}), 409 + return ( + jsonify( + { + "error": "concurrent job limit reached", + "max_active_jobs": queue.max_active_per_org, + } + ), + 409, + ) _METRICS["jobs_enqueued_total"] += 1 return ( jsonify({"run_id": run_id, "deduplicated": not created}), diff --git a/tests/integration/test_selfhosted_server.py b/tests/integration/test_selfhosted_server.py index 69b21a8..71aa24d 100644 --- a/tests/integration/test_selfhosted_server.py +++ b/tests/integration/test_selfhosted_server.py @@ -159,6 +159,33 @@ def test_health(self, client) -> None: def test_readyz(self, client) -> None: assert client.get("/readyz").status_code == 200 + def test_readyz_does_not_leak_the_database_error( + self, store: Store, monkeypatch + ) -> None: + """An unauthenticated probe must not describe why it is unhappy. + + `/readyz` returned `str(exc)` from the sqlite3 failure, which carries + the database path and often schema detail, to anyone who could reach + the port. CodeQL flagged it as py/stack-trace-exposure and was right. + A readiness probe needs one bit; the detail goes to the server log. + """ + + class _Boom: + def execute(self, *_args: object, **_kw: object) -> object: + raise RuntimeError("no such table: runs in /srv/secret/fleet.db") + + app = create_app(store) + app.config["TESTING"] = True + monkeypatch.setattr(store, "conn", _Boom()) + + resp = app.test_client().get("/readyz") + + assert resp.status_code == 503 + assert resp.get_json() == {"status": "not-ready"} + body = resp.get_data(as_text=True) + assert "fleet.db" not in body + assert "no such table" not in body + def test_metrics_exposed(self, client) -> None: text = client.get("/metrics").get_data(as_text=True) assert "apiverity_requests_total" in text From 6143fa5c5476671ebb3251f43efc5da7efe93fcc Mon Sep 17 00:00:00 2001 From: Samran Asif Date: Tue, 8 Sep 2026 00:52:15 +0500 Subject: [PATCH 3/4] fix(ci): stop the e2e gate failing because the runner was busy The Windows leg added in #53 turned up a second latent problem, alongside the retention bug: `scripts/e2e.py` failed the build on any non-zero exit from `apiverity regression`. EXIT_FINDINGS means "a regression was detected" -- the command working exactly as documented. And on a contended runner a mock server on localhost genuinely does go from 1.7ms to 16ms, past even the 400% tolerance that was chosen to be generous. The step's own comment already said its job was to "verify the command wiring and exit codes end-to-end"; it then gated on the one thing that depends on how loaded the machine is. That is the failure mode where a red build teaches people to ignore red builds. The step now accepts EXIT_OK or EXIT_FINDINGS and fails only on a usage, unreachable or internal code -- which is what "the wiring works" actually means. The comparison logic keeps its unit tests. Worth noting what the tool did right while failing this: it separated three genuine threshold breaches from four it reported as inconclusive, with the overlapping intervals printed and a suggestion to raise --iterations. The statistics were not the problem. Also formats the test added in the previous commit; CI checks `tests` too and I had only re-run ruff over `apiverity`. --- scripts/e2e.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/scripts/e2e.py b/scripts/e2e.py index a668fd2..6d626ae 100644 --- a/scripts/e2e.py +++ b/scripts/e2e.py @@ -5,6 +5,7 @@ import sys from pathlib import Path +from apiverity.cli.commands.common import EXIT_FINDINGS, EXIT_OK from apiverity.cli.main import main as cli_main from apiverity.mock import MockServer from apiverity.specs.loader import detect_and_load @@ -104,9 +105,17 @@ def main() -> None: ) if code != 0: failures.append(f"baseline -> {code}") - # Tolerance is deliberately generous: localhost timings are noisy and - # the strict comparison logic is unit-tested in the pytest suite. - # Here we verify the command wiring and exit codes end-to-end. + # This step checks the command *wires up*: it loads a baseline, runs a + # comparison and returns a code from the documented contract. It must + # not check whether the numbers came out fast. + # + # It used to fail the build on any non-zero exit. EXIT_FINDINGS means + # "a regression was detected", which is the command working correctly, + # and on a contended CI runner a mock server on localhost genuinely + # does go from 1.7ms to 16ms -- past even the 400% tolerance chosen to + # be generous. That made a correctness gate depend on runner load, and + # a gate that reddens for reasons nobody can act on is a gate someone + # eventually deletes. The comparison logic itself is unit-tested. code = run( [ "regression", @@ -123,8 +132,11 @@ def main() -> None: "GET /users p95 <= 5000ms", ] ) - if code != 0: - failures.append(f"regression -> {code}") + if code not in (EXIT_OK, EXIT_FINDINGS): + failures.append( + f"regression -> {code} (expected 0 or 1; anything else is a " + "usage, unreachable or internal error)" + ) # 5. redaction sanity from apiverity.traffic.redact import RedactionConfig, redact_headers, redact_json From 0a68f9959d2c7ddd893d28f20a42d5edcec9c8d8 Mon Sep 17 00:00:00 2001 From: Samran Asif Date: Tue, 8 Sep 2026 00:53:30 +0500 Subject: [PATCH 4/4] style: ruff format the readyz regression test CI's Format check covers tests as well as apiverity; the cherry-pick carried the code change but I re-ran the formatter before it, not after. --- tests/integration/test_selfhosted_server.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/test_selfhosted_server.py b/tests/integration/test_selfhosted_server.py index 71aa24d..96f0c28 100644 --- a/tests/integration/test_selfhosted_server.py +++ b/tests/integration/test_selfhosted_server.py @@ -159,9 +159,7 @@ def test_health(self, client) -> None: def test_readyz(self, client) -> None: assert client.get("/readyz").status_code == 200 - def test_readyz_does_not_leak_the_database_error( - self, store: Store, monkeypatch - ) -> None: + def test_readyz_does_not_leak_the_database_error(self, store: Store, monkeypatch) -> None: """An unauthenticated probe must not describe why it is unhappy. `/readyz` returned `str(exc)` from the sqlite3 failure, which carries