From 60ecff1b8939ee03f156dee2dbd05d815255c320 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 18:58:18 +0000 Subject: [PATCH 1/8] feat(assurance): machine-checkable claim register and its CI check Customers' privacy, legal and audit teams need every claim in the documentation traced to a proof, a test or a measured report (roadmap T17). tools/claims_check.py checks docs/assurance/claims.yaml: quotes must still be in their documents, evidence must resolve (proposition labels numbered as LaTeX prints them, check ids, pytest nodes, report files), claim-like sentences of the documents in scope must be registered or exempted with a reason, and Prop. N references in any Markdown document must name a defined proposition. tests/unit/test_assurance.py runs the check, tests each kind of drift it must catch, and confines network access to the service modules the security guide names. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PSVqwE13trs5MdpBoE4YyJ Signed-off-by: Claude --- .github/workflows/ci.yml | 3 + tests/unit/test_assurance.py | 136 +++++++++++++++ tools/claims_check.py | 321 +++++++++++++++++++++++++++++++++++ 3 files changed, 460 insertions(+) create mode 100644 tests/unit/test_assurance.py create mode 100644 tools/claims_check.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e43066e..9ddd20c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,9 @@ jobs: run: python tools/check_spdx.py - name: Dependency licences run: python tools/check_licenses.py --report licenses.md && cat licenses.md >> "$GITHUB_STEP_SUMMARY" + - name: Claim register (every documented claim traces to a proof, test or report) + shell: bash # -o pipefail: the step fails when the check does + run: python tools/claims_check.py --json claims-resolved.json | tee -a "$GITHUB_STEP_SUMMARY" types: runs-on: ubuntu-latest diff --git a/tests/unit/test_assurance.py b/tests/unit/test_assurance.py new file mode 100644 index 0000000..edb7871 --- /dev/null +++ b/tests/unit/test_assurance.py @@ -0,0 +1,136 @@ +# SPDX-License-Identifier: Apache-2.0 +"""The claim register (docs/assurance/claims.yaml) and the facts the assurance +case relies on that no other test covers. + +* Every registered claim still appears in its document and every piece of + evidence resolves; every claim-like sentence of the documents in scope is + registered (tools/claims_check.py, also run by the CI job ``lint``). +* The checker itself: proposition numbering as LaTeX prints it, and each kind + of drift it must catch. +* The package opens network connections only in the modules the security + guide names (docs/security.md, "Supply chain and code"). +""" +from __future__ import annotations + +import ast +import importlib.util +import sys +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[2] + + +def _checker(): + spec = importlib.util.spec_from_file_location("claims_check", ROOT / "tools" / "claims_check.py") + mod = importlib.util.module_from_spec(spec) + sys.modules["claims_check"] = mod + spec.loader.exec_module(mod) + return mod + + +cc = _checker() + + +def test_claim_register_traces_every_claim(): + res = cc.run(ROOT) + assert res.problems == [], "\n".join(res.problems) + assert len(res.claims) >= 50 + + +def test_proposition_numbers_follow_latex_counters(): + props = cc.proposition_numbers(ROOT) + assert props["prop:pit"] == 1 + assert props["prop:pdsuff"] == 100 and props["prop:pdeligible"] == 105 + assert props["prop:pusplit"] == 600 and props["prop:puprofile"] == 605 # out-of-order setcounter + assert props["prop:ledger"] == 800 and props["prop:govchain"] == 803 + assert props["prop:dgauss"] == 900 and props["prop:run"] == 906 + assert props["prop:tschange"] == 1200 + assert props["prop:irround"] == 106 # restored after a numbered block + assert len(set(props.values())) == len(props), "two propositions print the same number" + + +def _mini_repo(tmp_path: Path, register: str, doc: str) -> Path: + (tmp_path / "docs/assurance").mkdir(parents=True) + (tmp_path / "docs/proofs").mkdir() + (tmp_path / "legacy/checks").mkdir(parents=True) + (tmp_path / "tests/unit").mkdir(parents=True) + (tmp_path / "docs/proofs/relsynth-proofs.tex").write_text( + "\\begin{proposition}\\label{prop:a}x\\end{proposition}\n" + "\\newcounter{sv}\\setcounter{sv}{\\value{proposition}}\\setcounter{proposition}{899}\n" + "\\begin{proposition}[T]\\label{prop:b}y\\end{proposition}\n" + "\\setcounter{proposition}{\\value{sv}}\n" + "\\begin{proposition}\\label{prop:c}z\\end{proposition} % \\begin{proposition}\n") + (tmp_path / "legacy/checks/run_checks.py").write_text('@check("C01", "x", "Prop. 1")\n') + (tmp_path / "tests/unit/test_x.py").write_text( + "def test_c900_b():\n pass\n\nclass TestK:\n def test_m(self):\n pass\n") + (tmp_path / "docs/guide.md").write_text(doc) + (tmp_path / "docs/assurance/claims.yaml").write_text(register) + return tmp_path + + +GOOD = """ +scope: [docs/guide.md] +claims: + - id: CL-1 + doc: docs/guide.md + quote: Every key resolves. + status: supported + evidence: [{prop: prop:b}, {check: C900}, {check: C01}, {test: "tests/unit/test_x.py::TestK::test_m"}] +not_claims: + - {doc: docs/guide.md, quote: "Run it every night.", reason: instruction} +""" + + +def test_checker_accepts_a_traced_document(tmp_path): + root = _mini_repo(tmp_path, GOOD, "# Guide\n\nEvery key\nresolves. Run it every night.\n\nSee Prop. 900 and Props. 1–2.\n") + res = cc.run(root) + assert res.problems == [] + assert cc.proposition_numbers(root) == {"prop:a": 1, "prop:b": 900, "prop:c": 2} + assert res.claims[0]["evidence"][0] == {"kind": "prop", "ref": "prop:b", "number": 900} + + +@pytest.mark.parametrize(("doc", "register", "expect"), [ + ("Every key is unique. Run it every night.\n", GOOD, "quote not found"), + ("Every key resolves. Run it every night. Rows are never copied.\n", GOOD, "unregistered claim"), + ("Every key resolves. Run it every night. See Prop. 3.\n", GOOD, "Proposition 3, which the proofs do not define"), + ("Every key resolves. Run it every night.\n", GOOD.replace("prop:b", "prop:zz"), "no proposition labelled prop:zz"), + ("Every key resolves. Run it every night.\n", GOOD.replace("C900", "C901"), "no check C901"), + ("Every key resolves. Run it every night.\n", GOOD.replace("TestK::test_m", "TestK::test_n"), "has no TestK::test_n"), + ("Every key resolves. Run it every night.\n", + GOOD.replace('[{prop: prop:b}, {check: C900}, {check: C01}, {test: "tests/unit/test_x.py::TestK::test_m"}]', + "[{source: 'EDPB 02/2026 para. 55'}]"), "needs a prop, check, test or report"), + ("Every key resolves. Run it every night.\n", GOOD.replace("reason: instruction", "reason: ''"), "give a reason"), +]) +def test_checker_catches_drift(tmp_path, doc, register, expect): + res = cc.run(_mini_repo(tmp_path, register, doc)) + assert any(expect in p for p in res.problems), res.problems + + +# Modules allowed to open network connections, and why (docs/security.md). +NETWORK_ALLOWED = { + "relsynth/service/auth.py": "OIDC discovery and keys of the identity provider the operator configures", + "relsynth/service/secrets.py": "the Vault server named by VAULT_ADDR", + "relsynth/service/telemetry.py": "OpenTelemetry export, only when otel is turned on", +} +NETWORK_MODULES = ("urllib.request", "http.client", "requests", "httpx", "aiohttp", "urllib3", "opentelemetry.exporter") + + +def test_network_access_is_confined_to_named_modules(): + found = {} + for path in sorted((ROOT / "src").rglob("*.py")): + rel = path.relative_to(ROOT / "src").as_posix() + for node in ast.walk(ast.parse(path.read_text(encoding="utf-8"))): + names = [] + if isinstance(node, ast.Import): + names = [a.name for a in node.names] + elif isinstance(node, ast.ImportFrom) and node.module: + names = [node.module] + elif isinstance(node, ast.Attribute) and node.attr in ("create_connection", "connect") and \ + isinstance(node.value, ast.Name) and node.value.id == "socket": + names = ["socket." + node.attr] + hits = [n for n in names if n.startswith(NETWORK_MODULES) or n.startswith("socket.")] + if hits: + found.setdefault(rel, set()).update(hits) + assert set(found) <= set(NETWORK_ALLOWED), {k: v for k, v in found.items() if k not in NETWORK_ALLOWED} diff --git a/tools/claims_check.py b/tools/claims_check.py new file mode 100644 index 0000000..132ecd5 --- /dev/null +++ b/tools/claims_check.py @@ -0,0 +1,321 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Check the claim register: every claim in the documentation traces to evidence. + + python tools/claims_check.py # exit 1 and list the problems + python tools/claims_check.py --suggest FILE # print register stubs for FILE's unregistered claims + python tools/claims_check.py --json out.json # also write the resolved register (for the assurance case) + +The register is ``docs/assurance/claims.yaml``. Each entry binds a sentence of +a document (``quote``, which must still appear in ``doc``) to the evidence that +supports it. Evidence is one of: + +``prop`` a proposition label in ``docs/proofs/*.tex`` (``prop:dgauss``); +``check`` a numbered check (``C902``): a ``@check("C902", ...)`` in + ``legacy/checks/run_checks.py`` or a ``def test_c902_...`` in ``tests/``; +``test`` a pytest node, ``tests/unit/test_dp.py::test_name`` (or ``::Class::test``); +``report`` a file in the repository that records a measurement; +``code`` ``src/relsynth/x.py::symbol``, for claims that are enforced by code + (a refusal, a mask) and also have a ``test``; +``source`` an external text (a regulation, a standard): only for claims that + say what a source says, never for a claim about relsynth. + +The checker fails when: + +1. a quote is no longer in its document (the claim changed: update the register); +2. a piece of evidence does not resolve (a renamed test, a removed proposition); +3. a claim about relsynth has no evidence other than ``source`` or ``code``; +4. a sentence in a document under ``scope`` reads as a claim (it matches the + trigger words) and is neither covered by a register quote nor listed under + ``not_claims`` with a reason; +5. a ``Prop. N`` reference in any Markdown document names a proposition number + that the proofs do not define. + +Proposition numbers are computed the way LaTeX numbers them: sequentially, +following every ``\\setcounter`` (including the save-and-restore pattern of +the numbered sections), over the files ``relsynth-proofs.tex`` inputs. +""" +from __future__ import annotations + +import argparse +import ast +import json +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +REGISTER = Path("docs/assurance/claims.yaml") +PROOFS = Path("docs/proofs") +EVIDENCE_KINDS = ("prop", "check", "test", "report", "code", "source") +STRONG_KINDS = ("prop", "check", "test", "report") +STATUSES = ("supported", "partial", "limitation", "scope") +TRIGGER = re.compile( + r"\b(guarantee[sd]?|prove[sn]?|proofs?|exact(?:ly)?|never|always|every|identical|byte[- ]for[- ]byte|" + r"cannot|refuse[sd]?|tamper\w*|signed|bound(?:ed|s)?|differential privacy|calibrated|masked|enforce[sd]?|" + r"no row|value for value|resolves?|holds?)\b", + re.I, +) +PROP_REF = re.compile(r"\bProp(?:osition)?s?\.?[ ~]+(\d+(?:\s*(?:[–-]|to|and|,)\s*\d+)*)") + + +# ---------------------------------------------------------------- text helpers + +def norm(text: str) -> str: + """Whitespace collapsed, Markdown emphasis and links removed, for quote matching.""" + text = re.sub(r"\[([^\]]+)\]\([^)]*\)", r"\1", text) + text = text.replace("*", "").replace("`", "") + return " ".join(text.split()) + + +def sentences(text: str) -> list[str]: + """Prose sentences and table rows of a Markdown document (code blocks, headings and comments dropped).""" + text = re.sub(r"```.*?```", "", text, flags=re.S) + text = re.sub(r"", "", text, flags=re.S) + out: list[str] = [] + for para in re.split(r"\n\s*\n", text): + lines = [ln for ln in para.split("\n") if ln.strip() and not ln.lstrip().startswith("#")] + if not lines: + continue + if all(ln.lstrip().startswith("|") for ln in lines): + out += [norm(ln) for ln in lines if not re.fullmatch(r"\s*\|[\s|:-]*\|?\s*", ln)] + continue + items: list[str] = [] + for ln in lines: # list items are separate units + if re.match(r"\s*(?:[-*]|\d+\.)\s", ln) or not items: + items.append(ln) + else: + items[-1] += " " + ln + for item in items: + out += [s for s in re.split(r"(?<=[.!?:])\s+(?=[A-Z(*`])", norm(item)) if s] + return out + + +# ---------------------------------------------------------------- evidence indices + +def _strip_comments(line: str) -> str: + return re.sub(r"(? str: + out = [] + for line in path.read_text(encoding="utf-8").splitlines(): + line = _strip_comments(line) + m = re.search(r"\\input\{([^}]+)\}", line) + if m: + name = m.group(1) if m.group(1).endswith(".tex") else m.group(1) + ".tex" + if (path.parent / name).exists(): + out.append(_expand(path.parent / name)) + continue + out.append(line) + return "\n".join(out) + + +def proposition_numbers(root: Path = ROOT) -> dict[str, int]: + """Label -> printed number of every proposition in the proofs document.""" + text = _expand(root / PROOFS / "relsynth-proofs.tex") + token = re.compile( + r"\\begin\{proposition\}|\\setcounter\{(\w+)\}\{(?:\\value\{(\w+)\}|(-?\d+))\}|\\label\{(prop:[^}]+)\}" + ) + counter, saved, labels, pending = 0, {}, {}, False + for m in token.finditer(text): + if m.group(0).startswith("\\begin"): + counter += 1 + pending = True + elif m.group(1): + value = counter if m.group(2) == "proposition" else saved.get(m.group(2), 0) if m.group(2) else int(m.group(3)) + if m.group(1) == "proposition": + counter = value + else: + saved[m.group(1)] = value + elif pending: + labels[m.group(4)] = counter + pending = False + return labels + + +def check_ids(root: Path = ROOT) -> set[str]: + ids = set(re.findall(r'@check\("(C\d+)"', (root / "legacy/checks/run_checks.py").read_text(encoding="utf-8"))) + for path in (root / "tests").rglob("test_*.py"): + ids |= {"C" + n for n in re.findall(r"^\s*def test_[cC](\d+)_", path.read_text(encoding="utf-8"), re.M)} + return ids + + +def _symbols(path: Path) -> set[str]: + tree = ast.parse(path.read_text(encoding="utf-8")) + names: set[str] = set() + + def walk(node: ast.AST, prefix: str) -> None: + for child in ast.iter_child_nodes(node): + if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)): + names.add(prefix + child.name) + walk(child, prefix + child.name + "::") + elif isinstance(child, (ast.Assign, ast.AnnAssign)) and not prefix: + targets = child.targets if isinstance(child, ast.Assign) else [child.target] + names.update(t.id for t in targets if isinstance(t, ast.Name)) + + walk(tree, "") + return names + + +# ---------------------------------------------------------------- the check + +@dataclass +class Result: + problems: list[str] = field(default_factory=list) + unclaimed: dict[str, list[str]] = field(default_factory=dict) + claims: list[dict] = field(default_factory=list) + props: dict[str, int] = field(default_factory=dict) + + +def _resolve(kind: str, value: str, root: Path, props: dict[str, int], checks: set[str], cache: dict) -> str | None: + """None when the evidence resolves, else the reason it does not.""" + if kind == "prop": + return None if value in props else f"no proposition labelled {value}" + if kind == "check": + return None if value in checks else f"no check {value}" + if kind in ("test", "code"): + file, _, symbol = value.partition("::") + path = root / file + if not path.is_file(): + return f"no file {file}" + if not symbol: + return None if kind == "code" else f"{value}: name the test (file::test)" + if path not in cache: + cache[path] = _symbols(path) + return None if symbol in cache[path] else f"{file} has no {symbol}" + if kind == "report": + return None if (root / value.split("#")[0]).exists() else f"no file {value}" + if kind == "source": + return None if value.strip() else "empty source" + return f"unknown evidence kind {kind}" + + +def run(root: Path = ROOT) -> Result: + res = Result() + reg = yaml.safe_load((root / REGISTER).read_text(encoding="utf-8")) + res.props = props = proposition_numbers(root) + numbers = set(props.values()) + checks = check_ids(root) + sym_cache: dict = {} + docs: dict[str, str] = {} + + def doc_text(name: str) -> str | None: + if name not in docs: + p = root / name + docs[name] = norm(p.read_text(encoding="utf-8")) if p.is_file() else None + return docs[name] + + seen: set[str] = set() + quotes: dict[str, list[str]] = {} + for c in reg.get("claims", []): + cid = c.get("id", "?") + where = f"{cid}" + if cid in seen: + res.problems.append(f"{where}: duplicate id") + seen.add(cid) + for key in ("doc", "quote", "status", "evidence"): + if key not in c: + res.problems.append(f"{where}: missing '{key}'") + if c.get("status") not in STATUSES: + res.problems.append(f"{where}: status must be one of {', '.join(STATUSES)}") + doc, quote = c.get("doc", ""), c.get("quote", "") + qs = quote if isinstance(quote, list) else [quote] + text = doc_text(doc) + if text is None: + res.problems.append(f"{where}: no document {doc}") + else: + for q in qs: + if norm(q) not in text: + res.problems.append(f"{where}: quote not found in {doc}: \"{q[:80]}\"") + quotes.setdefault(doc, []).extend(norm(q) for q in qs) + kinds = [] + resolved = [] + for ev in c.get("evidence") or []: + if not isinstance(ev, dict) or len(ev) != 1: + res.problems.append(f"{where}: evidence items are one-key maps, e.g. {{prop: prop:dgauss}}") + continue + (kind, value), = ev.items() + value = str(value) + kinds.append(kind) + why = _resolve(kind, value, root, props, checks, sym_cache) + if why: + res.problems.append(f"{where}: {why}") + item = {"kind": kind, "ref": value} + if kind == "prop" and value in props: + item["number"] = props[value] + resolved.append(item) + if c.get("status") in ("supported", "partial", "limitation") and not set(kinds) & set(STRONG_KINDS): + res.problems.append(f"{where}: a claim about relsynth needs a prop, check, test or report") + res.claims.append({**{k: v for k, v in c.items() if k != "evidence"}, "evidence": resolved}) + + exempt: dict[str, list[str]] = {} + for item in reg.get("not_claims", []): + if not item.get("reason"): + res.problems.append(f"not_claims in {item.get('doc')}: give a reason for \"{str(item.get('quote'))[:60]}\"") + q = norm(str(item.get("quote", ""))) + text = doc_text(item.get("doc", "")) + if text is None or q not in text: + res.problems.append(f"not_claims: quote not found in {item.get('doc')}: \"{q[:80]}\"") + exempt.setdefault(item.get("doc", ""), []).append(q) + + for doc in reg.get("scope", []): + path = root / doc + if not path.is_file(): + res.problems.append(f"scope: no document {doc}") + continue + covered = quotes.get(doc, []) + exempt.get(doc, []) + for s in sentences(path.read_text(encoding="utf-8")): + if TRIGGER.search(s) and not any(q and (q in s or s in q) for q in covered): + res.unclaimed.setdefault(doc, []).append(s) + for s in res.unclaimed.get(doc, []): + res.problems.append(f"{doc}: unregistered claim: \"{s[:110]}\"") + + for path in sorted(root.glob("*.md")) + sorted((root / "docs").rglob("*.md")): + rel = path.relative_to(root).as_posix() + if rel.startswith(("docs/research/", "CHANGELOG")): + continue + for m in PROP_REF.finditer(path.read_text(encoding="utf-8")): + for a, b in re.findall(r"(\d+)(?:\s*[–-]\s*(\d+))?", m.group(1)): + lo, hi = int(a), int(b or a) + for n in (lo, hi): + if n not in numbers: + ref = m.group(0).strip() + res.problems.append(f"{rel}: '{ref}' names Proposition {n}, which the proofs do not define") + return res + + +def suggest(res: Result, doc: str) -> str: + stubs = [ + {"id": "CL-XXX", "doc": doc, "quote": s, "status": "supported", "area": "", "evidence": [{"test": ""}]} + for s in res.unclaimed.get(doc, []) + ] + return yaml.safe_dump(stubs, sort_keys=False, allow_unicode=True, width=120) + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--suggest", metavar="DOC", help="print register stubs for the unregistered claims of DOC") + ap.add_argument("--json", metavar="FILE", help="write the resolved register as JSON") + args = ap.parse_args(argv) + res = run() + if args.suggest: + print(suggest(res, args.suggest)) + return 0 + if args.json: + Path(args.json).write_text(json.dumps({"claims": res.claims}, indent=2, ensure_ascii=False), encoding="utf-8") + for p in res.problems: + print(p) + n = len(res.claims) + if res.problems: + print(f"{len(res.problems)} problem(s) in {n} claims") + return 1 + reg = yaml.safe_load((ROOT / REGISTER).read_text(encoding="utf-8")) + print(f"ok: {n} claims traced, {len(reg.get('scope', []))} documents in scope, {len(res.props)} propositions") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 7cf6a8025c1144d918317e8de41fe55b246fd57f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 18:58:33 +0000 Subject: [PATCH 2/8] docs(assurance): register the claims of the privacy, DP, security and governance guides 60 claims of docs/privacy.md, docs/dp.md, docs/security.md and docs/governance/guide.md, each traced to propositions, checks, tests or the aggregate result files now kept in docs/assurance/evidence/. Registering them found statements in docs/security.md that 2.0 made false: 'formal differential privacy is not part of v1.0', 'privacy checks run by default' (the distance guards are not a privacy check, docs/privacy.md), five core dependencies (seven), subprocess used once (four places) and no network access (the service talks to the OIDC provider and Vault). The guide now says what the code does. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PSVqwE13trs5MdpBoE4YyJ Signed-off-by: Claude --- docs/assurance/claims.yaml | 721 ++++++++++++++++ docs/assurance/evidence/README.md | 15 + docs/assurance/evidence/T06/guards_adult.json | 681 +++++++++++++++ .../evidence/T06/repro_anonymeter.json | 342 ++++++++ docs/assurance/evidence/T06/repro_mia.json | 803 ++++++++++++++++++ .../evidence/T07/audit_contoso_eps1.jsonl | 1 + .../evidence/T07/utility_vs_epsilon.jsonl | 24 + docs/security.md | 12 +- 8 files changed, 2593 insertions(+), 6 deletions(-) create mode 100644 docs/assurance/claims.yaml create mode 100644 docs/assurance/evidence/README.md create mode 100644 docs/assurance/evidence/T06/guards_adult.json create mode 100644 docs/assurance/evidence/T06/repro_anonymeter.json create mode 100644 docs/assurance/evidence/T06/repro_mia.json create mode 100644 docs/assurance/evidence/T07/audit_contoso_eps1.jsonl create mode 100644 docs/assurance/evidence/T07/utility_vs_epsilon.jsonl diff --git a/docs/assurance/claims.yaml b/docs/assurance/claims.yaml new file mode 100644 index 0000000..5351de9 --- /dev/null +++ b/docs/assurance/claims.yaml @@ -0,0 +1,721 @@ +# Claim register: every claim of the documents in `scope` traced to its evidence. +# +# Checked by `python tools/claims_check.py` (CI job `lint`, and tests/unit/test_assurance.py). +# How to keep it green, and what each field means: docs/assurance/README.md#the-claim-register. +# +# id stable identifier, cited by the assurance case (AC-...) and the regulation mapping +# doc the document that makes the claim +# quote the claim's words (one string or a list); must still appear in `doc` +# area structure | statistics | privacy-measured | privacy-formal | governance | security | scope +# status supported the evidence establishes the claim as written +# partial the evidence establishes part of it; `note` says which part is open +# limitation the claim states a limit of relsynth; the evidence shows the limit is enforced or measured +# scope says what a source (a regulation, a paper) says, not what relsynth does +# evidence list of {prop: label} {check: Cnnn} {test: file::name} {report: path} {code: file::symbol} {source: text} +# note optional: what the evidence does not show + +scope: + - docs/privacy.md + - docs/dp.md + - docs/security.md + - docs/governance/guide.md + +claims: + + # ------------------------------------------------------------------ docs/privacy.md (measured privacy, T06) + + - id: PRV-01 + doc: docs/privacy.md + area: privacy-measured + quote: "relsynth privacy attacks a synthetic release the way a recipient could, person by person, and writes a privacy report with one page per release context (internal, partner, public)." + status: supported + evidence: + - test: tests/unit/test_privacy.py::test_assess_report_and_contexts + - test: tests/unit/test_privacy.py::test_cli_privacy_writes_report_and_gates + + - id: PRV-02 + doc: docs/privacy.md + area: privacy-measured + quote: "A data protection officer can read the pages without the annex; the annex and the JSON file hold the technical results." + status: partial + note: Readability by a DPO is a design aim; the test shows the report's structure (one page per context, annex, JSON), not that it is readable. + evidence: + - test: tests/unit/test_privacy.py::test_assess_report_and_contexts + + - id: PRV-03 + doc: docs/privacy.md + area: privacy-measured + quote: + - "The report never shows a real key." + - "People are named by keyed hashes (P-3f2a…, HMAC-SHA256 of the key) under a secret 256-bit salt that is not the split seed" + status: supported + evidence: + - test: tests/unit/test_privacy.py::test_assess_report_and_contexts + - test: tests/unit/test_privacy.py::test_cli_privacy_writes_report_and_gates + + - id: PRV-04 + doc: docs/privacy.md + area: privacy-measured + quote: "| No record isolation (sec. 3.4.1) | Can someone single out a real person?" + status: supported + evidence: + - prop: prop:puctrl + - check: C601 + - test: tests/unit/test_privacy.py::test_wilson_and_residual_match_anonymeter_formulas + - report: docs/assurance/evidence/T06/repro_anonymeter.json + - source: "EDPB Guidelines 02/2026 v1.0, sec. 3.4.1, paras. 54-57" + + - id: PRV-05 + doc: docs/privacy.md + area: privacy-measured + quote: "| No linkage (sec. 3.4.2) | Can someone connect two records about one person?" + status: supported + evidence: + - test: tests/unit/test_privacy.py::test_copy_release_is_caught_and_fresh_release_is_not + - report: docs/assurance/evidence/T06/repro_anonymeter.json + - source: "EDPB Guidelines 02/2026 v1.0, sec. 3.4.2, paras. 58-65" + + - id: PRV-06 + doc: docs/privacy.md + area: privacy-measured + quote: "| No inference (sec. 3.4.3) | Can someone learn a person's attribute?" + status: supported + evidence: + - prop: prop:puattr + - check: C606 + - test: tests/unit/test_privacy.py::test_attribute_inference_against_population + - source: "EDPB Guidelines 02/2026 v1.0, sec. 3.4.3, paras. 66-67" + + - id: PRV-07 + doc: docs/privacy.md + area: privacy-measured + quote: "| Membership | Can someone tell that a person's data was used?" + status: supported + evidence: + - prop: prop:pushadow + - prop: prop:putpr + - check: C602 + - check: C604 + - test: tests/unit/test_privacy.py::test_shadow_attack_raw_release_vs_population_generator + - test: tests/unit/test_privacy.py::test_domias_detects_overfitting + - report: docs/assurance/evidence/T06/repro_mia.json + + - id: PRV-08 + doc: docs/privacy.md + area: privacy-measured + quote: + - "Every number has a confidence interval (95% by default)." + - "Membership attacks are run on the most exposed people, chosen by the Meeus et al. (2023) outlier score, and on as many random people." + status: supported + evidence: + - prop: prop:putpr + - check: C604 + - test: tests/unit/test_privacy.py::test_clopper_pearson_edges + - test: tests/unit/test_privacy.py::test_auc_interval_never_zero_width + - test: tests/unit/test_privacy.py::test_vulnerability_ranks_outliers_first + + - id: PRV-09 + doc: docs/privacy.md + area: privacy-measured + quote: + - "The privacy unit is the table that holds the people" + - "Every table that reaches it through foreign keys belongs to it: a customer's invoices and invoice lines are attacked together with the customer." + - "Chains stop at other person tables: a customer's sales representative does not own the customer." + status: supported + evidence: + - prop: prop:pusplit + - check: C600 + - test: tests/unit/test_privacy.py::test_unit_paths_and_split_keep_people_whole + - test: tests/unit/test_privacy.py::test_unit_chain_stops_at_other_person_tables + + - id: PRV-10 + doc: docs/privacy.md + area: privacy-measured + quote: "Profiles hold no key values, because synthetic keys are new labels." + status: supported + evidence: + - prop: prop:puprofile + - check: C605 + - test: tests/unit/test_privacy.py::test_profiles_ignore_keys_and_row_order + + - id: PRV-11 + doc: docs/privacy.md + area: privacy-measured + quote: + - "Control people are never seen: the attacks' success on them is what the synthetic data teaches about anyone, and only the excess on training people is a risk to them." + - "Training and control have the same size so the record-isolation rates are exactly comparable (Prop. 601); Anonymeter's fitted size correction showed risks of 0.10–0.14 under the null in our checks." + status: supported + evidence: + - prop: prop:puctrl + - check: C601 + - report: docs/assurance/evidence/T06/repro_anonymeter.json + + - id: PRV-12 + doc: docs/privacy.md + area: privacy-measured + quote: + - "It never assumes a less motivated attacker (para. 31)." + - "Each check compares the upper confidence bound with the limit (safety margin, para. 88):" + - "pass when it is below, fail when even the lower bound is above, review otherwise." + status: supported + evidence: + - test: tests/unit/test_privacy.py::test_assess_report_and_contexts + - test: tests/unit/test_privacy.py::test_review_sign_off_per_person_and_small_population + - source: "EDPB Guidelines 02/2026 v1.0, para. 31 (motivation), para. 88 (safety margins)" + + - id: PRV-13 + doc: docs/privacy.md + area: governance + quote: "Contexts outside the organisation are blocked unless the release carries a differential-privacy guarantee: the organisation decided that DP is required for every external release, with ε set by the organisation (there is no built-in ε)." + status: supported + evidence: + - test: tests/unit/test_privacy.py::test_assess_report_and_contexts + - test: tests/unit/test_privacy.py::test_assess_with_dp_is_not_blocked + - test: tests/unit/governance/test_governance.py::test_partner_without_dp_is_refused + + - id: PRV-14 + doc: docs/privacy.md + area: governance + quote: "A named person (the DPO) signs off a review with relsynth privacy ... --signed-off-by \"Name, DPO\"; the report records the name." + status: supported + evidence: + - test: tests/unit/test_privacy.py::test_review_sign_off_per_person_and_small_population + + - id: PRV-15 + doc: docs/privacy.md + area: privacy-measured + quote: "With fewer than small_population (1000) people in the unit, the report warns, and every context outside the organisation fails without DP (check small_population)." + status: supported + evidence: + - test: tests/unit/test_privacy.py::test_review_sign_off_per_person_and_small_population + + - id: PRV-16 + doc: docs/privacy.md + area: privacy-formal + quote: + - "When the release carries a DP guarantee (release.dp: {epsilon, delta}), the page also shows the largest TPR any attack can reach at FPR 1% (Kairouz et al. 2015; Prop. 603)." + - "With cumulative_epsilon (and cumulative_delta), the budget spent by all releases from the same domain so far, it also shows the bound for all of them together, since releases about the same people compose." + status: supported + evidence: + - prop: prop:pudp + - check: C603 + - test: tests/unit/test_privacy.py::test_dp_bound + - test: tests/unit/test_privacy.py::test_assess_with_dp_is_not_blocked + + - id: PRV-17 + doc: docs/privacy.md + area: privacy-measured + quote: "One set of fits serves every target (Prop. 602)." + status: supported + evidence: + - prop: prop:pushadow + - check: C602 + - test: tests/unit/test_privacy.py::test_shadow_cache_reuses_fits + + - id: PRV-18 + doc: docs/privacy.md + area: privacy-measured + quote: + - "The scorecard's distance-to-closest-record ratio and exact-match rate stay in every run (relsynth run) for continuity, but they are not a privacy check: calibration showed the distance guard misses even 20% copied records, which DOMIAS catches at 5%." + status: supported + evidence: + - report: docs/assurance/evidence/T06/guards_adult.json + - test: tests/unit/test_privacy.py::test_copy_release_is_caught_and_fresh_release_is_not + + - id: PRV-19 + doc: docs/privacy.md + area: scope + quote: + - "Results hold for the attacks run, not for every possible attack." + - "Only differential privacy gives a guarantee against all attacks; this module measures, it does not protect." + status: limitation + evidence: + - prop: prop:pudp + - source: "NIST SP 800-226, sec. 1.1 (DP protects against all attacks, including with auxiliary data)" + + # ------------------------------------------------------------------ docs/dp.md (formal privacy, T07) + + - id: DP-01 + doc: docs/dp.md + area: privacy-formal + quote: + - "relsynth can produce a synthetic database with a formal differential privacy (DP) guarantee." + - "The guarantee is user level: anyone who sees the synthetic data learns about the same about one person (one row of the privacy unit table and every row that belongs to it) whether or not that person was in the real database." + status: supported + evidence: + - prop: prop:run + - prop: prop:bound + - prop: prop:sens + - check: C901 + - check: C902 + - test: tests/unit/dp/test_dp.py::test_end_to_end_statement_and_charges + + - id: DP-02 + doc: docs/dp.md + area: governance + quote: + - "An external release (a context with external: true in the organisation policy) must be a DP run." + - "relsynth enforces this before it reads any data" + status: supported + evidence: + - check: C906 + - test: tests/unit/governance/test_governance.py::test_partner_without_dp_is_refused + - test: tests/unit/governance/test_governance.py::test_invariants_hold_under_a_permissive_policy + + - id: DP-03 + doc: docs/dp.md + area: privacy-formal + quote: + - "relsynth has no default ε." + - "A DP run without epsilon or rho is refused, and so is a δ above 10⁻⁵." + status: supported + evidence: + - check: C906 + - test: tests/unit/dp/test_dp.py::test_config_needs_a_budget_unit_and_small_delta + + - id: DP-04 + doc: docs/dp.md + area: privacy-formal + quote: + - "Plan, before any read. relsynth finds the tables that hold data of the unit: those with a chain of foreign keys to it." + - "Every other table must be listed in public_tables, or all_unowned must be set." + - "The plan refuses three cases, before reading anything or charging the budget: a table with two chains to the unit, a self-reference on an owned table, and a foreign key between owned tables off the chain." + status: supported + evidence: + - check: C906 + - test: tests/unit/dp/test_dp.py::test_plan_traces_ownership_and_public_tables + + - id: DP-05 + doc: docs/dp.md + area: privacy-formal + quote: "Each statistic is charged to an accountant that refuses anything beyond the run's budget." + status: supported + evidence: + - prop: prop:dgauss + - check: C900 + - test: tests/science/test_dp_checks.py::test_c900_accountant_refuses_beyond_the_plan + + - id: DP-06 + doc: docs/dp.md + area: privacy-formal + quote: "This is post-processing, so it keeps the guarantee." + status: supported + evidence: + - prop: prop:post + - check: C905 + + - id: DP-07 + doc: docs/dp.md + area: privacy-formal + quote: + - "The proofs are Propositions 900 to 906 in docs/proofs." + - "C902, for example, removes one customer at a time and confirms that no released statistic moves by more than the sensitivity it was charged for." + status: supported + evidence: + - prop: prop:dgauss + - prop: prop:run + - check: C902 + - test: tests/science/test_dp_checks.py::test_c902_every_released_statistic_respects_its_sensitivity + + - id: DP-08 + doc: docs/dp.md + area: privacy-formal + quote: + - "Every DP run produces a statement (schema relsynth.dp-statement/1; write it with --dp-statement)." + - "the guarantee in ε, δ and ρ, and how it was computed (zCDP, converted with Canonne, Kamath and Steinke 2020);" + - "It holds no data and no exact counts." + status: supported + evidence: + - prop: prop:run + - test: tests/unit/dp/test_dp.py::test_end_to_end_statement_and_charges + - test: tests/unit/dp/test_dp.py::test_epsilon_is_converted_at_the_delta_left_after_level_selection + + - id: DP-09 + doc: docs/dp.md + area: governance + quote: + - "In a governed run it is signed into the manifest and goes into the release package." + - "A run refused by the per-release, period or lifetime cap reads nothing." + status: supported + evidence: + - prop: prop:reserve + - check: C802 + - test: tests/unit/governance/test_governance.py::test_external_dp_release_with_quorum_and_token + - test: tests/unit/governance/test_governance.py::test_budget_exceeded_is_refused + + - id: DP-10 + doc: docs/dp.md + area: privacy-formal + quote: "--assess runs the attack suite against the DP pipeline itself, and compares each attack's true-positive rate with the bound that ε implies." + status: supported + evidence: + - prop: prop:pudp + - test: tests/unit/test_privacy.py::test_assess_with_dp_is_not_blocked + - report: docs/assurance/evidence/T07/audit_contoso_eps1.jsonl + + - id: DP-11 + doc: docs/dp.md + area: scope + quote: "Those values are outside the guarantee unless you write the spec from public facts only and set spec_is_public: true." + status: limitation + evidence: + - test: tests/unit/test_governance_design.py::test_external_context_cannot_waive_dp_or_approval + - code: src/relsynth/dp/__init__.py + + - id: DP-12 + doc: docs/dp.md + area: privacy-formal + quote: + - "What DP mode refuses: tune, verify, export-model and doctor --apply refuse a DP spec, because they would turn real data into models or spec values outside the accounting." + - "The ordinary pipeline (relsynth.run, execute) also refuses a DP spec rather than run it without DP." + status: supported + evidence: + - test: tests/unit/dp/test_dp.py::test_ordinary_pipeline_refuses_a_dp_spec + - test: tests/unit/dp/test_dp.py::test_cli_refuses_tune_and_export_model_on_a_dp_spec + note: The CLI test covers tune and export-model; verify and doctor --apply are refused by the same guard but have no test of their own. + + - id: DP-13 + doc: docs/dp.md + area: scope + quote: "Not supported yet: in-database pushdown in DP mode (every owned table is loaded), more than one privacy unit, composite parent keys (generated as (i, 1)), and values of pii, text and binary columns (placeholders)." + status: limitation + evidence: + - test: tests/unit/dp/test_dp.py::test_plan_traces_ownership_and_public_tables + + - id: DP-14 + doc: docs/dp.md + area: privacy-formal + quote: + - "Membership-inference TPR at FPR 0.01 is 0.014, below the bound of 0.027." + - "Shadow-model AUCs are 0.45 to 0.52, and attribute inference gains nothing over the population baseline." + status: supported + evidence: + - report: docs/assurance/evidence/T07/audit_contoso_eps1.jsonl + + - id: DP-15 + doc: docs/dp.md + area: statistics + quote: "The curve is nearly flat, so the loss comes from what stage 1 measures, not from the noise." + status: supported + evidence: + - report: docs/assurance/evidence/T07/utility_vs_epsilon.jsonl + note: The cause (one-dimensional stage-1 statistics) is an inference from the flat curve; roadmap T07b tests it with two-way marginals. + + # ------------------------------------------------------------------ docs/security.md + + - id: SEC-01 + doc: docs/security.md + area: security + quote: "| Database credentials | Leaked through spec files, logs, error messages, CI output, diagnostics bundles, process lists | Credentials come from the environment, and every output masks them (see below) |" + status: supported + evidence: + - test: tests/unit/test_security.py::test_mask_url + - test: tests/unit/test_security.py::test_spec_urls_masked_everywhere + - test: tests/unit/test_security.py::test_bundle_holds_no_password + - test: tests/e2e/test_cli.py::test_connection_errors_exit_3_and_mask_passwords + + - id: SEC-02 + doc: docs/security.md + area: privacy-measured + quote: "| Source data | Synthetic rows that copy real ones; rare values that identify a person; bundles that carry rows |" + status: supported + evidence: + - test: tests/unit/test_security.py::test_synthetic_rows_are_not_copies + - test: tests/unit/test_security.py::test_privacy_checks_run_by_default + - prop: prop:guard + - test: tests/unit/test_privacy.py::test_assess_report_and_contexts + + - id: SEC-03 + doc: docs/security.md + area: security + quote: + - "| The target database | SQL injection through table or column names or values | SQLAlchemy Core with quoted identifiers and bound values everywhere |" + - "Tables are reflected by name and every statement is built with SQLAlchemy Core." + - "Identifiers are quoted by the dialect, and values are bound parameters." + - "Rule expressions are never translated into SQL text; they are evaluated in Python on loaded rows." + status: supported + evidence: + - test: tests/unit/test_connectors.py::test_hostile_table_and_column_names_are_quoted + - test: tests/integration/test_servers.py::test_hostile_identifiers_on_server + + - id: SEC-04 + doc: docs/security.md + area: security + quote: + - "Masked everywhere. mask_url replaces:" + - "It is applied to every error message the CLI prints (_scrub), the JSON output, saved specs, diagnostics bundles (again when they are written, whoever built them), exported models and ping results." + status: supported + evidence: + - test: tests/unit/test_security.py::test_mask_url + - test: tests/unit/test_security.py::test_scrub_masks_urls_inside_messages + - test: tests/unit/test_security.py::test_spec_urls_masked_everywhere + - test: tests/unit/test_security.py::test_bundle_holds_no_password + - test: tests/unit/test_security.py::test_failed_connection_logs_no_password + - test: tests/unit/test_spec.py::test_save_masks_passwords + + - id: SEC-05 + doc: docs/security.md + area: security + quote: + - "YAML is read with a safe loader that also refuses anchors and aliases." + - "Python tags (!!python/object) cannot construct objects, and an alias bomb (a few lines that expand to billions of nodes) is refused before it expands." + status: supported + evidence: + - test: tests/unit/test_spec.py::test_yaml_aliases_are_refused + - test: tests/unit/test_spec.py::test_yaml_python_tags_are_refused + - test: tests/unit/test_security.py::test_spec_cannot_construct_python_objects + + - id: SEC-06 + doc: docs/security.md + area: security + quote: + - "Rule expressions are parsed with Python's ast module and evaluated by relsynth's own interpreter, never by eval:" + - "attribute access, subscripts, lambdas, comprehensions, f-strings, assignment expressions and keyword arguments are refused;" + - "expressions longer than 10,000 characters or nested more than 100 levels are refused." + status: supported + evidence: + - test: tests/unit/test_security.py::test_rule_expressions_are_sandboxed + - test: tests/unit/test_security.py::test_expression_size_and_depth_limits + - test: tests/unit/test_security.py::test_allowed_expressions_still_work + + - id: SEC-07 + doc: docs/security.md + area: security + quote: "Patterns that can backtrack catastrophically are refused, such as nested repetition (a+)+ or repeated alternatives that can match the same text (a|aa)+, and so are patterns longer than 500 characters." + status: supported + evidence: + - test: tests/unit/test_security.py::test_catastrophic_regexes_are_refused + - test: tests/unit/test_security.py::test_ordinary_regexes_are_accepted + + - id: SEC-08 + doc: docs/security.md + area: security + quote: "Unknown keys are errors (with a \"did you mean\" suggestion), so a misspelt privacy setting cannot silently turn a protection off." + status: supported + evidence: + - test: tests/unit/test_settings.py::test_validation_errors + - test: tests/unit/test_privacy.py::test_config_rejects_unknown_keys + - test: tests/unit/test_spec.py::test_invalid_settings_are_refused + + - id: SEC-09 + doc: docs/security.md + area: security + quote: + - ".., ., empty names, /, \\, NUL and absolute paths are refused;" + - "so is any name whose file resolves outside the folder, through a symbolic link for example." + status: supported + evidence: + - test: tests/unit/test_security.py::test_inside_refuses_non_plain_names + - test: tests/unit/test_security.py::test_symlinked_table_cannot_escape + - test: tests/unit/test_connectors.py::test_file_table_names_cannot_escape_the_folder + + - id: SEC-10 + doc: docs/security.md + area: privacy-measured + quote: + - "Diagnostics bundles, verify reports and profile output hold counts and statistics, not rows." + - "Rows of pushed facts never leave the source as a whole; only aggregates and a sample do." + status: partial + note: The bundle test checks that no password is in it; there is no test that a bundle holds no row. Pushdown reads are aggregate queries plus a seeded sample (Props. 100 and 103). + evidence: + - test: tests/unit/test_security.py::test_bundle_holds_no_password + - prop: prop:pdsuff + - prop: prop:hashsample + - test: tests/unit/test_pushdown.py::test_full_sample_pushdown_equals_in_memory + + - id: SEC-11 + doc: docs/security.md + area: privacy-formal + quote: + - "Differential privacy is optional." + - "Without it, protection is measured, not guaranteed: relsynth privacy attacks the release per person (privacy.md)." + - "With --epsilon, a run has a user-level (ε, δ) guarantee (dp.md)." + - "Under the governance policy, a release outside the organisation is refused without DP (governance/guide.md)." + status: supported + evidence: + - prop: prop:run + - test: tests/unit/test_privacy.py::test_assess_report_and_contexts + - test: tests/unit/governance/test_governance.py::test_partner_without_dp_is_refused + + - id: SEC-12 + doc: docs/security.md + area: security + quote: + - "The core package depends on seven well-known libraries (numpy, pandas, scipy, SQLAlchemy, PyYAML, DuckDB and its SQLAlchemy dialect) and the database driver you choose; every other dependency comes with an extra you install." + - "Versions have lower bounds only, so you can patch them yourself." + status: supported + evidence: + - report: pyproject.toml + + - id: SEC-13 + doc: docs/security.md + area: security + quote: + - "CI runs on every change:" + - "Every suppressed finding is annotated in the code with its test ID and a reason, for example # nosec B608 where a query is built with SQLAlchemy and not by string formatting." + status: supported + evidence: + - report: .github/workflows/ci.yml + - report: pyproject.toml + + - id: SEC-14 + doc: docs/security.md + area: security + quote: + - "The package and the CLI connect only to the databases, warehouses and storage you name, and send no telemetry." + - "The service also contacts the OIDC identity provider and the Vault server you configure, and exports traces only when you turn on OpenTelemetry (relsynth[service-otel]) and give it an endpoint." + status: supported + evidence: + - test: tests/unit/test_assurance.py::test_network_access_is_confined_to_named_modules + note: The test checks which modules may open connections; third-party drivers (database and warehouse clients) are outside it. + + - id: SEC-15 + doc: docs/security.md + area: security + quote: "subprocess runs fixed argument lists, never a shell, in four places: pdflatex (when present) for the exported model's PDF, the OPA binary when the policy engine is opa, the GPU listing commands of relsynth compute, and the service worker's job runner." + status: supported + evidence: + - report: pyproject.toml + - test: tests/unit/governance/test_policy_parity.py::test_engine_failure_denies + note: Enforced by bandit B602-B607 in CI (no shell=True accepted); the four call sites are annotated with nosec B404 and a reason. + + # ------------------------------------------------------------------ docs/governance/guide.md (T08) + + - id: GOV-01 + doc: docs/governance/guide.md + area: governance + quote: + - "relsynth can record every run in a registry: a folder with signed run manifests, the organisation's privacy budget, approvals and releases, kept in a hash chain so that any later change is detected." + status: supported + evidence: + - prop: prop:govchain + - check: C803 + - test: tests/unit/governance/test_governance.py::test_tampered_registry_is_detected + - test: tests/unit/governance/test_governance.py::test_tampered_report_or_manifest_object_is_detected + + - id: GOV-02 + doc: docs/governance/guide.md + area: governance + quote: + - "Every person or service that signs gets an Ed25519 key." + - "Private keys are written with mode 0600." + status: supported + evidence: + - test: tests/unit/governance/test_governance.py::test_key_file_roundtrip_and_mode + - test: tests/unit/governance/test_governance.py::test_dsse_signature_binds_payload_type_and_bytes + + - id: GOV-03 + doc: docs/governance/guide.md + area: governance + quote: + - "Until then, gov policy check refuses the policy with a sentence that says what is missing." + - "Only a trusted key with the privacy owner's role can sign it." + - "Every manifest records the digest of the policy it was judged under." + status: supported + evidence: + - test: tests/unit/governance/test_governance.py::test_policy_without_budget_is_refused_with_guidance + - test: tests/unit/governance/test_governance.py::test_policy_must_be_signed_by_privacy_owner + - test: tests/unit/test_governance_design.py::test_budget_has_no_default_and_bounded_delta + - test: tests/unit/governance/test_governance.py::test_cli_end_to_end + + - id: GOV-04 + doc: docs/governance/guide.md + area: governance + quote: "Four invariants hold whatever the policy says: an external release needs DP, an external release needs an approval by someone other than the operator, a fail or blocked privacy verdict is never released, and no domain may exceed its lifetime budget." + status: supported + evidence: + - test: tests/unit/governance/test_governance.py::test_invariants_hold_under_a_permissive_policy + - test: tests/unit/governance/test_governance.py::test_failed_verdict_quality_and_target_are_refused + - test: tests/unit/test_governance_design.py::test_external_context_cannot_waive_dp_or_approval + - test: tests/unit/governance/test_policy_parity.py::test_builtin_engine_matches_fixture + + - id: GOV-05 + doc: docs/governance/guide.md + area: governance + quote: + - "A context outside the organisation is refused unless the run is differentially private (--epsilon, --dp or a dp section; see dp.md)." + - "Without either, the verdict is missing and the run cannot be released." + - "context \"partner\" is outside the organisation: differential privacy is required, and this run has none" + status: supported + evidence: + - test: tests/unit/governance/test_governance.py::test_partner_without_dp_is_refused + - test: tests/unit/governance/test_governance.py::test_failed_verdict_quality_and_target_are_refused + + - id: GOV-06 + doc: docs/governance/guide.md + area: governance + quote: + - "The run writes its synthetic data as usual and then records a signed manifest: hashes of the outputs and the spec, versions, seeds, the verdicts, the quality score and what the policy still needs before a release." + - "The manifest holds no data, no row counts and no credentials." + status: supported + evidence: + - test: tests/unit/governance/test_governance.py::test_manifest_holds_no_data_or_credentials + - test: tests/unit/test_governance_design.py::test_example_policy_and_manifest_validate + + - id: GOV-07 + doc: docs/governance/guide.md + area: governance + quote: + - "On allow it signs a release token and writes the package: the synthetic tables byte for byte as signed, the manifest, the approvals, the token, a checkpoint, and the DP statement for DP runs." + - "The privacy report never goes into a package: it was computed on real data." + status: supported + evidence: + - test: tests/unit/governance/test_governance.py::test_external_dp_release_with_quorum_and_token + - test: tests/unit/governance/test_governance.py::test_privacy_report_cannot_leave_the_organisation + + - id: GOV-08 + doc: docs/governance/guide.md + area: governance + quote: + - "Revoking never refunds privacy budget, because the data has already left." + - "Budget is reserved before a DP run reads data and refunded only if it never read any (Proposition 802)." + status: supported + evidence: + - prop: prop:reserve + - check: C802 + - test: tests/unit/governance/test_governance.py::test_ledger_reserve_touch_settle_refund + - test: tests/unit/governance/test_governance.py::test_external_dp_release_with_quorum_and_token + - test: tests/unit/governance/test_governance.py::test_rejection_and_revocation + + - id: GOV-09 + doc: docs/governance/guide.md + area: governance + quote: "A registry check catches edited, deleted, inserted, reordered and truncated log entries and any changed report or manifest (Proposition 803, check C803)." + status: supported + evidence: + - prop: prop:govchain + - check: C803 + - test: tests/unit/governance/test_governance.py::test_tampered_registry_is_detected + - test: tests/unit/governance/test_governance.py::test_resigned_manifest_by_untrusted_key_is_detected + + - id: GOV-10 + doc: docs/governance/guide.md + area: privacy-formal + quote: + - "Values below 1 give strong protection; deployments use 1 < ε ≤ 20, and the risk above 1 should be measured, for example with relsynth privacy." + - "δ should be at most 1/n² for n people and never above 10⁻⁵ (the policy schema refuses more)." + - "The lifetime cap never resets; a period cap only spreads it over time (Proposition 801)." + status: supported + evidence: + - prop: prop:period + - prop: prop:ledger + - check: C800 + - check: C801 + - test: tests/unit/governance/test_governance.py::test_policy_refuses_external_without_dp_and_large_delta + - test: tests/unit/governance/test_governance.py::test_ledger_lifetime_cap_under_concurrency + - source: "NIST SP 800-226, sec. 2.2 (no rule for choosing epsilon)" + note: The ε ranges are the deployment practice NIST SP 800-226 reports, not something relsynth proves. + + - id: GOV-11 + doc: docs/governance/guide.md + area: governance + quote: "each DP run's state (reserved, touched, settled, refunded) and the number of releases made without DP." + status: supported + evidence: + - test: tests/unit/governance/test_governance.py::test_nondp_runs_are_recorded + - test: tests/unit/governance/test_governance.py::test_ledger_reserve_touch_settle_refund + +not_claims: + - {doc: docs/privacy.md, quote: "Every key is optional.", reason: "describes the configuration file format"} + - {doc: docs/privacy.md, quote: "Viswanath, The composition theorem for differential privacy, ICML 2015.", reason: "bibliography entry"} + - {doc: docs/security.md, quote: "Pass secrets as masked CI variables (see cicd.md), and use read-only database accounts for sources.", reason: "advice to the operator"} diff --git a/docs/assurance/evidence/README.md b/docs/assurance/evidence/README.md new file mode 100644 index 0000000..ae3cde0 --- /dev/null +++ b/docs/assurance/evidence/README.md @@ -0,0 +1,15 @@ +# Evidence files + +Aggregate results that claims in the [claim register](../claims.yaml) cite as `report` evidence. They hold scores, rates and intervals only: no rows, no keys and no person labels. Each was copied unchanged from the roadmap report named below, which also has the scripts and settings that produced it. + +| File | What it measures | Produced by | +|---|---|---| +| `T06/repro_anonymeter.json` | Singling-out, linkage and inference risks reproduced against Anonymeter, and the bias of its size correction under the null | Roadmap T06, `benchmarks/privacy/reproduce_anonymeter.py`, 2026-10-27 | +| `T06/repro_mia.json` | Shadow-model and DOMIAS membership inference on a raw release and on a population generator | Roadmap T06, `benchmarks/privacy/reproduce_mia.py`, 2026-10-27 | +| `T06/guards_adult.json` | Which planted leaks (copied records, 1% to 20%) the distance guard, the exact-match rate, singling out and DOMIAS detect, on Adult | Roadmap T06, `benchmarks/privacy/calibrate_guards.py`, 2026-10-27 | +| `T07/utility_vs_epsilon.jsonl` | Share of scorecard checks passing for DP runs at ε = 0.5 to 8 on Chinook, Contoso and AdventureWorks DW | Roadmap T07, 2026-10-31 | +| `T07/audit_contoso_eps1.jsonl` | The attack suite against the DP pipeline on Contoso at ε = 1, with the TPR bound ε implies | Roadmap T07, 2026-10-31 | + +Datasets: UCI Adult (CC BY 4.0), Chinook (MIT), Contoso Data Generator V2 (MIT, SQLBI) and AdventureWorks DW (MIT, Microsoft sample code), all marked allowed in [docs/datasets.md](../../datasets.md). + +A new file belongs here only when a documented claim needs it. Re-running a benchmark replaces the file and may change the documents that quote its numbers; `tools/claims_check.py` then points at every claim whose quote no longer matches. diff --git a/docs/assurance/evidence/T06/guards_adult.json b/docs/assurance/evidence/T06/guards_adult.json new file mode 100644 index 0000000..d1e4b95 --- /dev/null +++ b/docs/assurance/evidence/T06/guards_adult.json @@ -0,0 +1,681 @@ +{ + "table": "adult", + "rows": 20000, + "fit_seconds": 14.987272024154663, + "results": [ + { + "near_copies": false, + "share": 0.0, + "rep": 0, + "dcr_ratio": 1.458016682664604, + "dcr_flag": false, + "exact_match_rate": 0.0, + "exact_flag": false, + "singling_out_risk": 0.015815231947675864, + "singling_out_ci": [ + 0.0, + 0.08736016077030015 + ], + "singling_out_flag": false, + "domias_auc": 0.50697646875, + "domias_auc_ci": [ + 0.498030090522928, + 0.515922846977072 + ], + "domias_flag": false, + "domias_tpr_at_fpr_0.01": 0.00675 + }, + { + "near_copies": false, + "share": 0.0, + "rep": 1, + "dcr_ratio": 1.458016682664604, + "dcr_flag": false, + "exact_match_rate": 0.0, + "exact_flag": false, + "singling_out_risk": 0.0, + "singling_out_ci": [ + 0.0, + 0.06252083332314186 + ], + "singling_out_flag": false, + "domias_auc": 0.50697646875, + "domias_auc_ci": [ + 0.498030090522928, + 0.515922846977072 + ], + "domias_flag": false, + "domias_tpr_at_fpr_0.01": 0.01125 + }, + { + "near_copies": false, + "share": 0.001, + "rep": 0, + "dcr_ratio": 1.458016682664604, + "dcr_flag": false, + "exact_match_rate": 0.0, + "exact_flag": false, + "singling_out_risk": 0.015815231947675864, + "singling_out_ci": [ + 0.0, + 0.08736016077030015 + ], + "singling_out_flag": false, + "domias_auc": 0.5070519375, + "domias_auc_ci": [ + 0.4981055435521842, + 0.5159983314478157 + ], + "domias_flag": false, + "domias_tpr_at_fpr_0.01": 0.00675 + }, + { + "near_copies": false, + "share": 0.001, + "rep": 1, + "dcr_ratio": 1.4586390883177716, + "dcr_flag": false, + "exact_match_rate": 0.0, + "exact_flag": false, + "singling_out_risk": 0.0, + "singling_out_ci": [ + 0.0, + 0.06252083332314186 + ], + "singling_out_flag": false, + "domias_auc": 0.50737284375, + "domias_auc_ci": [ + 0.4984265274572664, + 0.5163191600427336 + ], + "domias_flag": false, + "domias_tpr_at_fpr_0.01": 0.01125 + }, + { + "near_copies": false, + "share": 0.005, + "rep": 0, + "dcr_ratio": 1.4439231457478077, + "dcr_flag": false, + "exact_match_rate": 0.0006666666666666666, + "exact_flag": false, + "singling_out_risk": 0.011861423960756888, + "singling_out_ci": [ + 0.0, + 0.08327074680530952 + ], + "singling_out_flag": false, + "domias_auc": 0.50730725, + "domias_auc_ci": [ + 0.49836098722063205, + 0.5162535127793679 + ], + "domias_flag": false, + "domias_tpr_at_fpr_0.01": 0.00675 + }, + { + "near_copies": false, + "share": 0.005, + "rep": 1, + "dcr_ratio": 1.4345144876467364, + "dcr_flag": false, + "exact_match_rate": 0.0013333333333333333, + "exact_flag": false, + "singling_out_risk": 0.0, + "singling_out_ci": [ + 0.0, + 0.06932991125603416 + ], + "singling_out_flag": false, + "domias_auc": 0.5072405, + "domias_auc_ci": [ + 0.4982939933851982, + 0.5161870066148018 + ], + "domias_flag": false, + "domias_tpr_at_fpr_0.01": 0.01175 + }, + { + "near_copies": false, + "share": 0.02, + "rep": 0, + "dcr_ratio": 1.4491092460074713, + "dcr_flag": false, + "exact_match_rate": 0.004, + "exact_flag": false, + "singling_out_risk": 0.023353506794762977, + "singling_out_ci": [ + 0.0, + 0.09188035018861543 + ], + "singling_out_flag": false, + "domias_auc": 0.5089065625000001, + "domias_auc_ci": [ + 0.4999607371690943, + 0.5178523878309059 + ], + "domias_flag": false, + "domias_tpr_at_fpr_0.01": 0.00725 + }, + { + "near_copies": false, + "share": 0.02, + "rep": 1, + "dcr_ratio": 1.4211486019983015, + "dcr_flag": false, + "exact_match_rate": 0.002, + "exact_flag": false, + "singling_out_risk": 0.0, + "singling_out_ci": [ + 0.0, + 0.0759555112188903 + ], + "singling_out_flag": false, + "domias_auc": 0.5076562968749999, + "domias_auc_ci": [ + 0.49871015852544565, + 0.5166024352245542 + ], + "domias_flag": false, + "domias_tpr_at_fpr_0.01": 0.0115 + }, + { + "near_copies": false, + "share": 0.05, + "rep": 0, + "dcr_ratio": 1.272847035575486, + "dcr_flag": false, + "exact_match_rate": 0.01, + "exact_flag": false, + "singling_out_risk": 0.03163046389535169, + "singling_out_ci": [ + 0.0, + 0.10369295717797652 + ], + "singling_out_flag": false, + "domias_auc": 0.509692515625, + "domias_auc_ci": [ + 0.5007474769900064, + 0.5186375542599935 + ], + "domias_flag": true, + "domias_tpr_at_fpr_0.01": 0.00725 + }, + { + "near_copies": false, + "share": 0.05, + "rep": 1, + "dcr_ratio": 1.3009108147330073, + "dcr_flag": false, + "exact_match_rate": 0.008666666666666666, + "exact_flag": false, + "singling_out_risk": 0.004001268731726907, + "singling_out_ci": [ + 0.0, + 0.07764962573395676 + ], + "singling_out_flag": false, + "domias_auc": 0.509413453125, + "domias_auc_ci": [ + 0.5004679212706831, + 0.5183589849793169 + ], + "domias_flag": true, + "domias_tpr_at_fpr_0.01": 0.01125 + }, + { + "near_copies": false, + "share": 0.2, + "rep": 0, + "dcr_ratio": 0.8964874127906735, + "dcr_flag": false, + "exact_match_rate": 0.030666666666666665, + "exact_flag": true, + "singling_out_risk": 0.08470449425486222, + "singling_out_ci": [ + 0.007605080506312228, + 0.1618039080034122 + ], + "singling_out_flag": true, + "domias_auc": 0.518996796875, + "domias_auc_ci": [ + 0.5100579739549046, + 0.5279356197950954 + ], + "domias_flag": true, + "domias_tpr_at_fpr_0.01": 0.009 + }, + { + "near_copies": false, + "share": 0.2, + "rep": 1, + "dcr_ratio": 0.5048269201857659, + "dcr_flag": false, + "exact_match_rate": 0.038, + "exact_flag": true, + "singling_out_risk": 0.0987976615180447, + "singling_out_ci": [ + 0.018005154914113128, + 0.17959016812197626 + ], + "singling_out_flag": true, + "domias_auc": 0.5195425312499999, + "domias_auc_ci": [ + 0.5106041321932232, + 0.5284809303067767 + ], + "domias_flag": true, + "domias_tpr_at_fpr_0.01": 0.01575 + }, + { + "near_copies": true, + "share": 0.0, + "rep": 0, + "dcr_ratio": 1.458016682664604, + "dcr_flag": false, + "exact_match_rate": 0.0, + "exact_flag": false, + "singling_out_risk": 0.015815231947675864, + "singling_out_ci": [ + 0.0, + 0.08736016077030015 + ], + "singling_out_flag": false, + "domias_auc": 0.50697646875, + "domias_auc_ci": [ + 0.498030090522928, + 0.515922846977072 + ], + "domias_flag": false, + "domias_tpr_at_fpr_0.01": 0.00675 + }, + { + "near_copies": true, + "share": 0.0, + "rep": 1, + "dcr_ratio": 1.458016682664604, + "dcr_flag": false, + "exact_match_rate": 0.0, + "exact_flag": false, + "singling_out_risk": 0.0, + "singling_out_ci": [ + 0.0, + 0.06252083332314186 + ], + "singling_out_flag": false, + "domias_auc": 0.50697646875, + "domias_auc_ci": [ + 0.498030090522928, + 0.515922846977072 + ], + "domias_flag": false, + "domias_tpr_at_fpr_0.01": 0.01125 + }, + { + "near_copies": true, + "share": 0.001, + "rep": 0, + "dcr_ratio": 1.458016682664604, + "dcr_flag": false, + "exact_match_rate": 0.0, + "exact_flag": false, + "singling_out_risk": 0.015815231947675864, + "singling_out_ci": [ + 0.0, + 0.08736016077030015 + ], + "singling_out_flag": false, + "domias_auc": 0.5070519375, + "domias_auc_ci": [ + 0.4981055435521842, + 0.5159983314478157 + ], + "domias_flag": false, + "domias_tpr_at_fpr_0.01": 0.00675 + }, + { + "near_copies": true, + "share": 0.001, + "rep": 1, + "dcr_ratio": 1.4586390883177716, + "dcr_flag": false, + "exact_match_rate": 0.0, + "exact_flag": false, + "singling_out_risk": 0.0, + "singling_out_ci": [ + 0.0, + 0.06252083332314186 + ], + "singling_out_flag": false, + "domias_auc": 0.50737284375, + "domias_auc_ci": [ + 0.4984265274572664, + 0.5163191600427336 + ], + "domias_flag": false, + "domias_tpr_at_fpr_0.01": 0.01125 + }, + { + "near_copies": true, + "share": 0.005, + "rep": 0, + "dcr_ratio": 1.4426467770348794, + "dcr_flag": false, + "exact_match_rate": 0.0, + "exact_flag": false, + "singling_out_risk": 0.015815231947675864, + "singling_out_ci": [ + 0.0, + 0.08736016077030015 + ], + "singling_out_flag": false, + "domias_auc": 0.50731596875, + "domias_auc_ci": [ + 0.4983697060277778, + 0.5162622314722222 + ], + "domias_flag": false, + "domias_tpr_at_fpr_0.01": 0.00675 + }, + { + "near_copies": true, + "share": 0.005, + "rep": 1, + "dcr_ratio": 1.4402564485094576, + "dcr_flag": false, + "exact_match_rate": 0.0, + "exact_flag": false, + "singling_out_risk": 0.0, + "singling_out_ci": [ + 0.0, + 0.06593874855806459 + ], + "singling_out_flag": false, + "domias_auc": 0.507319390625, + "domias_auc_ci": [ + 0.49837288354607157, + 0.5162658977039284 + ], + "domias_flag": false, + "domias_tpr_at_fpr_0.01": 0.0115 + }, + { + "near_copies": true, + "share": 0.02, + "rep": 0, + "dcr_ratio": 1.3820018858516343, + "dcr_flag": false, + "exact_match_rate": 0.0, + "exact_flag": false, + "singling_out_risk": 0.035167139530808654, + "singling_out_ci": [ + 0.0, + 0.10495039286801258 + ], + "singling_out_flag": false, + "domias_auc": 0.508442171875, + "domias_auc_ci": [ + 0.49949623800666765, + 0.5173881057433323 + ], + "domias_flag": false, + "domias_tpr_at_fpr_0.01": 0.007 + }, + { + "near_copies": true, + "share": 0.02, + "rep": 1, + "dcr_ratio": 1.4263172041846213, + "dcr_flag": false, + "exact_match_rate": 0.0, + "exact_flag": false, + "singling_out_risk": 0.0, + "singling_out_ci": [ + 0.0, + 0.07514340915457575 + ], + "singling_out_flag": false, + "domias_auc": 0.507620015625, + "domias_auc_ci": [ + 0.49867377155063713, + 0.5165662596993629 + ], + "domias_flag": false, + "domias_tpr_at_fpr_0.01": 0.0115 + }, + { + "near_copies": true, + "share": 0.05, + "rep": 0, + "dcr_ratio": 1.3894370806892347, + "dcr_flag": false, + "exact_match_rate": 0.0, + "exact_flag": false, + "singling_out_risk": 0.020167736727348177, + "singling_out_ci": [ + 0.0, + 0.0959094829741555 + ], + "singling_out_flag": false, + "domias_auc": 0.50919071875, + "domias_auc_ci": [ + 0.5002456109383747, + 0.5181358265616253 + ], + "domias_flag": true, + "domias_tpr_at_fpr_0.01": 0.007 + }, + { + "near_copies": true, + "share": 0.05, + "rep": 1, + "dcr_ratio": 1.3152320529927175, + "dcr_flag": false, + "exact_match_rate": 0.0, + "exact_flag": false, + "singling_out_risk": 0.004099692561667058, + "singling_out_ci": [ + 0.0, + 0.08271112001894992 + ], + "singling_out_flag": false, + "domias_auc": 0.508243953125, + "domias_auc_ci": [ + 0.49929813197706757, + 0.5171897742729324 + ], + "domias_flag": false, + "domias_tpr_at_fpr_0.01": 0.01175 + }, + { + "near_copies": true, + "share": 0.2, + "rep": 0, + "dcr_ratio": 0.9023093514969838, + "dcr_flag": false, + "exact_match_rate": 0.0, + "exact_flag": false, + "singling_out_risk": 0.03720226878422708, + "singling_out_ci": [ + 0.0, + 0.11800635624612321 + ], + "singling_out_flag": false, + "domias_auc": 0.51744021875, + "domias_auc_ci": [ + 0.5084997961613955, + 0.5263806413386045 + ], + "domias_flag": true, + "domias_tpr_at_fpr_0.01": 0.0105 + }, + { + "near_copies": true, + "share": 0.2, + "rep": 1, + "dcr_ratio": 0.9327289170999513, + "dcr_flag": false, + "exact_match_rate": 0.0, + "exact_flag": false, + "singling_out_risk": 0.09093758369913614, + "singling_out_ci": [ + 0.017276861559173476, + 0.1645983058390988 + ], + "singling_out_flag": true, + "domias_auc": 0.5183050156249999, + "domias_auc_ci": [ + 0.5093657868524074, + 0.5272442443975923 + ], + "domias_flag": true, + "domias_tpr_at_fpr_0.01": 0.017 + } + ], + "summary": [ + { + "near_copies": false, + "share": 0.0, + "dcr_flag": 0.0, + "exact_flag": 0.0, + "singling_out_flag": 0.0, + "domias_flag": 0.0, + "dcr_ratio": 1.4580166827, + "exact_match_rate": 0.0, + "singling_out_risk": 0.007907616, + "domias_auc": 0.5069764688 + }, + { + "near_copies": false, + "share": 0.001, + "dcr_flag": 0.0, + "exact_flag": 0.0, + "singling_out_flag": 0.0, + "domias_flag": 0.0, + "dcr_ratio": 1.4583278855, + "exact_match_rate": 0.0, + "singling_out_risk": 0.007907616, + "domias_auc": 0.5072123906 + }, + { + "near_copies": false, + "share": 0.005, + "dcr_flag": 0.0, + "exact_flag": 0.0, + "singling_out_flag": 0.0, + "domias_flag": 0.0, + "dcr_ratio": 1.4392188167, + "exact_match_rate": 0.001, + "singling_out_risk": 0.005930712, + "domias_auc": 0.507273875 + }, + { + "near_copies": false, + "share": 0.02, + "dcr_flag": 0.0, + "exact_flag": 0.0, + "singling_out_flag": 0.0, + "domias_flag": 0.0, + "dcr_ratio": 1.435128924, + "exact_match_rate": 0.003, + "singling_out_risk": 0.0116767534, + "domias_auc": 0.5082814297 + }, + { + "near_copies": false, + "share": 0.05, + "dcr_flag": 0.0, + "exact_flag": 0.0, + "singling_out_flag": 0.0, + "domias_flag": 1.0, + "dcr_ratio": 1.2868789252, + "exact_match_rate": 0.0093333333, + "singling_out_risk": 0.0178158663, + "domias_auc": 0.5095529844 + }, + { + "near_copies": false, + "share": 0.2, + "dcr_flag": 0.0, + "exact_flag": 1.0, + "singling_out_flag": 1.0, + "domias_flag": 1.0, + "dcr_ratio": 0.7006571665, + "exact_match_rate": 0.0343333333, + "singling_out_risk": 0.0917510779, + "domias_auc": 0.5192696641 + }, + { + "near_copies": true, + "share": 0.0, + "dcr_flag": 0.0, + "exact_flag": 0.0, + "singling_out_flag": 0.0, + "domias_flag": 0.0, + "dcr_ratio": 1.4580166827, + "exact_match_rate": 0.0, + "singling_out_risk": 0.007907616, + "domias_auc": 0.5069764688 + }, + { + "near_copies": true, + "share": 0.001, + "dcr_flag": 0.0, + "exact_flag": 0.0, + "singling_out_flag": 0.0, + "domias_flag": 0.0, + "dcr_ratio": 1.4583278855, + "exact_match_rate": 0.0, + "singling_out_risk": 0.007907616, + "domias_auc": 0.5072123906 + }, + { + "near_copies": true, + "share": 0.005, + "dcr_flag": 0.0, + "exact_flag": 0.0, + "singling_out_flag": 0.0, + "domias_flag": 0.0, + "dcr_ratio": 1.4414516128, + "exact_match_rate": 0.0, + "singling_out_risk": 0.007907616, + "domias_auc": 0.5073176797 + }, + { + "near_copies": true, + "share": 0.02, + "dcr_flag": 0.0, + "exact_flag": 0.0, + "singling_out_flag": 0.0, + "domias_flag": 0.0, + "dcr_ratio": 1.404159545, + "exact_match_rate": 0.0, + "singling_out_risk": 0.0175835698, + "domias_auc": 0.5080310938 + }, + { + "near_copies": true, + "share": 0.05, + "dcr_flag": 0.0, + "exact_flag": 0.0, + "singling_out_flag": 0.0, + "domias_flag": 0.5, + "dcr_ratio": 1.3523345668, + "exact_match_rate": 0.0, + "singling_out_risk": 0.0121337146, + "domias_auc": 0.5087173359 + }, + { + "near_copies": true, + "share": 0.2, + "dcr_flag": 0.0, + "exact_flag": 0.0, + "singling_out_flag": 0.5, + "domias_flag": 1.0, + "dcr_ratio": 0.9175191343, + "exact_match_rate": 0.0, + "singling_out_risk": 0.0640699262, + "domias_auc": 0.5178726172 + } + ] +} \ No newline at end of file diff --git a/docs/assurance/evidence/T06/repro_anonymeter.json b/docs/assurance/evidence/T06/repro_anonymeter.json new file mode 100644 index 0000000..c3a5254 --- /dev/null +++ b/docs/assurance/evidence/T06/repro_anonymeter.json @@ -0,0 +1,342 @@ +{ + "summary": { + "singling_out_univariate": { + "published": { + "value": 0.0137, + "ci": [ + 0.0, + 0.0341 + ] + }, + "relsynth_mean": 0.009668674140865141, + "relsynth_ci_overlaps_published": "5/5", + "anonymeter_mean": 0.03686067021069149, + "relsynth_ci_overlaps_anonymeter_same_seed": "3/5" + }, + "singling_out_multivariate": { + "published": { + "value": 0.0288, + "ci": [ + 0.0, + 0.138 + ] + }, + "relsynth_mean": 0.048880009372219695, + "relsynth_ci_overlaps_published": "5/5", + "anonymeter_mean": 0.10414837000812119, + "relsynth_ci_overlaps_anonymeter_same_seed": "5/5" + }, + "linkability_k10": { + "published": { + "value": 0.0, + "ci": [ + 0.0, + 0.0019 + ], + "attack_rate": 0.0035, + "control_rate": 0.0054 + }, + "relsynth_mean": 0.0011024493550853022, + "relsynth_ci_overlaps_published": "5/5", + "anonymeter_mean": 0.000300814362795576, + "relsynth_ci_overlaps_anonymeter_same_seed": "5/5" + } + }, + "runs": [ + { + "seed": 0, + "relsynth": { + "singling_out_univariate": { + "value": 0.009441539061459312, + "ci": [ + 0.0, + 0.033530286610146745 + ], + "s": 2.0752360820770264 + }, + "singling_out_multivariate": { + "value": 0.07277143084200198, + "ci": [ + 0.0, + 0.16404059966730997 + ], + "s": 12.237275123596191 + }, + "linkability_k10": { + "value": 0.0030061314117282474, + "ci": [ + 0.0, + 0.007374480191384514 + ], + "attack_rate": 0.00694702135693895, + "control_rate": 0.003952772498782709, + "s": 25.93265986442566 + } + }, + "anonymeter": { + "singling_out_univariate": { + "value": 0.032068775412498585, + "ci": [ + 0.01325659785162905, + 0.050880952973368115 + ], + "s": 14.00034236907959 + }, + "singling_out_multivariate": { + "value": 0.11119807881360078, + "ci": [ + 0.03180044363743173, + 0.19059571398976982 + ], + "s": 1.890343427658081 + }, + "linkability_k10": { + "value": 0.0, + "ci": [ + 0.0, + 0.002290138019968494 + ], + "attack_rate": 0.003453731022423335, + "control_rate": 0.004950855451501456, + "s": 28.90398359298706 + } + } + }, + { + "seed": 1, + "relsynth": { + "singling_out_univariate": { + "value": 0.0, + "ci": [ + 0.0, + 0.006597064188267984 + ], + "s": 1.7388732433319092 + }, + "singling_out_multivariate": { + "value": 0.0, + "ci": [ + 0.0, + 0.06614283861808506 + ], + "s": 15.231893301010132 + }, + "linkability_k10": { + "value": 0.0, + "ci": [ + 0.0, + 0.0037690121101455485 + ], + "attack_rate": 0.004950855451501456, + "control_rate": 0.005449896927860829, + "s": 18.942060470581055 + } + }, + "anonymeter": { + "singling_out_univariate": { + "value": 0.0335217411796439, + "ci": [ + 0.015028517395419033, + 0.05201496496386876 + ], + "s": 2.8062517642974854 + }, + "singling_out_multivariate": { + "value": 0.08778748086131655, + "ci": [ + 0.01585809205622679, + 0.1597168696664063 + ], + "s": 2.7506775856018066 + }, + "linkability_k10": { + "value": 0.0010020438039094154, + "ci": [ + 0.0, + 0.004912369170687757 + ], + "attack_rate": 0.004950855451501456, + "control_rate": 0.003952772498782709, + "s": 20.2576687335968 + } + } + }, + { + "seed": 2, + "relsynth": { + "singling_out_univariate": { + "value": 0.009405637532047288, + "ci": [ + 0.0, + 0.03350110764272561 + ], + "s": 1.4357383251190186 + }, + "singling_out_multivariate": { + "value": 0.0, + "ci": [ + 0.0, + 0.10376417162933807 + ], + "s": 15.614069700241089 + }, + "linkability_k10": { + "value": 0.0, + "ci": [ + 0.0, + 0.0037690121101455485 + ], + "attack_rate": 0.004950855451501456, + "control_rate": 0.005449896927860829, + "s": 17.767802953720093 + } + }, + "anonymeter": { + "singling_out_univariate": { + "value": 0.045620808010567754, + "ci": [ + 0.02430078784915649, + 0.06694082817197902 + ], + "s": 2.9771311283111572 + }, + "singling_out_multivariate": { + "value": 0.0872471841020539, + "ci": [ + 0.005151744959691826, + 0.16934262324441596 + ], + "s": 3.046260356903076 + }, + "linkability_k10": { + "value": 0.0005020280100684648, + "ci": [ + 0.0, + 0.005200282987184901 + ], + "attack_rate": 0.006447979880579577, + "control_rate": 0.005948938404220202, + "s": 20.054949522018433 + } + } + }, + { + "seed": 3, + "relsynth": { + "singling_out_univariate": { + "value": 0.0, + "ci": [ + 0.0, + 0.0 + ], + "s": 1.9466679096221924 + }, + "singling_out_multivariate": { + "value": 0.04773646603211855, + "ci": [ + 0.0, + 0.14670518317226477 + ], + "s": 15.729119062423706 + }, + "linkability_k10": { + "value": 0.0015045733542712989, + "ci": [ + 0.0, + 0.0059877546980347945 + ], + "attack_rate": 0.006447979880579577, + "control_rate": 0.004950855451501456, + "s": 15.887316465377808 + } + }, + "anonymeter": { + "singling_out_univariate": { + "value": 0.02708502573473792, + "ci": [ + 0.008816963822701487, + 0.04535308764677435 + ], + "s": 3.973442554473877 + }, + "singling_out_multivariate": { + "value": 0.1044073109768376, + "ci": [ + 0.022902773718366853, + 0.18591184823530835 + ], + "s": 3.513134002685547 + }, + "linkability_k10": { + "value": 0.0, + "ci": [ + 0.0, + 0.0 + ], + "attack_rate": 0.002455648069704588, + "control_rate": 0.00694702135693895, + "s": 19.968678951263428 + } + } + }, + { + "seed": 4, + "relsynth": { + "singling_out_univariate": { + "value": 0.02949619411081911, + "ci": [ + 0.006471003225552628, + 0.05252138499608559 + ], + "s": 1.2783808708190918 + }, + "singling_out_multivariate": { + "value": 0.12389214998697795, + "ci": [ + 0.04371884545607026, + 0.20406545451788566 + ], + "s": 15.461420774459839 + }, + "linkability_k10": { + "value": 0.0010015420094269653, + "ci": [ + 0.0, + 0.004657102783944402 + ], + "attack_rate": 0.004451813975142082, + "control_rate": 0.003453731022423335, + "s": 15.681336164474487 + } + }, + "anonymeter": { + "singling_out_univariate": { + "value": 0.046007000716009265, + "ci": [ + 0.025485826027770622, + 0.06652817540424791 + ], + "s": 1.7887303829193115 + }, + "singling_out_multivariate": { + "value": 0.13010179528679708, + "ci": [ + 0.051836375017679306, + 0.20836721555591486 + ], + "s": 1.8787341117858887 + }, + "linkability_k10": { + "value": 0.0, + "ci": [ + 0.0, + 0.0005353934078742351 + ], + "attack_rate": 0.0029546895460639613, + "control_rate": 0.006447979880579577, + "s": 13.1881422996521 + } + } + } + ] +} \ No newline at end of file diff --git a/docs/assurance/evidence/T06/repro_mia.json b/docs/assurance/evidence/T06/repro_mia.json new file mode 100644 index 0000000..5de1cd4 --- /dev/null +++ b/docs/assurance/evidence/T06/repro_mia.json @@ -0,0 +1,803 @@ +{ + "raw_release": { + "pooled": { + "classifier": { + "auc": 0.9991759122193906, + "auc_ci": [ + 0.99666937035169, + 1.0 + ], + "tpr_at_fpr": [ + { + "fpr_target": 0.1, + "identified": true, + "threshold": -1.2112150326749518, + "tpr": 1.0, + "tpr_ci": [ + 0.9722333948899635, + 1.0 + ], + "fpr": 0.08, + "fpr_ci": [ + 0.039029918964379615, + 0.14220310282821583 + ], + "n_members": 131, + "n_non_members": 125 + }, + { + "fpr_target": 0.01, + "identified": true, + "threshold": 0.11098656240664218, + "tpr": 0.9770992366412213, + "tpr_ci": [ + 0.9345334122372114, + 0.9952521819713963 + ], + "fpr": 0.016, + "fpr_ci": [ + 0.0019435818023689144, + 0.05660211961975487 + ], + "n_members": 131, + "n_non_members": 125 + }, + { + "fpr_target": 0.001, + "identified": false, + "reason": "needs at least 1000 calibration non-members, has 134" + } + ], + "mean_auc_vulnerable": 1.0, + "mean_auc_random": 0.9934612171454277 + }, + "lira": { + "auc": 1.0, + "auc_ci": [ + 0.9997588511379375, + 1.0 + ], + "tpr_at_fpr": [ + { + "fpr_target": 0.1, + "identified": true, + "threshold": -2.9668408199996972e+16, + "tpr": 1.0, + "tpr_ci": [ + 0.9722333948899635, + 1.0 + ], + "fpr": 0.136, + "fpr_ci": [ + 0.08126488247065346, + 0.20879827174915336 + ], + "n_members": 131, + "n_non_members": 125 + }, + { + "fpr_target": 0.01, + "identified": true, + "threshold": -4741775848828320.0, + "tpr": 1.0, + "tpr_ci": [ + 0.9722333948899635, + 1.0 + ], + "fpr": 0.008, + "fpr_ci": [ + 0.0002025219535342447, + 0.04376653637502473 + ], + "n_members": 131, + "n_non_members": 125 + }, + { + "fpr_target": 0.001, + "identified": false, + "reason": "needs at least 1000 calibration non-members, has 134" + } + ], + "mean_auc_vulnerable": 1.0, + "mean_auc_random": 1.0 + } + }, + "seconds": 30.484925508499146, + "targets": [ + { + "kind": "vulnerable", + "vulnerability": 0.43342925, + "classifier_auc": 1.0, + "classifier_auc_ci": [ + 0.980113456, + 1.0 + ], + "lira_auc": 1.0 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3453256514, + "classifier_auc": 1.0, + "classifier_auc_ci": [ + 0.9849263561, + 1.0 + ], + "lira_auc": 1.0 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3448868303, + "classifier_auc": 1.0, + "classifier_auc_ci": [ + 0.9846193699, + 1.0 + ], + "lira_auc": 1.0 + }, + { + "kind": "vulnerable", + "vulnerability": 0.331607959, + "classifier_auc": 1.0, + "classifier_auc_ci": [ + 0.9846193699, + 1.0 + ], + "lira_auc": 1.0 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3291765998, + "classifier_auc": 1.0, + "classifier_auc_ci": [ + 0.9852547964, + 1.0 + ], + "lira_auc": 1.0 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3218197399, + "classifier_auc": 1.0, + "classifier_auc_ci": [ + 0.9842116397, + 1.0 + ], + "lira_auc": 1.0 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3193475163, + "classifier_auc": 1.0, + "classifier_auc_ci": [ + 0.9842116397, + 1.0 + ], + "lira_auc": 1.0 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3160277824, + "classifier_auc": 1.0, + "classifier_auc_ci": [ + 0.9836935634, + 1.0 + ], + "lira_auc": 1.0 + }, + { + "kind": "random", + "vulnerability": 0.0003784228, + "classifier_auc": 1.0, + "classifier_auc_ci": [ + 0.9830504492, + 1.0 + ], + "lira_auc": 1.0 + }, + { + "kind": "random", + "vulnerability": 0.0839446355, + "classifier_auc": 1.0, + "classifier_auc_ci": [ + 0.9836935634, + 1.0 + ], + "lira_auc": 1.0 + }, + { + "kind": "random", + "vulnerability": 0.0022037225, + "classifier_auc": 1.0, + "classifier_auc_ci": [ + 0.9836935634, + 1.0 + ], + "lira_auc": 1.0 + }, + { + "kind": "random", + "vulnerability": 0.0868692697, + "classifier_auc": 1.0, + "classifier_auc_ci": [ + 0.9846193699, + 1.0 + ], + "lira_auc": 1.0 + }, + { + "kind": "random", + "vulnerability": 0.0259393387, + "classifier_auc": 0.963562753, + "classifier_auc_ci": [ + 0.8887324745, + 1.0 + ], + "lira_auc": 1.0 + }, + { + "kind": "random", + "vulnerability": 0.0613782024, + "classifier_auc": 0.9841269841, + "classifier_auc_ci": [ + 0.9362273945, + 1.0 + ], + "lira_auc": 1.0 + }, + { + "kind": "random", + "vulnerability": 0.0638613695, + "classifier_auc": 1.0, + "classifier_auc_ci": [ + 0.9836935634, + 1.0 + ], + "lira_auc": 1.0 + }, + { + "kind": "random", + "vulnerability": 0.1063787166, + "classifier_auc": 1.0, + "classifier_auc_ci": [ + 0.9852547964, + 1.0 + ], + "lira_auc": 1.0 + } + ] + }, + "population_marginals": { + "pooled": { + "classifier": { + "auc": 0.5188395623178231, + "auc_ci": [ + 0.46872879166045267, + 0.5689503329751936 + ], + "tpr_at_fpr": [ + { + "fpr_target": 0.1, + "identified": true, + "threshold": 2.20991270006166, + "tpr": 0.0916030534351145, + "tpr_ci": [ + 0.048232606292222584, + 0.15454525591786367 + ], + "fpr": 0.104, + "fpr_ci": [ + 0.05654885749481344, + 0.17127618765792438 + ], + "n_members": 131, + "n_non_members": 125 + }, + { + "fpr_target": 0.01, + "identified": true, + "threshold": 4.386018017607288, + "tpr": 0.007633587786259542, + "tpr_ci": [ + 0.00019324703518952029, + 0.04179686306423274 + ], + "fpr": 0.016, + "fpr_ci": [ + 0.0019435818023689144, + 0.05660211961975487 + ], + "n_members": 131, + "n_non_members": 125 + }, + { + "fpr_target": 0.001, + "identified": false, + "reason": "needs at least 1000 calibration non-members, has 134" + } + ], + "mean_auc_vulnerable": 0.5520616070765427, + "mean_auc_random": 0.4866884582780481 + }, + "lira": { + "auc": 0.5421429334472814, + "auc_ci": [ + 0.49214519743486046, + 0.5921406694597022 + ], + "tpr_at_fpr": [ + { + "fpr_target": 0.1, + "identified": true, + "threshold": 0.5618522521772137, + "tpr": 0.08396946564885496, + "tpr_ci": [ + 0.04266381778741492, + 0.14527746052856297 + ], + "fpr": 0.136, + "fpr_ci": [ + 0.08126488247065346, + 0.20879827174915336 + ], + "n_members": 131, + "n_non_members": 125 + }, + { + "fpr_target": 0.01, + "identified": true, + "threshold": 1.135342736759033, + "tpr": 0.030534351145038167, + "tpr_ci": [ + 0.008381012720473995, + 0.07633430782864756 + ], + "fpr": 0.048, + "fpr_ci": [ + 0.017816322500502527, + 0.10154868108862163 + ], + "n_members": 131, + "n_non_members": 125 + }, + { + "fpr_target": 0.001, + "identified": false, + "reason": "needs at least 1000 calibration non-members, has 134" + } + ], + "mean_auc_vulnerable": 0.5174770452688652, + "mean_auc_random": 0.5735635397883849 + } + }, + "seconds": 21.80527138710022, + "targets": [ + { + "kind": "vulnerable", + "vulnerability": 0.43342925, + "classifier_auc": 0.4978354978, + "classifier_auc_ci": [ + 0.2800580488, + 0.7156129469 + ], + "lira_auc": 0.5194805195 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3453256514, + "classifier_auc": 0.496031746, + "classifier_auc_ci": [ + 0.2878888033, + 0.7041746888 + ], + "lira_auc": 0.4603174603 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3448868303, + "classifier_auc": 0.5254901961, + "classifier_auc_ci": [ + 0.3136429373, + 0.7373374548 + ], + "lira_auc": 0.568627451 + }, + { + "kind": "vulnerable", + "vulnerability": 0.331607959, + "classifier_auc": 0.5803921569, + "classifier_auc_ci": [ + 0.3719611429, + 0.7888231708 + ], + "lira_auc": 0.4980392157 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3291765998, + "classifier_auc": 0.475, + "classifier_auc_ci": [ + 0.2594458818, + 0.6905541182 + ], + "lira_auc": 0.5958333333 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3218197399, + "classifier_auc": 0.62890625, + "classifier_auc_ci": [ + 0.4295139593, + 0.8282985407 + ], + "lira_auc": 0.5078125 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3193475163, + "classifier_auc": 0.7265625, + "classifier_auc_ci": [ + 0.542900964, + 0.910224036 + ], + "lira_auc": 0.625 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3160277824, + "classifier_auc": 0.4862745098, + "classifier_auc_ci": [ + 0.2778460265, + 0.6947029931 + ], + "lira_auc": 0.3647058824 + }, + { + "kind": "random", + "vulnerability": 0.0003784228, + "classifier_auc": 0.6587301587, + "classifier_auc_ci": [ + 0.4640569045, + 0.853403413 + ], + "lira_auc": 0.5119047619 + }, + { + "kind": "random", + "vulnerability": 0.0839446355, + "classifier_auc": 0.4392156863, + "classifier_auc_ci": [ + 0.2323035469, + 0.6461278256 + ], + "lira_auc": 0.5098039216 + }, + { + "kind": "random", + "vulnerability": 0.0022037225, + "classifier_auc": 0.4196078431, + "classifier_auc_ci": [ + 0.2101367881, + 0.6290788982 + ], + "lira_auc": 0.6196078431 + }, + { + "kind": "random", + "vulnerability": 0.0868692697, + "classifier_auc": 0.6156862745, + "classifier_auc_ci": [ + 0.4131932841, + 0.8181792649 + ], + "lira_auc": 0.5098039216 + }, + { + "kind": "random", + "vulnerability": 0.0259393387, + "classifier_auc": 0.5263157895, + "classifier_auc_ci": [ + 0.3063329665, + 0.7462986124 + ], + "lira_auc": 0.6761133603 + }, + { + "kind": "random", + "vulnerability": 0.0613782024, + "classifier_auc": 0.3650793651, + "classifier_auc_ci": [ + 0.1608218281, + 0.569336902 + ], + "lira_auc": 0.6666666667 + }, + { + "kind": "random", + "vulnerability": 0.0638613695, + "classifier_auc": 0.4980392157, + "classifier_auc_ci": [ + 0.2874305483, + 0.708647883 + ], + "lira_auc": 0.6196078431 + }, + { + "kind": "random", + "vulnerability": 0.1063787166, + "classifier_auc": 0.3708333333, + "classifier_auc_ci": [ + 0.1576055915, + 0.5840610752 + ], + "lira_auc": 0.475 + } + ] + }, + "relsynth": { + "pooled": { + "classifier": { + "auc": 0.5596013856883423, + "auc_ci": [ + 0.5098937038662978, + 0.6093090675103867 + ], + "tpr_at_fpr": [ + { + "fpr_target": 0.1, + "identified": true, + "threshold": 2.6063852139943546, + "tpr": 0.1450381679389313, + "tpr_ci": [ + 0.08963828061992946, + 0.21719465157818257 + ], + "fpr": 0.096, + "fpr_ci": [ + 0.05059487235144333, + 0.16168861021681072 + ], + "n_members": 131, + "n_non_members": 125 + }, + { + "fpr_target": 0.01, + "identified": true, + "threshold": 5.815694092479607, + "tpr": 0.015267175572519083, + "tpr_ci": [ + 0.001854303171805998, + 0.054061380891561546 + ], + "fpr": 0.016, + "fpr_ci": [ + 0.0019435818023689144, + 0.05660211961975487 + ], + "n_members": 131, + "n_non_members": 125 + }, + { + "fpr_target": 0.001, + "identified": false, + "reason": "needs at least 1000 calibration non-members, has 134" + } + ], + "mean_auc_vulnerable": 0.6299340320882141, + "mean_auc_random": 0.5027984979416869 + }, + "lira": { + "auc": 0.5259969173012652, + "auc_ci": [ + 0.4758126850708006, + 0.5761811495317298 + ], + "tpr_at_fpr": [ + { + "fpr_target": 0.1, + "identified": true, + "threshold": 0.7686906405436926, + "tpr": 0.16793893129770993, + "tpr_ci": [ + 0.10834144127007268, + 0.2431481762266256 + ], + "fpr": 0.104, + "fpr_ci": [ + 0.05654885749481344, + 0.17127618765792438 + ], + "n_members": 131, + "n_non_members": 125 + }, + { + "fpr_target": 0.01, + "identified": true, + "threshold": 2.407364492432068, + "tpr": 0.030534351145038167, + "tpr_ci": [ + 0.008381012720473995, + 0.07633430782864756 + ], + "fpr": 0.008, + "fpr_ci": [ + 0.0002025219535342447, + 0.04376653637502473 + ], + "n_members": 131, + "n_non_members": 125 + }, + { + "fpr_target": 0.001, + "identified": false, + "reason": "needs at least 1000 calibration non-members, has 134" + } + ], + "mean_auc_vulnerable": 0.6120990713993931, + "mean_auc_random": 0.4158300575628362 + } + }, + "seconds": 234.679758310318, + "targets": [ + { + "kind": "vulnerable", + "vulnerability": 0.43342925, + "classifier_auc": 0.5930735931, + "classifier_auc_ci": [ + 0.3797583997, + 0.8063887865 + ], + "lira_auc": 0.4761904762 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3453256514, + "classifier_auc": 0.7420634921, + "classifier_auc_ci": [ + 0.5706598402, + 0.9134671439 + ], + "lira_auc": 0.4920634921 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3448868303, + "classifier_auc": 0.5921568627, + "classifier_auc_ci": [ + 0.3887593939, + 0.7955543316 + ], + "lira_auc": 0.6039215686 + }, + { + "kind": "vulnerable", + "vulnerability": 0.331607959, + "classifier_auc": 0.7058823529, + "classifier_auc_ci": [ + 0.5154357856, + 0.8963289203 + ], + "lira_auc": 0.6862745098 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3291765998, + "classifier_auc": 0.7791666667, + "classifier_auc_ci": [ + 0.6064452123, + 0.951888121 + ], + "lira_auc": 0.675 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3218197399, + "classifier_auc": 0.55859375, + "classifier_auc_ci": [ + 0.3429323257, + 0.7742551743 + ], + "lira_auc": 0.6953125 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3193475163, + "classifier_auc": 0.5234375, + "classifier_auc_ci": [ + 0.3119292526, + 0.7349457474 + ], + "lira_auc": 0.65234375 + }, + { + "kind": "vulnerable", + "vulnerability": 0.3160277824, + "classifier_auc": 0.5450980392, + "classifier_auc_ci": [ + 0.3356295023, + 0.7545665761 + ], + "lira_auc": 0.6156862745 + }, + { + "kind": "random", + "vulnerability": 0.0003784228, + "classifier_auc": 0.5357142857, + "classifier_auc_ci": [ + 0.327232337, + 0.7441962344 + ], + "lira_auc": 0.4007936508 + }, + { + "kind": "random", + "vulnerability": 0.0839446355, + "classifier_auc": 0.4352941176, + "classifier_auc_ci": [ + 0.2283360969, + 0.6422521384 + ], + "lira_auc": 0.4078431373 + }, + { + "kind": "random", + "vulnerability": 0.0022037225, + "classifier_auc": 0.4274509804, + "classifier_auc_ci": [ + 0.2192959922, + 0.6356059686 + ], + "lira_auc": 0.3764705882 + }, + { + "kind": "random", + "vulnerability": 0.0868692697, + "classifier_auc": 0.7019607843, + "classifier_auc_ci": [ + 0.5031560303, + 0.9007655384 + ], + "lira_auc": 0.4941176471 + }, + { + "kind": "random", + "vulnerability": 0.0259393387, + "classifier_auc": 0.4412955466, + "classifier_auc_ci": [ + 0.2354843371, + 0.647106756 + ], + "lira_auc": 0.4291497976 + }, + { + "kind": "random", + "vulnerability": 0.0613782024, + "classifier_auc": 0.3571428571, + "classifier_auc_ci": [ + 0.1500770948, + 0.5642086195 + ], + "lira_auc": 0.3650793651 + }, + { + "kind": "random", + "vulnerability": 0.0638613695, + "classifier_auc": 0.6235294118, + "classifier_auc_ci": [ + 0.4196656839, + 0.8273931396 + ], + "lira_auc": 0.4156862745 + }, + { + "kind": "random", + "vulnerability": 0.1063787166, + "classifier_auc": 0.5, + "classifier_auc_ci": [ + 0.2885953354, + 0.7114046646 + ], + "lira_auc": 0.4375 + } + ] + } +} \ No newline at end of file diff --git a/docs/assurance/evidence/T07/audit_contoso_eps1.jsonl b/docs/assurance/evidence/T07/audit_contoso_eps1.jsonl new file mode 100644 index 0000000..796aeb4 --- /dev/null +++ b/docs/assurance/evidence/T07/audit_contoso_eps1.jsonl @@ -0,0 +1 @@ +{"eps": 1.0, "secs": 2821, "release": {"dp": {"epsilon": 1.0, "delta": 1e-09, "unit": "Customer"}}, "contexts": {"internal": {"verdict": "review", "bound": {"dp_bound_tpr_at_fpr_0.01": 0.027182819284590452}, "checks": [{"attack": "singling_out_univariate", "value": 0.022078918783467693, "limit": 0.1, "status": "pass"}, {"attack": "singling_out_multivariate", "value": 0.014896129414813325, "limit": 0.1, "status": "pass"}, {"attack": "nearest_neighbour_linkage", "value": 0.0, "limit": 0.1, "status": "pass"}, {"attack": "nearest_neighbour_inference:Orders.OrderDate.mean", "value": 0.07960321152197222, "limit": 0.1, "status": "review"}, {"attack": "nearest_neighbour_inference:Orders.DeliveryDate.mean", "value": 0.07182335770901495, "limit": 0.1, "status": "review"}, {"attack": "nearest_neighbour_inference:Orders.Store.StoreCode.mean", "value": 0.06596765989032324, "limit": 0.1, "status": "review"}, {"attack": "nearest_neighbour_inference:Orders.Store.GeoAreaKey.mean", "value": 0.06596765989032324, "limit": 0.1, "status": "review"}, {"attack": "nearest_neighbour_inference:Orders.Store.CountryCode.mode", "value": 0.06691595276538304, "limit": 0.1, "status": "review"}, {"attack": "attribute_inference_vs_population:Orders.OrderDate.mean", "value": -0.000375, "limit": 0.05, "status": "pass"}, {"attack": "attribute_inference_vs_population:Orders.DeliveryDate.mean", "value": -0.00025, "limit": 0.05, "status": "pass"}, {"attack": "attribute_inference_vs_population:Orders.Store.StoreCode.mean", "value": 0.0017500000000000016, "limit": 0.05, "status": "pass"}, {"attack": "attribute_inference_vs_population:Orders.Store.GeoAreaKey.mean", "value": 0.0022499999999999985, "limit": 0.05, "status": "pass"}, {"attack": "attribute_inference_vs_population:Orders.Store.CountryCode.mode", "value": -0.000125, "limit": 0.05, "status": "pass"}, {"attack": "domias_auc", "value": 0.497808328125, "limit": 0.65, "status": "pass"}, {"attack": "domias_tpr_at_fpr_0.01", "value": 0.01375, "limit": 0.1, "status": "pass"}, {"attack": "shadow_classifier_auc", "value": 0.5171428571428571, "limit": 0.65, "status": "pass"}, {"attack": "shadow_lira_auc", "value": 0.4487301587301588, "limit": 0.65, "status": "pass"}, {"attack": "most_vulnerable_record_auc", "value": 0.7666666666666668, "limit": 0.85, "status": "info"}, {"attack": "shadow_classifier_vulnerable_auc", "value": 0.48721227621483365, "limit": 0.65, "status": "pass"}, {"attack": "shadow_lira_vulnerable_auc", "value": 0.47570332480818406, "limit": 0.65, "status": "pass"}]}, "partner": {"verdict": "review", "bound": {"dp_bound_tpr_at_fpr_0.01": 0.027182819284590452}, "checks": [{"attack": "singling_out_univariate", "value": 0.022078918783467693, "limit": 0.05, "status": "review"}, {"attack": "singling_out_multivariate", "value": 0.014896129414813325, "limit": 0.05, "status": "pass"}, {"attack": "nearest_neighbour_linkage", "value": 0.0, "limit": 0.05, "status": "pass"}, {"attack": "nearest_neighbour_inference:Orders.OrderDate.mean", "value": 0.08537960994313942, "limit": 0.05, "status": "review"}, {"attack": "nearest_neighbour_inference:Orders.DeliveryDate.mean", "value": 0.05556182965669362, "limit": 0.05, "status": "review"}, {"attack": "nearest_neighbour_inference:Orders.Store.StoreCode.mean", "value": 0.053182002612564054, "limit": 0.05, "status": "review"}, {"attack": "nearest_neighbour_inference:Orders.Store.GeoAreaKey.mean", "value": 0.054429106509128085, "limit": 0.05, "status": "review"}, {"attack": "nearest_neighbour_inference:Orders.Store.CountryCode.mode", "value": 0.09627709075335278, "limit": 0.05, "status": "review"}, {"attack": "attribute_inference_vs_population:Orders.OrderDate.mean", "value": -0.002625000000000016, "limit": 0.03, "status": "pass"}, {"attack": "attribute_inference_vs_population:Orders.DeliveryDate.mean", "value": 0.0017500000000000016, "limit": 0.03, "status": "pass"}, {"attack": "attribute_inference_vs_population:Orders.Store.StoreCode.mean", "value": -0.0005000000000000004, "limit": 0.03, "status": "pass"}, {"attack": "attribute_inference_vs_population:Orders.Store.GeoAreaKey.mean", "value": -0.002124999999999988, "limit": 0.03, "status": "pass"}, {"attack": "attribute_inference_vs_population:Orders.Store.CountryCode.mode", "value": -0.0018750000000000155, "limit": 0.03, "status": "pass"}, {"attack": "domias_auc", "value": 0.497808328125, "limit": 0.6, "status": "pass"}, {"attack": "domias_tpr_at_fpr_0.01", "value": 0.01375, "limit": 0.05, "status": "pass"}, {"attack": "shadow_classifier_auc", "value": 0.5171428571428571, "limit": 0.6, "status": "review"}, {"attack": "shadow_lira_auc", "value": 0.4487301587301588, "limit": 0.6, "status": "pass"}, {"attack": "most_vulnerable_record_auc", "value": 0.7666666666666668, "limit": 0.75, "status": "review"}]}, "public": {"verdict": "review", "bound": {"dp_bound_tpr_at_fpr_0.01": 0.027182819284590452}, "checks": [{"attack": "singling_out_univariate", "value": 0.022078918783467693, "limit": 0.02, "status": "review"}, {"attack": "singling_out_multivariate", "value": 0.014896129414813325, "limit": 0.02, "status": "review"}, {"attack": "nearest_neighbour_linkage", "value": 0.0, "limit": 0.02, "status": "pass"}, {"attack": "nearest_neighbour_inference:Orders.OrderDate.mean", "value": 0.08537960994313942, "limit": 0.02, "status": "review"}, {"attack": "nearest_neighbour_inference:Orders.DeliveryDate.mean", "value": 0.05556182965669362, "limit": 0.02, "status": "review"}, {"attack": "nearest_neighbour_inference:Orders.Store.StoreCode.mean", "value": 0.053182002612564054, "limit": 0.02, "status": "review"}, {"attack": "nearest_neighbour_inference:Orders.Store.GeoAreaKey.mean", "value": 0.054429106509128085, "limit": 0.02, "status": "review"}, {"attack": "nearest_neighbour_inference:Orders.Store.CountryCode.mode", "value": 0.09627709075335278, "limit": 0.02, "status": "review"}, {"attack": "attribute_inference_vs_population:Orders.OrderDate.mean", "value": -0.002625000000000016, "limit": 0.02, "status": "pass"}, {"attack": "attribute_inference_vs_population:Orders.DeliveryDate.mean", "value": 0.0017500000000000016, "limit": 0.02, "status": "pass"}, {"attack": "attribute_inference_vs_population:Orders.Store.StoreCode.mean", "value": -0.0005000000000000004, "limit": 0.02, "status": "pass"}, {"attack": "attribute_inference_vs_population:Orders.Store.GeoAreaKey.mean", "value": -0.002124999999999988, "limit": 0.02, "status": "pass"}, {"attack": "attribute_inference_vs_population:Orders.Store.CountryCode.mode", "value": -0.0018750000000000155, "limit": 0.02, "status": "pass"}, {"attack": "domias_auc", "value": 0.497808328125, "limit": 0.55, "status": "pass"}, {"attack": "domias_tpr_at_fpr_0.01", "value": 0.01375, "limit": 0.02, "status": "pass"}, {"attack": "shadow_classifier_auc", "value": 0.5171428571428571, "limit": 0.55, "status": "review"}, {"attack": "shadow_lira_auc", "value": 0.4487301587301588, "limit": 0.55, "status": "pass"}, {"attack": "most_vulnerable_record_auc", "value": 0.7666666666666668, "limit": 0.65, "status": "review"}]}}} diff --git a/docs/assurance/evidence/T07/utility_vs_epsilon.jsonl b/docs/assurance/evidence/T07/utility_vs_epsilon.jsonl new file mode 100644 index 0000000..e262fcf --- /dev/null +++ b/docs/assurance/evidence/T07/utility_vs_epsilon.jsonl @@ -0,0 +1,24 @@ +{"db": "chinook", "eps": null, "passed": 138, "total": 140, "score": 0.9857142857142858, "secs": 16.9} +{"db": "chinook", "eps": 0.5, "rep": 0, "passed": 124, "total": 140, "score": 0.8857142857142857, "eps_stated": 0.49999999974072723, "rho": 0.0037963229466495864, "secs": 18.4, "caps": {"Customer": {"Invoice": 8}, "Invoice": {"InvoiceLine": 8}, "InvoiceLine": null}} +{"db": "chinook", "eps": 0.5, "rep": 1, "passed": 123, "total": 140, "score": 0.8785714285714286, "eps_stated": 0.49999999974072723, "rho": 0.0037963229466495864, "secs": 17.1, "caps": {"Customer": {"Invoice": 8}, "Invoice": {"InvoiceLine": 8}, "InvoiceLine": null}} +{"db": "chinook", "eps": 0.5, "rep": 2, "passed": 129, "total": 140, "score": 0.9214285714285714, "eps_stated": 0.49999999974072723, "rho": 0.0037963229466495864, "secs": 15.2, "caps": {"Customer": {"Invoice": 8}, "Invoice": {"InvoiceLine": 8}, "InvoiceLine": null}} +{"db": "chinook", "eps": 1, "rep": 0, "passed": 123, "total": 140, "score": 0.8785714285714286, "eps_stated": 0.9999999994787825, "rho": 0.014407578572581993, "secs": 16.4, "caps": {"Customer": {"Invoice": 8}, "Invoice": {"InvoiceLine": 8}, "InvoiceLine": null}} +{"db": "chinook", "eps": 1, "rep": 1, "passed": 125, "total": 140, "score": 0.8928571428571429, "eps_stated": 0.9999999994787825, "rho": 0.014407578572581993, "secs": 22.2, "caps": {"Customer": {"Invoice": 8}, "Invoice": {"InvoiceLine": 8}, "InvoiceLine": null}} +{"db": "chinook", "eps": 1, "rep": 2, "passed": 128, "total": 140, "score": 0.9142857142857143, "eps_stated": 0.9999999994787825, "rho": 0.014407578572581993, "secs": 16.0, "caps": {"Customer": {"Invoice": 8}, "Invoice": {"InvoiceLine": 8}, "InvoiceLine": null}} +{"db": "chinook", "eps": 2, "rep": 0, "passed": 123, "total": 140, "score": 0.8785714285714286, "eps_stated": 1.999999998946572, "rho": 0.054131659186149955, "secs": 23.3, "caps": {"Customer": {"Invoice": 7}, "Invoice": {"InvoiceLine": 8}, "InvoiceLine": null}} +{"db": "chinook", "eps": 2, "rep": 1, "passed": 119, "total": 140, "score": 0.85, "eps_stated": 1.999999998946572, "rho": 0.054131659186149955, "secs": 16.7, "caps": {"Customer": {"Invoice": 7}, "Invoice": {"InvoiceLine": 8}, "InvoiceLine": null}} +{"db": "chinook", "eps": 2, "rep": 2, "passed": 119, "total": 140, "score": 0.85, "eps_stated": 1.999999998946572, "rho": 0.054131659186149955, "secs": 19.0, "caps": {"Customer": {"Invoice": 7}, "Invoice": {"InvoiceLine": 8}, "InvoiceLine": null}} +{"db": "chinook", "eps": 4, "rep": 0, "passed": 119, "total": 140, "score": 0.85, "eps_stated": 3.9999999978516465, "rho": 0.1995042791607679, "secs": 25.4, "caps": {"Customer": {"Invoice": 7}, "Invoice": {"InvoiceLine": 8}, "InvoiceLine": null}} +{"db": "chinook", "eps": 4, "rep": 1, "passed": 122, "total": 140, "score": 0.8714285714285714, "eps_stated": 3.9999999978516465, "rho": 0.1995042791607679, "secs": 17.2, "caps": {"Customer": {"Invoice": 7}, "Invoice": {"InvoiceLine": 8}, "InvoiceLine": null}} +{"db": "chinook", "eps": 4, "rep": 2, "passed": 125, "total": 140, "score": 0.8928571428571429, "eps_stated": 3.9999999978516465, "rho": 0.1995042791607679, "secs": 18.2, "caps": {"Customer": {"Invoice": 7}, "Invoice": {"InvoiceLine": 8}, "InvoiceLine": null}} +{"db": "chinook", "eps": 8, "rep": 0, "passed": 126, "total": 140, "score": 0.9, "eps_stated": 8.000000000000002, "rho": 0.7110560428191794, "secs": 16.9, "caps": {"Customer": {"Invoice": 7}, "Invoice": {"InvoiceLine": 2}, "InvoiceLine": null}} +{"db": "chinook", "eps": 8, "rep": 1, "passed": 127, "total": 140, "score": 0.9071428571428571, "eps_stated": 8.000000000000002, "rho": 0.7110560428191794, "secs": 16.5, "caps": {"Customer": {"Invoice": 7}, "Invoice": {"InvoiceLine": 14}, "InvoiceLine": null}} +{"db": "chinook", "eps": 8, "rep": 2, "passed": 123, "total": 140, "score": 0.8785714285714286, "eps_stated": 8.000000000000002, "rho": 0.7110560428191794, "secs": 17.3, "caps": {"Customer": {"Invoice": 7}, "Invoice": {"InvoiceLine": 9}, "InvoiceLine": null}} +{"db": "contoso_sqlite", "eps": null, "passed": 246, "total": 246, "score": 1.0, "secs": 505.7} +{"db": "contoso_sqlite", "eps": 0.5, "rep": 0, "passed": 197, "total": 246, "score": 0.8008130081300813, "eps_stated": 0.49999999974072734, "rho": 0.003796322946649587, "secs": 452.4, "caps": {"Customer": {"Orders": 4}, "Orders": {"OrderRows": 7}, "OrderRows": null}} +{"db": "contoso_sqlite", "eps": 1, "rep": 0, "passed": 199, "total": 246, "score": 0.8089430894308943, "eps_stated": 0.9999999994787825, "rho": 0.014407578572581993, "secs": 853.8, "caps": {"Customer": {"Orders": 4}, "Orders": {"OrderRows": 7}, "OrderRows": null}} +{"db": "contoso_sqlite", "eps": 2, "rep": 0, "passed": 200, "total": 246, "score": 0.8130081300813008, "eps_stated": 1.9999999989465718, "rho": 0.05413165918614995, "secs": 666.5, "caps": {"Customer": {"Orders": 4}, "Orders": {"OrderRows": 7}, "OrderRows": null}} +{"db": "contoso_sqlite", "eps": 4, "rep": 0, "passed": 199, "total": 246, "score": 0.8089430894308943, "eps_stated": 3.9999999978516465, "rho": 0.1995042791607679, "secs": 742.6, "caps": {"Customer": {"Orders": 4}, "Orders": {"OrderRows": 7}, "OrderRows": null}} +{"db": "contoso_sqlite", "eps": 8.0, "rep": 0, "passed": 203, "total": 246, "score": 0.8252032520325203, "eps_stated": 8.000000000000002, "rho": 0.7110560428191796, "secs": 896.4, "caps": {"Customer": {"Orders": 4}, "Orders": {"OrderRows": 7}, "OrderRows": null}} +{"db": "adventureworks", "eps": null, "passed": 1210, "total": 1257, "score": 0.9626093874303898, "secs": 1484.6} +{"db": "adventureworks", "eps": 2.0, "rep": 0, "passed": 1149, "total": 1257, "score": 0.9140811455847255, "eps_stated": 1.999999998946572, "rho": 0.054131659186149955, "secs": 951.1, "caps": {"DimCustomer": {"FactInternetSales": 8, "FactSurveyResponse": 2}, "FactInternetSales": {"FactInternetSalesReason": 2}, "FactInternetSalesReason": null, "FactSurveyResponse": null}} diff --git a/docs/security.md b/docs/security.md index 37557c5..82b6371 100644 --- a/docs/security.md +++ b/docs/security.md @@ -7,7 +7,7 @@ relsynth reads production databases and writes data that is meant to be shared. | Asset | Threat | Protection | |---|---|---| | Database credentials | Leaked through spec files, logs, error messages, CI output, diagnostics bundles, process lists | Credentials come from the environment, and every output masks them (see below) | -| Source data | Synthetic rows that copy real ones; rare values that identify a person; bundles that carry rows | Bayesian synthesis (no row is copied), memorisation guards for text, privacy checks, and aggregate-only bundles | +| Source data | Synthetic rows that copy real ones; rare values that identify a person; bundles that carry rows | Bayesian synthesis (no row is copied), memorisation guards for text, privacy attacks per person ([privacy.md](privacy.md)), optional differential privacy ([dp.md](dp.md)), and aggregate-only bundles | | The machine running relsynth | A malicious spec (YAML tricks, rule expressions, regexes) or a malicious file name | Safe YAML loading, a sandboxed expression language, regex vetting, and confined paths | | The target database | SQL injection through table or column names or values | SQLAlchemy Core with quoted identifiers and bound values everywhere | | The source database | Accidental writes, or heavy load | Only `SELECT` is issued against a source; pushdown statistics are plain aggregate queries | @@ -59,7 +59,7 @@ Tests: `test_inside_refuses_non_plain_names`, `test_symlinked_table_cannot_escap ## Data protection - **Nothing is copied.** Synthetic rows are drawn from a fitted model; only tables declared `mode: copy` (a calendar, a code list) are copied. Key values are generated in the source's format, so a synthetic key that equals a real key does not carry that real row's data. `test_synthetic_rows_are_not_copies` checks that fewer than 5% of synthetic rows match a real row on their high-cardinality attributes. -- **Privacy checks run by default.** +- **Copy guards run by default.** They catch gross copying; they are not a privacy measurement (see [privacy.md](privacy.md#relation-to-the-existing-guards)). - For each table, the distance from synthetic rows to their closest real row (in rank-normal-score and one-hot space) is compared with the distance from real rows to their closest other real row (leave-one-out). The check fails when the 5th percentile of the first is below half that of the second, or when synthetic rows equal real rows more often than real rows equal each other. - Text outputs that reproduce rare source content are rejected (below). - `--no-privacy` turns the checks off, and the bundle records that. @@ -73,19 +73,19 @@ Tests: `test_inside_refuses_non_plain_names`, `test_symlinked_table_cannot_escap - Frequent values appear only with an explicit `profile --top N`. - Counts below 5 are suppressed in what a bundle reports about category levels. - **Pushdown.** Rows of pushed facts never leave the source as a whole; only aggregates and a sample do. -- **Formal differential privacy is not part of v1.0.** The privacy guarantees are empirical. For a release outside your organisation, review the privacy section of the scorecard, and consider `mode: copy` only for tables that are public anyway. +- **Differential privacy is optional.** Without it, protection is measured, not guaranteed: `relsynth privacy` attacks the release per person ([privacy.md](privacy.md)). With `--epsilon`, a run has a user-level (ε, δ) guarantee ([dp.md](dp.md)). Under the governance policy, a release outside the organisation is refused without DP ([governance/guide.md](governance/guide.md)). Use `mode: copy` only for tables that are public anyway. ## Supply chain and code -- The package depends on five well-known libraries (numpy, pandas, scipy, SQLAlchemy, PyYAML) and the database driver you choose. Versions have lower bounds only, so you can patch them yourself. +- The core package depends on seven well-known libraries (numpy, pandas, scipy, SQLAlchemy, PyYAML, DuckDB and its SQLAlchemy dialect) and the database driver you choose; every other dependency comes with an extra you install. Versions have lower bounds only, so you can patch them yourself. - CI runs on every change: - `ruff` with the `S` rule set (the flake8-bandit security rules); - `bandit` over `src/`; - `pip-audit` over the installed dependencies. Every suppressed finding is annotated in the code with its test ID and a reason, for example `# nosec B608` where a query is built with SQLAlchemy and not by string formatting. -- No network access other than to the databases you name, and no telemetry. -- `subprocess` is used once: to run `pdflatex` (when present) for the exported model's PDF, with a fixed argument list and no shell. +- The package and the CLI connect only to the databases, warehouses and storage you name, and send no telemetry. The service also contacts the OIDC identity provider and the Vault server you configure, and exports traces only when you turn on OpenTelemetry (`relsynth[service-otel]`) and give it an endpoint. +- `subprocess` runs fixed argument lists, never a shell, in four places: `pdflatex` (when present) for the exported model's PDF, the OPA binary when the policy engine is `opa`, the GPU listing commands of `relsynth compute`, and the service worker's job runner. ## Reporting a vulnerability From b5c0e62e8f657a4c1ab607d88f468be9df2d7b46 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 19:04:03 +0000 Subject: [PATCH 3/8] docs(assurance): assurance case, regulation mapping, DPIA template and release guidance The deliverables of roadmap T17 for customers' privacy, legal and audit teams: - assurance-case.md: top claim, sub-claims for structure, measured and formal privacy, governance and traceability, assumptions A1-A5 and defeaters, every leaf citing the claim register or a proposition; - regulation-mapping.md: EDPB Guidelines 02/2026 (v1.0, paras. 6-103) and EDPS v SRB (C-413/23 P, paras. 71-112), AI Act Art. 10 as amended by Regulation (EU) 2026/1744 (Art. 10(5) moved to Art. 4a), 45 CFR 164.514 and the HHS 2012 guidance, NIST SP 800-226 and SP 800-188, and ISO/IEC 27559 and 20889 by topic (texts not available), with rows a lawyer must decide marked as such; - dpia-template.md following GDPR Art. 35(7); - release-guidance.md for internal, partner and public recipients, with the Five Safes and an approval checklist. The claim checker now also fails when these documents cite a claim id, a proposition label or a check that does not exist. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PSVqwE13trs5MdpBoE4YyJ Signed-off-by: Claude --- CHANGELOG.md | 3 + docs/assurance/README.md | 38 +++++++ docs/assurance/assurance-case.md | 92 +++++++++++++++++ docs/assurance/dpia-template.md | 133 ++++++++++++++++++++++++ docs/assurance/regulation-mapping.md | 149 +++++++++++++++++++++++++++ docs/assurance/release-guidance.md | 67 ++++++++++++ docs/privacy.md | 2 + tests/unit/test_assurance.py | 10 ++ tools/claims_check.py | 11 ++ 9 files changed, 505 insertions(+) create mode 100644 docs/assurance/README.md create mode 100644 docs/assurance/assurance-case.md create mode 100644 docs/assurance/dpia-template.md create mode 100644 docs/assurance/regulation-mapping.md create mode 100644 docs/assurance/release-guidance.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b3b2a8..18d7b01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ All notable user-visible changes are recorded here, following [Keep a Changelog ### Added +- **Compliance evidence and assurance case** (`docs/assurance/`; roadmap T17): an assurance case for a release (claims, evidence, assumptions, defeaters), a mapping to EDPB Guidelines 02/2026 and *EDPS v SRB*, the AI Act (Art. 10 and the new Art. 4a), HIPAA expert determination, NIST SP 800-226 and SP 800-188 and ISO/IEC 27559 and 20889, a DPIA template for a synthetic release, and release guidance for internal, partner and public recipients. +- **Claim register** (`docs/assurance/claims.yaml`, `tools/claims_check.py`, CI job `lint`): every claim of the privacy, DP, security and governance guides traced to a proposition, check, test or measured report; CI fails when a claim changes, its evidence disappears, or a new claim is not registered. - **Engine settings** (`relsynth.settings`, `relsynth settings show|explain|validate|template|reference`, guide `docs/settings.md`, reference `docs/settings-reference.md`): one registry of 101 engine settings with type, range, meaning and the checks each affects. 24 constants that were fixed in the code are now settings (copula MH steps and MCSE target, marginal and FD sample sizes, occupancy and frailty thresholds, rake limits, the new `repair` and `detection` groups, scorecard thresholds and sample sizes under `evaluation`). Settings come from defaults < a settings file (`synthesis.settings_file`, `--settings`, `RELSYNTH_SETTINGS`, `preset:fast`, `preset:thorough`) < the spec < `--set`; `per_table` overrides the copula, marginal and text groups for one table. In Python, `relsynth.Settings` has editable attributes validated on assignment, `Spec.with_settings()`, and `fit`/`run(settings=)`. With default settings the output is byte for byte unchanged (`tools/identity_check.py`, CI job `settings-autotune`). - **Self-tuning** (`relsynth autotune`, guide `docs/autotune.md`, research `docs/research/self-tuning.md`): runs the engine on several seeds, separates systematic failures from noise, and changes settings and (optionally) applies doctor spec fixes until the user's objectives hold. Hard constraints (keys, links, rules, trees, privacy, domain identities, any check id) must hold on every seed; statistical families are calibrated (failures at most the 0.99 quantile of their null count). Guided moves from the registry, learned effects from the history, successive halving, paired-seed acceptance and confirmation on held-out seeds. The history (`.tune/history.jsonl`, aggregates only) persists and is resumed. A constraints report gives, for every objective not met, the reason (no setting, insensitive, conflict, undecided, budget), the seed evidence and the noise floor. Refused in DP mode. @@ -29,6 +31,7 @@ All notable user-visible changes are recorded here, following [Keep a Changelog ### Changed +- `docs/security.md` now describes 2.0: differential privacy is optional rather than absent, the distance guards are copy guards rather than privacy checks, seven core dependencies, four `subprocess` call sites, and the service's connections to the OIDC provider and Vault. - The ordinary pipeline, `tune`, `verify`, `export-model` and `doctor --apply` refuse a spec in DP mode instead of reading it without DP. - **MySQL 8 and MariaDB connector** (`mysql://`, `mariadb://`, `relsynth[mysql]`): read, write in every mode, `clone-schema`, and fitting inside the database (GROUP BY statistics, window-function rank sums, a seeded SHA-256 hash sample). Text is grouped and compared in a binary collation, so statistics stay exact on case- and accent-insensitive columns. Created columns keep the data: `DOUBLE`, `DATETIME(6)`, `utf8mb4` text with a binary no-pad collation, key columns within MySQL's index limit and wide tables within its row limit. Tested live on MySQL 8.4 and MariaDB 11.4. diff --git a/docs/assurance/README.md b/docs/assurance/README.md new file mode 100644 index 0000000..13dc7e7 --- /dev/null +++ b/docs/assurance/README.md @@ -0,0 +1,38 @@ +# Compliance evidence and assurance + +This folder is for the privacy, legal and audit teams who decide whether a synthetic dataset made with relsynth may be released. It does not decide for them and is not legal advice; it gives them the argument, the evidence and the templates. + +| Document | Use it to | +|---|---| +| [assurance-case.md](assurance-case.md) | Follow the argument that a release is fit for its recipient: claims, the evidence for each, assumptions and known defeaters | +| [regulation-mapping.md](regulation-mapping.md) | See what relsynth contributes to each provision of the GDPR (EDPB Guidelines 02/2026, *EDPS v SRB*), the AI Act (Art. 10, Art. 4a), HIPAA expert determination, NIST SP 800-226 and SP 800-188, ISO/IEC 27559 and 20889, and what stays with the organisation | +| [dpia-template.md](dpia-template.md) | Document one release: a DPIA following GDPR Art. 35(7), pre-filled with what relsynth does | +| [release-guidance.md](release-guidance.md) | Decide how to make and approve a release for an internal, partner or public recipient | +| [claims.yaml](claims.yaml) | Look up the evidence behind a sentence of the documentation | +| [evidence/](evidence/README.md) | The aggregate measurement files that claims cite | + +## The claim register + +Every sentence of the documents listed under `scope` in [claims.yaml](claims.yaml) that reads as a claim (it says *never*, *every*, *exact*, *guarantee*, *refused* and similar) has an entry. Each entry quotes the sentence and lists its evidence: a proposition of the [proofs](../proofs/relsynth-proofs.tex) by label, a numbered check (C902), a pytest test, a measured report in this repository, or the code that enforces it. A `status` says whether the evidence supports the claim fully, partly (with a note), or shows a limitation. + +`python tools/claims_check.py` checks the register, and CI runs it on every pull request. It fails when: + +- a quoted sentence is no longer in its document; +- a cited proposition, check, test or file does not exist; +- a claim about relsynth rests only on an external source or on code, without a proposition, check, test or report; +- a claim-like sentence of a document in scope has no entry, and is not listed under `not_claims` with a reason; +- a `Prop. N` in any Markdown document names a number the proofs do not define; +- these documents cite a claim id, a proposition label or a check that does not exist. + +When your change makes it fail: + +| Message | Fix | +|---|---| +| `quote not found` | You changed a registered sentence. Update `quote`, and check that the evidence still supports the new wording | +| `unregistered claim` | You wrote a new claim. Add an entry with its evidence (`python tools/claims_check.py --suggest docs/x.md` prints stubs), or, if it is not a claim about relsynth, add it to `not_claims` with a reason | +| `no test ...`, `no proposition ...` | You renamed or removed evidence. Point the entry at the new name | +| `needs a prop, check, test or report` | Write the test that shows the claim, or weaken the claim | + +`python tools/claims_check.py --json claims.json` writes the register with every proposition label resolved to its printed number, for an auditor's tooling. + +Documents in scope today: docs/privacy.md, docs/dp.md, docs/security.md and docs/governance/guide.md. The README and the remaining guides are added one by one; until then their `Prop. N` references are still checked. diff --git a/docs/assurance/assurance-case.md b/docs/assurance/assurance-case.md new file mode 100644 index 0000000..73bb7bb --- /dev/null +++ b/docs/assurance/assurance-case.md @@ -0,0 +1,92 @@ +# Assurance case for a relsynth release + +An assurance case is a structured argument that a system has a property, supported by evidence (claims, arguments and evidence, as in ISO/IEC/IEEE 15026-2 and the Goal Structuring Notation). This one argues that a synthetic dataset produced and released with relsynth is fit to give to its intended recipient. It is written for the organisation's privacy officer, legal counsel and auditors, who decide whether to approve a release; relsynth supplies the evidence, they supply the judgement. + +Every leaf points at the [claim register](claims.yaml) by id (PRV-, DP-, SEC-, GOV-) or at a proposition of the [proofs](../proofs/relsynth-proofs.tex) by its label. `tools/claims_check.py` checks in CI that every id cited here exists and that its evidence still resolves. The [regulation mapping](regulation-mapping.md) says which provision each part of the argument answers. + +## Top claim + +**G0.** A synthetic dataset produced by relsynth from a source, and released to a recipient through the governance workflow, (a) has the structure and statistics the recipient needs, (b) carries a re-identification risk for that recipient that is measured, below the organisation's limits and, for any recipient outside the organisation, bounded by differential privacy, and (c) was produced and released under recorded, tamper-evident and approved governance. + +Context of G0: + +- **C1. Release context.** One of the contexts of the organisation's policy: `internal`, `partner`, `public` or the organisation's own ([release guidance](release-guidance.md)). The context fixes what the attacker is assumed to know and the limits. +- **C2. Privacy unit.** The table of the people protected; every row reachable from it through foreign keys belongs to one person (PRV-09; Props. `prop:pusplit`, `prop:irowner`). +- **C3. Organisation policy.** Signed by the privacy owner; holds ε caps per privacy domain, the contexts and the approvals they need (GOV-03). + +Assumptions (if one is false, the argument does not hold): + +- **A1.** The spec describes the source correctly: keys, links and rules the spec does not declare are reproduced statistically, not exactly. +- **A2.** The machine and accounts that run relsynth, the registry and the signing keys are protected as the organisation's other production systems are (NIST SP 800-226 sec. 4.4: a breach of the raw data voids any DP guarantee). +- **A3.** OpenDP's discrete Gaussian sampler and the operating system's random source are correct (DP mode relies on them; NIST SP 800-226 sec. 4.3 recommends validated libraries). +- **A4.** The ε in the policy was chosen by the organisation for the context and the domain. relsynth ships no ε (DP-03). +- **A5.** For a DP release, the spec was written from public facts only (`spec_is_public: true`), as the policy requires for external contexts (DP-11). + +Strategy **S0**: argue each of (a), (b) and (c) separately, then argue that the claims in the documentation used for this argument are themselves traced (G4). + +## G1. Structure and statistics (part a) + +| Claim | Argument | Evidence | +|---|---|---| +| **G1.1** Every primary and foreign key resolves, every declared or found rule holds on every row | Keys are generated, never sampled; rules are imposed by exact conditioning (rejection, redraw instead of delete) | Props. `prop:rules`, `prop:k3`, `prop:computed`, `prop:unique`, `prop:refill`; checks C00–C78 (`pytest -m science`); with any generator plug-in: `prop:genskeleton`; with partitions: `prop:rtkeys` | +| **G1.2** Statistics match the source within limits calibrated to sampling noise | Each scorecard limit is the larger of a fixed tolerance and the sampling limit under the posterior predictive null; families are Šidák-corrected | Props. `prop:splitnull`, `prop:pitcurve`, `prop:arearule`, `prop:carry`; checks C79–C82 | +| **G1.3** This holds on real databases, not only on fake data with known ground truth | The bench re-runs four real databases on every change against a five-seed baseline | `bench/` and its baseline; the T18/T10 equivalence tests on warehouses and lakehouses | +| **G1.4** In DP mode, the loss of utility is known | Utility measured against ε on three real databases | DP-15 | + +Defeater **D1.1**: a check can fail by chance on real data, because the source is one draw. *Rebuttal*: each check reports its level (`alpha`), so the expected number of false failures is known. + +## G2. Re-identification risk (part b) + +Strategy **S2**: show that the risk is measured per person against the EDPB's three criteria and membership inference, with intervals and a safety margin (G2.1–G2.4); that releases outside the organisation are additionally bounded by DP (G2.5–G2.7); and state what neither covers (G2.8). + +| Claim | Argument | Evidence | +|---|---|---| +| **G2.1** Risk is measured per person, not per row | Attacks run on one profile per person of the privacy unit, built from every owned table; keys are not in profiles | PRV-09, PRV-10 | +| **G2.2** The measurement answers the EDPB criteria | Singling out (No record isolation), linkage (No linkage), attribute and membership inference (No inference) | PRV-04, PRV-05, PRV-06, PRV-07 | +| **G2.3** A measured risk is a risk to the training people, not an artefact | Training and control people are split at random with equal sizes; only the excess on training people counts; under the null the rates are centred | PRV-11; Props. `prop:puctrl`, `prop:puattr`; checks C601, C606 | +| **G2.4** The decision is conservative | Verdicts use the upper confidence bound (safety margin); the most exposed people are attacked one by one; small populations fail external contexts; a *review* needs a named sign-off | PRV-08, PRV-12, PRV-14, PRV-15 | +| **G2.5** No release leaves the organisation without DP | Enforced before any data is read, and as an invariant that the policy cannot waive | DP-02, PRV-13, GOV-04, GOV-05 | +| **G2.6** A DP run satisfies user-level (ε, δ)-DP for the people of the unit | Contribution bounding is stable, every released statistic is charged its sensitivity, partition selection bounds δ, stage 2 is post-processing, the accountant refuses anything beyond the budget | DP-01, DP-04, DP-05, DP-06, DP-07; Props. `prop:dgauss` to `prop:run`; checks C900–C906 | +| **G2.7** The DP bound and the measurements agree | The largest TPR any attack can reach at a given FPR follows from ε (Kairouz et al. 2015); the audit on Contoso at ε = 1 stayed below it | PRV-16, DP-10, DP-14; Prop. `prop:pudp`; check C603 | +| **G2.8** What is not covered is stated | Empirical attacks cover the attacks run; the spec drafted by `introspect` is outside the DP guarantee; non-DP releases of the same people are not covered; DP mode has known gaps | PRV-19, DP-11, DP-13, GOV-11 | + +Defeaters and their status: + +- **D2.1** *The distance-to-closest-record guard misses copying.* True: it missed 20% copied records. *Rebuttal*: no verdict depends on it; DOMIAS and the shadow models detect copying at 5% (PRV-18). +- **D2.2** *Small databases cannot be judged.* True: with tens of people the intervals are wide. *Rebuttal*: the report says *review*, and external contexts fail below 1,000 people without DP (PRV-15). Chinook (59 customers) fails internally on membership inference (AUC 0.84, roadmap report T06). +- **D2.3** *The spec leaks.* Values that `introspect` reads (levels, ranges, rules) are in the spec. *Rebuttal*: for external contexts the policy requires a spec written from public facts (A5, DP-11); for internal contexts the spec stays inside the organisation. +- **D2.4** *Repeated releases about the same people compose.* *Rebuttal*: the budget is kept per privacy domain with a lifetime cap that never resets (GOV-08, GOV-10); the privacy report shows the cumulative bound (PRV-16). +- **D2.5** *A future attack may beat today's suite* (EDPB para. 92: AI lowers attack cost). Open for non-DP releases; for DP releases the bound holds against any attack (G2.7). Reassess on a schedule (EDPB para. 35; [DPIA](dpia-template.md) section 9). +- **D2.6** *Floating-point and timing side channels in DP mechanisms* (NIST SP 800-226 sec. 4.3). *Rebuttal*: noise is integer discrete Gaussian from OpenDP; timing is not addressed and is out of scope in the central model, where only the synthetic output leaves. + +## G3. Governance (part c) + +| Claim | Argument | Evidence | +|---|---|---| +| **G3.1** Every run is recorded and later changes are detected | Signed manifests in a hash chain with signed checkpoints; verification catches edits, deletions, insertions, reordering and truncation | GOV-01, GOV-09; Prop. `prop:govchain`; check C803 | +| **G3.2** Only accountable people act | Ed25519 keys per person and service, roles from the privacy owner's root key; policy signed by the privacy owner | GOV-02, GOV-03 | +| **G3.3** Privacy budget cannot be overspent | Reserve before reading, never refund after a read, lifetime cap per domain, period caps only throttle | GOV-08, GOV-10, DP-09; Props. `prop:ledger`, `prop:period`, `prop:reserve`; checks C800–C802 | +| **G3.4** A release needs a verdict and approvals | No verdict, a `fail` or `blocked` verdict, or a missing approval denies the release; external releases need someone other than the operator | GOV-04, GOV-05 | +| **G3.5** What leaves is what was approved, and nothing computed on real data | The package holds the tables byte for byte as signed, the manifest, approvals and token; the privacy report never leaves; manifests hold no data or credentials | GOV-06, GOV-07 | + +Defeater **D3.1**: *the registry is a folder; an administrator could delete it.* Deletion of the log is detected against checkpoints held elsewhere (the recipient verifies a package against the root key id); keeping a copy of the checkpoints outside the registry is the organisation's control (A2). + +## G4. The documentation the argument relies on is traced + +**G4.1** Every claim that this argument cites appears in the product documentation and traces to a proposition, a check, a test or a measured report. *Argument*: `tools/claims_check.py` runs in CI on every pull request (job `lint`): it fails when a quote leaves its document, a test or proposition is renamed or removed, a claim-like sentence in a document in scope is not registered, or a `Prop. N` reference names an undefined proposition. *Evidence*: the register, `tests/unit/test_assurance.py`. + +Defeater **D4.1**: *a claim without a trigger word escapes the scan.* True: the scan finds sentences with words such as "never", "every", "exact", "guarantee", "refused". *Rebuttal*: documents are reviewed by pull request; the register is extended document by document (the README is next). + +## How to use this case for one release + +1. Run the release through the governance workflow with `--assess` (or `--privacy-report`) and keep the registry. +2. Fill in the [DPIA template](dpia-template.md); its sections cite the parts of this case they rely on. +3. Check the assumptions A1–A5 for this release and record any that do not hold. +4. Approve or reject in the registry (`relsynth gov approve`). The approval records who decided. + +## Sources + +- ISO/IEC/IEEE 15026-2:2022, *Systems and software engineering — Systems and software assurance — Part 2: Assurance case*. +- Assurance Case Working Group, *Goal Structuring Notation Community Standard*, version 3, 2021. +- P. Kairouz, S. Oh, P. Viswanath, *The composition theorem for differential privacy*, ICML 2015. +- The texts in the [regulation mapping](regulation-mapping.md#sources). diff --git a/docs/assurance/dpia-template.md b/docs/assurance/dpia-template.md new file mode 100644 index 0000000..5772488 --- /dev/null +++ b/docs/assurance/dpia-template.md @@ -0,0 +1,133 @@ +# DPIA template: a synthetic data release + +A data protection impact assessment for generating a synthetic dataset with relsynth from personal data and giving it to a recipient. It follows the content GDPR Article 35(7) requires: (a) a description of the processing and its purposes, (b) necessity and proportionality, (c) the risks to the rights and freedoms of data subjects, and (d) the measures to address them. Copy this file, fill in the parts in *italics*, and keep it with the release's registry entry. + +Whether a DPIA is legally required (Art. 35(1) and (3), and the supervisory authority's list under Art. 35(4)) is for the controller's DPO to decide. The template is useful either way: EDPB Guidelines 02/2026 para. 41 asks for "adequate documentation of the anonymisation processing", including testing. **It is not legal advice.** + +The text in plain type is pre-filled with what relsynth does and where the evidence is. Check it against your installation: settings can change it. + +--- + +## 0. Summary + +| | | +|---|---| +| Release name and run id | *e.g. `crm-test-2026q4`, registry run `r-...`* | +| Controller | *organisation* | +| DPO consulted | *name, date* | +| Source system and privacy domain | *e.g. `dwh-prod`, domain `customers`* | +| Privacy unit (the people protected) | *table, approximate number of people* | +| Recipient and release context | *internal / partner / public / own context; recipient organisation* | +| Differential privacy | *ε, δ at the domain's δ, or "none (internal only)"* | +| Decision | *approve / approve with conditions / reject* — signed in the registry with `relsynth gov approve` | + +## 1. Description of the processing (Art. 35(7)(a)) + +**1.1 Purpose.** *Why synthetic data are needed (software testing, analytics, model development, sharing with a partner) and why real data or aggregates do not suffice.* + +**1.2 Data.** *Tables read, special categories (Art. 9) or criminal-offence data (Art. 10), the privacy unit and the owned tables (`relsynth privacy` lists them), approximate rows and people.* + +**1.3 The processing, step by step.** + +1. relsynth reads the source catalog and a sample of rows to draft a spec (`introspect`), unless the spec is written from public facts. +2. It fits a Bayesian model of the whole schema inside the organisation. Large facts are fitted from aggregates computed in the database plus a sample (docs/pushdown.md). In DP mode, it reads each owned table once and releases noisy statistics before fitting (docs/dp.md, section 2). +3. It draws a synthetic database. No source row is copied, except tables declared `mode: copy` (*list them, and confirm each holds no personal data*). +4. It attacks the synthetic database per person (`relsynth privacy`) and scores its quality. +5. It records a signed manifest, and releases only after the policy allows it and the required people approve. + +**1.4 Where it runs and who has access.** *Hosts, accounts, the registry location, who holds the signing keys.* + +**1.5 Retention.** *How long the source extract, the fitted model, the privacy report and the synthetic data are kept, and where. The privacy report and the model are computed on real data and stay inside the organisation (claim GOV-07).* + +**1.6 Legal basis.** *Article 6 basis for fitting the model, and Article 9(2) condition for special categories. EDPB 02/2026 para. 38: anonymisation needs a legal basis, presumed to be that of the processing it follows when it is part of the same activity and purpose.* **Lawyer.** + +## 2. Necessity and proportionality (Art. 35(7)(b)) + +| Question | Answer | +|---|---| +| Could the purpose be met with less data (fewer tables, columns, people)? | *Which tables and columns were left out; `relsynth` can be pointed at a subset* | +| Could it be met with public or already anonymous data? | *...* | +| Is a formal guarantee used? | DP is required for every release outside the organisation (claim DP-02). *For internal releases, say why DP is or is not used.* | +| Minimisation of what relsynth reads | Pushed-down facts are read as aggregates and a sample (SEC-10); DP mode reads each owned table once | +| Transparency to data subjects | *Do the privacy notices of the source mention the use of their data to build synthetic data? CJEU C-413/23 P para. 111: assessed from the controller's point of view at collection.* **Lawyer** | + +## 3. The recipient and the context (EDPB 02/2026 paras. 11–15, 26–35) + +| Question | Answer | +|---|---| +| Who will receive the data, and who could obtain it after them (onward transfers, leaks)? | *...* (para. 21; CJEU C-413/23 P para. 85) | +| What other data could they combine it with? | *Set the context's `attacker_knows` accordingly* | +| Which relevant entities must be considered? | *Rogue employees, journalists, unethical companies, cybercriminals, authorities (para. 30). Motivation is not a factor (para. 31).* | +| Are there legal prohibitions on re-identification, and are they enforced? | *A contract is not a legal prohibition (para. 34).* | +| Will the data be public? | *If yes, anyone may access it (para. 89): use the `public` context.* | + +## 4. Risks to data subjects (Art. 35(7)(c)) + +Fill in from the privacy report of this run (`privacy_report.md`, page of the release context). Each line has a value, its 95% interval, the limit and the status. + +| Risk (EDPB criterion) | Attack | Value [95% CI] | Limit | Status | +|---|---|---|---|---| +| Singling out (No record isolation, sec. 3.4.1) | Singling-out queries, univariate and multivariate | | | | +| Linkage (No linkage, sec. 3.4.2) | Nearest-neighbour linkage | | | | +| Inference of attributes (No inference, sec. 3.4.3) | Inference per sensitive attribute, against a population baseline | | | | +| Membership (No inference, para. 79) | Shadow models, LiRA-style ratio, DOMIAS: AUC and TPR at FPR 1% | | | | +| Most exposed person (para. 87) | Membership AUC of the person most at risk | | | | +| DP bound (if DP) | Largest TPR at FPR 1% any attack can reach, this release and cumulative for the domain | | | | + +**4.1 Verdict for the context:** *pass / review (signed off by: ...) / fail / blocked.* + +**4.2 What the measurement does not cover** (assurance case G2.8): + +- Results hold for the attacks run; only DP bounds every attack (PRV-19). +- The spec drafted by `introspect` holds levels, ranges and rules read from the data, outside any DP guarantee, unless written from public facts (DP-11). +- Releases of the same people without DP are not covered by the DP accounting (GOV-11). +- *Other risks: harm from inaccurate synthetic data used to make decisions about people; bias from suppressed rare groups in DP mode (docs/dp.md section 5); risks specific to this release.* + +**4.3 Severity and likelihood.** *For each risk, the impact on people if it materialised and how likely it is given section 3; use the organisation's risk matrix.* + +## 5. Measures (Art. 35(7)(d)) + +| Measure | In place? | Evidence | +|---|---|---| +| Synthetic rows drawn from a model; no row copied | built in | SEC-02 | +| Per-person attacks with a safety margin, verdict per context | built in, run with `--assess` | PRV-01, PRV-12 | +| Differential privacy for external contexts | built in, enforced before any read | DP-01, DP-02 | +| Privacy budget per domain with a lifetime cap, reserved before reading | built in | GOV-08, GOV-10 | +| Named sign-off of a *review*, approvals with separation of duties | built in, configured in the policy | PRV-14, GOV-04 | +| Signed, tamper-evident record of the run and the release | built in | GOV-01, GOV-09 | +| Privacy report and model stay inside the organisation | built in | GOV-07 | +| Credentials from the environment, masked in every output | built in | SEC-01, SEC-04 | +| *Contract with the recipient (no re-identification, no onward transfer)* | *yes / no* | *reference* | +| *Access control on the registry and keys* | *yes / no* | *reference* | +| *Other organisational measures* | | | + +## 6. Residual risk and decision + +*State the risk that remains after the measures, whether it is acceptable, and why. If a high risk remains, Art. 36 prior consultation of the supervisory authority applies.* **Lawyer / DPO.** + +## 7. Advice of the DPO (Art. 35(2)) + +*DPO's advice, and whether it was followed.* + +## 8. Views of data subjects or their representatives (Art. 35(9)), where appropriate + +*...* + +## 9. Review + +EDPB 02/2026 para. 35: re-identification risk can rise over time; reassess periodically. Para. 42: a security incident may trigger reassessment. + +| Trigger | Action | +|---|---| +| *Every 12 months, or the organisation's period* | Re-run `relsynth privacy` with the current release of relsynth on the same registry entry | +| The final EDPB guidelines are adopted | Check the release against the [regulation mapping](regulation-mapping.md#7-tracking-the-final-edpb-guidelines) | +| A new attack is published that the suite does not cover | Reassess non-DP releases | +| A security incident affects the source, the registry or the recipient | Reassess, and revoke the release if needed (`relsynth gov revoke`) | + +## Annex: files to attach + +- The privacy report (internal only): `privacy_report.json`, `.md`, `.html`. +- The DP statement (`dp.json`), for a DP run. +- The quality report or scorecard. +- `relsynth gov show RUN` output and `relsynth gov verify registry/` result. +- The [assurance case](assurance-case.md) at the version of relsynth used (`relsynth version`). diff --git a/docs/assurance/regulation-mapping.md b/docs/assurance/regulation-mapping.md new file mode 100644 index 0000000..95cf442 --- /dev/null +++ b/docs/assurance/regulation-mapping.md @@ -0,0 +1,149 @@ +# Regulation mapping + +This page maps what relsynth does to the texts a privacy, legal or audit team will test a synthetic release against. Each row cites the provision (article, paragraph or section), says what relsynth contributes, names the evidence by its [claim register](claims.yaml) id, and says what remains for the organisation. **It is not legal advice.** Rows marked **Lawyer** are decisions the controller's counsel or DPO must take; relsynth can only supply evidence for them. + +Texts read for this mapping (2026-10-31): EDPB Guidelines 02/2026 v1.0; CJEU judgment C-413/23 P; Regulation (EU) 2024/1689 as amended by Regulation (EU) 2026/1744; 45 CFR 164.514 and the HHS guidance of 26 November 2012; NIST SP 800-226 and SP 800-188. ISO/IEC 27559:2022 and ISO/IEC 20889:2018 are sold by ISO and were not available; their rows map by topic and say so. Where a paragraph was not read in full, the row says so too. + +Contents: [GDPR anonymisation](#1-gdpr-anonymisation-edpb-guidelines-022026-and-edps-v-srb) · [AI Act](#2-eu-ai-act-article-10-data-and-data-governance) · [HIPAA](#3-hipaa-de-identification-expert-determination) · [NIST SP 800-226](#4-nist-sp-800-226-evaluating-differential-privacy-guarantees) · [NIST SP 800-188](#5-nist-sp-800-188-de-identifying-government-datasets) · [ISO/IEC 27559 and 20889](#6-isoiec-275592022-and-isoiec-208892018) · [Tracking the final EDPB text](#7-tracking-the-final-edpb-guidelines) · [Sources](#sources) + +## 1. GDPR anonymisation: EDPB Guidelines 02/2026 and EDPS v SRB + +Status of the guidelines: version 1.0, "adopted on 07 July 2026", for public consultation until 30 October 2026. They are not final; section 7 says how this mapping tracks the final text. + +### 1.1 The legal test + +| Provision | What it requires | What relsynth contributes | Evidence | Left to the organisation | +|---|---|---|---|---| +| GDPR Art. 4(1), Recital 26; EDPB paras. 6–8 | Data are anonymous only if no one can identify a person with the means reasonably likely to be used; the question "needs to be viewed as one of likelihood" (para. 8) | Synthetic rows are drawn from a fitted model, not copied; the risk that remains is measured per person against the three criteria below | SEC-02, PRV-01 | **Lawyer**: whether the measured likelihood is "insignificant in reality" for this release | +| EDPB para. 22; CJEU C-413/23 P para. 82 | The likelihood "does not need to be zero" but "should be insignificant in reality"; a means is not reasonably likely when identification is prohibited by law or impossible in practice | Every risk is reported with a confidence interval and judged on its upper bound | PRV-08, PRV-12 | Choosing the limits (relsynth's defaults are suggestions, docs/privacy.md) | +| EDPB paras. 11–15 (perspective), para. 12 ("for whom is the data intended to be anonymous?"); CJEU C-413/23 P paras. 75–77, 86 | Anonymity is assessed from the perspective of those who will have access; pseudonymised data may be non-personal for a recipient only if measures "actually put in place" prevent attribution, "including by recourse to other means of identification such as cross-checking" (para. 77) | A report page per release context (internal, partner, public) with the attacker's knowledge set per context | PRV-01, PRV-12 | Defining each recipient and what they hold (the context's `attacker_knows`) | +| CJEU C-413/23 P paras. 84–85; EDPB para. 21 | Data become personal when put at the disposal of someone with means to identify, both for that recipient and, indirectly, for the provider; an onward transfer must be assessed for the third party | External contexts assume the attacker knows every attribute but the one guessed, plus population data; they are blocked without DP | PRV-13, DP-02 | Onward-transfer terms; **Lawyer** for transfers outside the EU | +| CJEU C-413/23 P para. 111 | For the information duty (Art. 15(1)(d) Reg. 2018/1725, Art. 13–14 GDPR), identifiability is assessed at collection and from the controller's point of view | Nothing: this concerns the source data | | Privacy notices must name synthetic-data recipients where the source data remain personal for the controller | +| EDPB paras. 26–34 (means reasonably likely); para. 30 (relevant entities include rogue employees, journalists, intelligence agencies, unethical companies, cybercriminals); para. 31 (no "lack of motivation") | Means are read broadly; motivation is not a factor; a contractual ban is not a legal prohibition (para. 34) | Contexts never assume a less motivated attacker; the partner context assumes everything but the target attribute | PRV-12 | Whether legal prohibitions apply and are enforced (para. 33) | +| EDPB para. 35 | Risk can rise over time; "good practice ... to periodically reassess the likelihood of re-identification" | Reports and manifests carry dates, versions and the policy digest, so a release can be re-assessed with the same code | GOV-06 | A reassessment schedule (DPIA section 9) | +| EDPB para. 36 | A dataset is anonymous only if "the anonymisation is effective for all of the included individuals" | The ten most exposed people are attacked individually; in partner and public contexts the most exposed person gates the verdict | PRV-08 | | +| EDPB para. 37 | A controller that keeps the source still processes personal data | Nothing: the source and the privacy report stay with the controller | GOV-07 | GDPR compliance for the source and the generation | +| EDPB para. 38 | "anonymisation must have a legal basis under Article 6 GDPR" (Art. 9(2) for special categories), presumed to be that of the preceding processing when it is part of the same activity and purposes | | | **Lawyer**: the legal basis for fitting the model | +| EDPB para. 40 | Do not call data "anonymous" if people are still identifiable | The report says *pass*, *review*, *fail* or *blocked* per context, never "anonymous" | PRV-12 | Wording of any release notice | +| EDPB para. 41 | "adequate documentation of the anonymisation processing", including testing, retained after completion | Signed manifest per run, privacy report, DP statement, quality report, this assurance case | GOV-06, DP-08, PRV-01 | Retention of the registry | +| EDPB para. 42; GDPR Art. 33–34 | A security incident may trigger reassessment | Tamper-evident registry; release revocation | GOV-09, GOV-08 | Incident response | + +### 1.2 The three technical criteria (EDPB section 3.4) + +| Criterion | EDPB text | relsynth attack | Evidence | Notes | +|---|---|---|---|---| +| No record isolation (sec. 3.4.1, paras. 54–57) | "met if the data does not contain a unique combination of attribute values that relate to a single individual" (para. 55) | Singling-out queries built from the synthetic data, univariate and multivariate; success on training people above unseen control people | PRV-04, PRV-11 | Section 3.5.3, which explains singling out "in detail" (fn. 37), was not available in the text read | +| No linkage (sec. 3.4.2, paras. 58–65) | "likely to be met if the information has not been recorded elsewhere and is not correlated to similar information" (para. 65); "noisy data may leak information in unexpected ways" (para. 64) | Nearest-neighbour linkage of two partial records through the synthetic data | PRV-05 | Paras. 58–63 were not read verbatim | +| No inference (sec. 3.4.3, paras. 66–80) | "met if no specific and meaningful inference can be drawn" (para. 67); test by comparing outputs with and without the person (para. 71(b)); membership inference (para. 79) | Attribute inference against a population baseline (only the excess over population data counts); shadow-model, LiRA-style and DOMIAS membership inference | PRV-06, PRV-07 | Para. 82: inference from "AI models or synthetic data" is the case this criterion is written for | +| Applying the criteria (sec. 3.5, paras. 83–94) | Factors: dimensionality, resolution, number of individuals, additional information (para. 85); success "against a single individual" suffices (para. 87); "adequate safety margins" (para. 88); public data is open to anyone (para. 89); access restrictions "are not by themselves sufficient" (para. 90) | Per-person results, upper confidence bounds, stricter limits for public than partner, small-population rule | PRV-08, PRV-12, PRV-15 | Para. 92: agentic AI lowers attack cost; relsynth's attacks are fixed and will not cover every future method (PRV-19) | +| Compiling the results (sec. 3.5.3) and Annex 1 flowchart | Not available in the text read | The verdict per context is the worst of its checks | PRV-12 | Re-check against the final text (section 7) | + +The guidelines do not mention differential privacy in the parts read. A DP guarantee bounds every membership test (Prop. 603) and so supports the No Inference criterion (para. 71(b) compares outputs with and without a person, which is the DP definition), but the EDPB does not say that DP alone makes data anonymous. **Lawyer**: whether a DP release with the organisation's ε meets the guidelines' test. + +### 1.3 Other GDPR duties around a synthetic release + +| Provision | Relevance | What relsynth contributes | +|---|---|---| +| Art. 5(1)(b), 5(1)(c) (purpose limitation, minimisation) | The source is read for a new purpose | Pushdown reads aggregates and a sample, not whole facts (SEC-10); DP mode reads each owned table once | +| Art. 25 (data protection by design and by default) | Choice of a privacy-enhancing technique | External releases need DP by default (DP-02); privacy checks and copy guards on by default (SEC-02) | +| Art. 32 (security of processing) | The generation reads production data | docs/security.md (SEC-01 to SEC-15) | +| Art. 35 (DPIA) | Likely high risk when special categories or large-scale data are processed; Art. 35(7) sets the content | [DPIA template](dpia-template.md), built on Art. 35(7)(a)–(d) | +| Art. 89(1) | Safeguards for research and statistics, including pseudonymisation | DP mode, per-person attacks | + +## 2. EU AI Act, Article 10 (data and data governance) + +Regulation (EU) 2024/1689, as amended by the Digital Omnibus on AI, Regulation (EU) 2026/1744 of 8 July 2026 (OJ L, 24 July 2026; in force 27 July 2026). The amendment **deleted Article 10(5)** and moved the bias-detection basis for special categories of personal data to a **new Article 4a**. Article 10's obligations apply from 2 December 2027 for Annex III high-risk systems and 2 August 2028 for Annex I systems (Art. 113(c) as amended). + +| Provision | What it requires | What relsynth contributes | Evidence | Left to the provider | +|---|---|---|---|---| +| Art. 10(2)(a)–(c) | Governance of design choices, data origin and original purpose, preparation operations | The spec, the IR document and the signed manifest record source, spec, versions, seeds and outputs of each synthetic dataset | GOV-06 | The provider's data governance documentation (Annex IV) | +| Art. 10(2)(d)–(e) | Assumptions about what the data represent; availability, quantity, suitability | The quality report states what is matched (marginals, dependence, joins, rules) and with what limits | | Whether the synthetic data are suitable for the intended purpose | +| Art. 10(2)(f)–(g) | Examine possible biases; measures to detect, prevent and mitigate them | Synthetic data preserve the source's group statistics, so bias found in synthetic data reflects the source; DP mode suppresses rare levels and caps rows per parent, which pulls data towards typical people | DP-15 | **Bias**: DP mode is known to under-represent small groups (NIST SP 800-226 sec. 3.3); do not use DP synthetic data alone for bias detection on small groups | +| Art. 10(2)(h) | Identify data gaps and shortcomings | Failed checks and the doctor's causes | | | +| Art. 10(3) | Relevant, sufficiently representative, free of errors, complete; appropriate statistical properties, including for the groups the system is used on | Calibrated scorecard per table and join | | **Statistical**: representativeness for the target population is the provider's to show | +| Art. 10(4) | Characteristics of the geographical, contextual, behavioural or functional setting | `--scale` and per-table sizes can generate test data for settings not in the source; their realism is not guaranteed | | | +| Art. 4a(1)(a) (former Art. 10(5)(a)) | Special categories may be processed for bias detection only if this "cannot be effectively fulfilled by processing other data, including synthetic or anonymised data" | Synthetic data are the "other data" this condition asks the provider to try first; the quality report is the evidence of whether they suffice | | **Lawyer**: the record of why synthetic data were or were not sufficient (Art. 4a(1)(f)) | +| Art. 59(1)(b) | Sandbox processing of personal data only where requirements "cannot effectively be fulfilled by processing anonymised, synthetic or other non-personal data" | Same as above | | | + +## 3. HIPAA de-identification (expert determination) + +| Provision | What it requires | What relsynth contributes | Left to the covered entity | +|---|---|---|---| +| 45 CFR 164.514(a) | De-identified information: "no reasonable basis to believe that the information can be used to identify an individual" | | | +| 164.514(b)(1)(i) | An expert "determines that the risk is very small that the information could be used, alone or in combination with other reasonably available information, by an anticipated recipient to identify an individual" | The release context is the anticipated recipient; the partner/public contexts assume the attacker holds every other attribute and population data | **Expert**: relsynth is not the expert; a qualified person must make the determination (HHS guidance sec. 2.2) | +| 164.514(b)(1)(ii) | The expert "documents the methods and results of the analysis" | Privacy report (JSON, Markdown, HTML) with every attack, interval and setting; DP statement; manifest (PRV-01, DP-08, GOV-06) | The expert's own determination letter | +| HHS guidance sec. 2.3 | "There is no explicit numerical level of identification risk" | Limits are configurable per context; defaults are suggestions | The threshold | +| HHS guidance sec. 2.4 | Time-limited determinations are common practice | Each report is dated and versioned | The validity period | +| HHS guidance sec. 2.6, Table 1 | Principles: replicability, data source availability, distinguishability | Distinguishability ≈ record isolation (PRV-04); data source availability ≈ the context's `attacker_knows` | Replicability of each attribute in the real world | +| 164.514(b)(2) (Safe Harbor) | Removal of 18 identifiers (names, geographic units below a state, all date elements except year, ...) and no actual knowledge | Not applicable as such: synthetic dates and places are generated, not removed. Direct identifiers declared `type: pii` are generated from a format (`email`, `person_name`, `phone`, `uuid`, or a pattern shape), not copied from source values | Safe Harbor is a different route; use expert determination for synthetic data | +| 164.514(c) | Re-identification codes not derived from the person | Synthetic keys are new labels; the privacy report's person labels are HMAC under a secret salt and never leave the organisation (PRV-03) | | + +## 4. NIST SP 800-226: evaluating differential privacy guarantees + +NIST SP 800-226 (March 2025) organises a DP guarantee as a pyramid (Fig. 1): unit of privacy and ε at the top, algorithms in the middle, deployment at the base. relsynth's DP statement follows it (DP-08). + +| Section | Hazard or requirement | relsynth | Evidence | +|---|---|---|---| +| 2.2 The privacy parameter ε | No rule for choosing ε; large ε may not protect | No default ε; the organisation sets it in a signed policy; above 1, measure with `relsynth privacy` | DP-03, GOV-10 | +| 2.3 Variants | Variants convert to (ε, δ) | zCDP accounting converted with Canonne, Kamath and Steinke (2020); δ ≤ 10⁻⁵ | DP-08, DP-03 | +| 2.4 Unit of privacy | User-level is stronger than event-level | User level: one person of the privacy unit and every row they own | DP-01, PRV-09 | +| 2.5 Comparing guarantees | A large δ misleads | δ capped at 10⁻⁵ and stated with ε | DP-03 | +| 2.6 Mixing with other releases | DP does not cover non-DP releases | The ledger counts non-DP releases of a domain separately and the guide says they are not covered | GOV-11 | +| 2.7 Auditing and empirical measures | Audits complement proofs | `--assess` compares attack TPR with the bound ε implies; Contoso ε = 1: 0.014 against 0.027 | DP-10, DP-14 | +| 3.2 Utility and accuracy | Report utility | Utility against ε on three real databases | DP-15 | +| 3.3 Bias | DP can magnify systemic bias | Rare levels suppressed and rows capped; stated in docs/dp.md section 5 | DP-15 | +| 3.6 Synthetic data | Two-stage measure-then-generate designs | Stage 1 noisy statistics, stage 2 post-processing | DP-06 | +| 4.2 Trust models | Central model trusts the curator | Central model: relsynth runs inside the organisation, which holds the data | | +| 4.3 Implementation challenges | Floating point, side channels; use validated libraries | Discrete Gaussian on integers from OpenDP; noise from the OS CSPRNG, not seeded | DP-05 | +| 4.4 Data security and access control | A breach of raw data voids the guarantee | docs/security.md; registry and keys | SEC-01, GOV-02 | +| 4.5 Data collection exposure | DP does not protect collection | Out of scope; stated | | + +## 5. NIST SP 800-188: de-identifying government datasets + +| Section | Topic | relsynth | Evidence | +|---|---|---|---| +| 3.2 | Risks and benefits of a release; probability and impact of re-identification | The DPIA template asks for both; the privacy report measures probability | PRV-01 | +| 3.4 | Data-sharing models: publish de-identified data, publish synthetic data, query interface, enclave | The three release contexts map to these models ([release guidance](release-guidance.md)) | | +| 3.5 | The Five Safes | Release guidance section 2 maps projects, people, settings, data and outputs | | +| 3.6 | Disclosure Review Boards | Approvals with quorum and separation of duties play the board's signing role | GOV-04 | +| 4.3.11 | Composition | DP budget per privacy domain with a lifetime cap | GOV-08, GOV-10 | +| 4.3.13 | Post-release monitoring | Manifests and revocation; reassessment schedule in the DPIA | GOV-06 | +| 4.4.4–4.4.5 | Fully synthetic data; synthetic data with validation | relsynth produces fully synthetic data (no row copied); validation against the source is the quality report, kept inside the organisation | SEC-02 | +| 4.4.7 | Synthetic data with DP | DP mode | DP-01 | +| 4.6.2–4.6.3 | Validating privacy; re-identification studies | The attack suite is a re-identification study run on every release | PRV-01 | +| 5 | Evaluate the software | This assurance case and the claim register | | + +## 6. ISO/IEC 27559:2022 and ISO/IEC 20889:2018 + +The texts are sold by ISO and were not available for this mapping; ISO's catalogue gives only their scope. The rows below map by topic. **Clause numbers must be added from the purchased standards before this table is used in an audit.** + +| Standard | Topic (from the published scope) | relsynth | +|---|---|---| +| ISO/IEC 27559 | A framework for identifying and mitigating re-identification risk across the lifecycle of de-identified data, for PII controllers and processors | Context assessment = release contexts and attacker knowledge; data assessment = the privacy unit and owned tables (PRV-09); identifiability assessment = the attack suite (PRV-04 to PRV-07); mitigation = DP mode (DP-01); governance and monitoring = registry, policy, approvals, revocation (GOV-01 to GOV-09) | +| ISO/IEC 20889 | Terminology and classification of de-identification techniques, including synthetic data and formal privacy models | relsynth's techniques in its terms: fully synthetic data from a statistical model; differential privacy as the formal model; the attacks measure singling out, linking and inference | + +## 7. Tracking the final EDPB guidelines + +The consultation on Guidelines 02/2026 closed on 30 October 2026. When the EDPB adopts the final version: + +1. Compare the final section 3.4 and 3.5 with the rows of section 1.2, paragraph by paragraph (the numbering may change). +2. Read section 3.5.3 and Annex 1, which were not available, and decide whether relsynth's per-context verdict compiles results the same way. +3. Check whether the final text mentions differential privacy or synthetic data beyond para. 82. +4. Update the paragraph numbers quoted in docs/privacy.md; the claim register then flags every quote that changed. +5. Record the change and its date in this section. + +| Date | Version checked | Change | +|---|---|---| +| 2026-10-31 | v1.0 (consultation draft) | First mapping | + +## Sources + +- EDPB, *Guidelines 02/2026 on Anonymisation*, version 1.0, adopted 7 July 2026 for public consultation. https://www.edpb.europa.eu/system/files/2026-07/edpb_guidelines_202602_anonymisation_v1_en_0.pdf +- Court of Justice of the EU, judgment of 4 September 2025, *EDPS v SRB*, C-413/23 P, ECLI:EU:C:2025:645. https://eur-lex.europa.eu/legal-content/EN/TXT/HTML/?uri=CELEX:62023CJ0413 +- Regulation (EU) 2016/679 (GDPR), Art. 4, 5, 6, 9, 25, 32, 35, 89 and Recital 26. +- Regulation (EU) 2024/1689 (AI Act), Art. 10, 59, 113; consolidated text as amended by Regulation (EU) 2026/1744 (Digital Omnibus on AI), Art. 4a. https://ai-act-service-desk.ec.europa.eu/en/ai-act/article-10 and /article-4a +- 45 CFR 164.514; HHS Office for Civil Rights, *Guidance Regarding Methods for De-identification of Protected Health Information in Accordance with the HIPAA Privacy Rule*, 26 November 2012. https://www.hhs.gov/hipaa/for-professionals/special-topics/de-identification/index.html +- J. Near, D. Darais, N. Lefkovitz, G. Howarth, *Guidelines for Evaluating Differential Privacy Guarantees*, NIST SP 800-226, March 2025. https://csrc.nist.gov/pubs/sp/800/226/final +- S. Garfinkel et al., *De-Identifying Government Datasets: Techniques and Governance*, NIST SP 800-188, September 2023. https://csrc.nist.gov/pubs/sp/800/188/final +- ISO/IEC 27559:2022, *Privacy enhancing data de-identification framework*. https://www.iso.org/standard/71677.html +- ISO/IEC 20889:2018, *Privacy enhancing data de-identification terminology and classification of techniques*. https://www.iso.org/standard/69373.html diff --git a/docs/assurance/release-guidance.md b/docs/assurance/release-guidance.md new file mode 100644 index 0000000..8721a62 --- /dev/null +++ b/docs/assurance/release-guidance.md @@ -0,0 +1,67 @@ +# Release guidance: internal, partner and public + +Who a synthetic dataset goes to decides how it must be made. The CJEU (*EDPS v SRB*, C-413/23 P, paras. 75–77 and 86) and the EDPB (Guidelines 02/2026, paras. 11–15 and 29) assess identifiability from the perspective of those who will have access, so relsynth judges every release for a named **context**. This page says, for each of the three built-in contexts, what the organisation's policy enforces, what to configure and what to check before approving. It is guidance for the people who approve releases, not legal advice. + +## 1. The three contexts at a glance + +| | internal | partner | public | +|---|---|---|---| +| Recipient | Staff of the organisation, for development, testing or analysis | A named organisation under contract | Anyone (open data, publication, a model or dataset shared openly) | +| NIST SP 800-188 sec. 3.4 sharing model | Restricted access inside the organisation | Restricted access under a data use agreement | Publishing synthetic data | +| Attacker assumed to know | The quasi-identifiers | Every attribute except the one guessed, plus population data | Same as partner | +| Differential privacy | Optional | **Required** (enforced before any data is read, DP-02) | **Required** | +| Default limits (`risk`, `mia_auc`, `record_auc`) | 0.10, 0.65, 0.85 | 0.05, 0.60, 0.75 | 0.02, 0.55, 0.65 | +| The most exposed person | Shown; the pooled top-10 AUC gates | Gates the verdict, with 128 test shadows | Gates the verdict, with 128 test shadows | +| Fewer than 1,000 people in the unit | Warning | Fails without DP | Fails without DP | +| Approval in the example policy | Only for a *review*, one DPO or privacy officer | Every release, one DPO | Every release, two DPOs | +| Spec | May be drafted by `introspect` | Must be written from public facts (`spec_is_public: true`) | Same as partner | + +The limits are relsynth's suggestions; the organisation sets its own in the signed policy (docs/governance/examples/org-policy.yaml). An organisation can define further contexts, for example a supervisory authority; an external context always requires DP (GOV-04). + +## 2. The Five Safes for each context + +NIST SP 800-188 sec. 3.5 recommends the Five Safes as a frame for a release decision. + +| Safe | internal | partner | public | +|---|---|---|---| +| Safe projects | The purpose is recorded in the DPIA | Purpose in the DPIA and the contract | Purpose in the DPIA; publication is irreversible (SP 800-188 sec. 1: a published release "will remain in the public domain indefinitely") | +| Safe people | Staff with a need; access to the target controlled | Named recipients; contract forbids re-identification and onward transfer (a contract is not a legal prohibition, EDPB para. 34, so it does not lower the attacker model) | No control: assume anyone, including the relevant entities of EDPB para. 30 | +| Safe settings | `allowed_targets` in the policy (for example a dev sandbox) | `allowed_targets`, for example a partner SFTP | An open data portal | +| Safe data | Measured risk below the internal limits | DP at the organisation's ε, and measured risk below the partner limits | DP at a smaller ε than partner, and measured risk below the public limits | +| Safe outputs | The privacy report and model stay inside | Package holds only the synthetic tables, manifest, approvals, token and DP statement (GOV-07) | Same as partner | + +## 3. Before approving: a checklist + +For every context: + +1. `relsynth gov verify registry/` passes. +2. The run was assessed (`--assess` or `--privacy-report`) against this context, and the verdict is *pass*, or *review* with a named sign-off. +3. The privacy unit is the right table, and every table that holds personal data is owned by it or listed as public with a reason (`relsynth privacy` lists the owned tables; DP mode refuses to guess). +4. Tables with `mode: copy` hold no personal data. +5. The quality report is fit for the purpose. A dataset that is wrong in a way that matters to decisions about people is a risk in itself. +6. The DPIA is filled in or updated ([template](dpia-template.md)). + +For partner and public, also: + +7. The run is a DP run, and the DP statement's unit, ε and δ are what the DPIA states. +8. The spec was written from public facts (`spec_is_public: true`), and the facts declared under `public` really are public. +9. `relsynth gov budget` shows the domain still below its lifetime cap after this release, and the number of earlier non-DP releases of the domain (they are not covered by the DP accounting). +10. Free text, binary and `pii` columns are placeholders in DP mode; check the recipient does not need them. +11. For public: no smaller audience would do. Publication cannot be recalled. + +## 4. When the answer is no + +| Situation | What to do | +|---|---| +| Verdict *fail* in internal | Drop or generalise the attributes the report names; raise `min_token_rows` for text; or use DP for the internal release too | +| *review* that nobody can sign | Run more shadows (`n_test_gated`) to narrow the intervals, or treat it as a fail | +| Small population (under 1,000 people) | Internal only, or DP with a small ε | +| The budget is exhausted | No further DP release of that domain; the lifetime cap never resets (GOV-10) | +| A released dataset turns out to be risky | `relsynth gov revoke`; for partner, ask the recipient to delete under the contract; for public, revocation cannot recall copies | + +## 5. What relsynth cannot decide + +- Whether a given ε is acceptable for this recipient and these people. NIST SP 800-226 sec. 2.2 gives no rule; the organisation decides (A4 of the [assurance case](assurance-case.md)). +- Whether the result meets the legal test of anonymity in a given jurisdiction (EDPB para. 22: "insignificant in reality"). **Lawyer.** +- Whether onward transfers by the recipient are lawful. **Lawyer.** +- Whether a HIPAA expert determination can rely on the report; that is the expert's call (45 CFR 164.514(b)(1)). diff --git a/docs/privacy.md b/docs/privacy.md index 039970e..9860688 100644 --- a/docs/privacy.md +++ b/docs/privacy.md @@ -2,6 +2,8 @@ `relsynth privacy` attacks a synthetic release the way a recipient could, person by person, and writes a privacy report with one page per release context (internal, partner, public). A data protection officer can read the pages without the annex; the annex and the JSON file hold the technical results. +For the people who approve a release, the assurance case, the regulation mapping and a DPIA template are in [docs/assurance](assurance/README.md). + ```bash relsynth privacy spec.yaml --config privacy.yaml --out privacy_report --cache .relsynth-cache --jobs 4 relsynth privacy spec.yaml --config privacy.yaml --context internal --fail-on review # CI gate: exit 1 diff --git a/tests/unit/test_assurance.py b/tests/unit/test_assurance.py index edb7871..e7e229c 100644 --- a/tests/unit/test_assurance.py +++ b/tests/unit/test_assurance.py @@ -108,6 +108,16 @@ def test_checker_catches_drift(tmp_path, doc, register, expect): assert any(expect in p for p in res.problems), res.problems +def test_checker_catches_bad_citations_in_the_assurance_documents(tmp_path): + root = _mini_repo(tmp_path, GOOD, "Every key resolves. Run it every night.\n") + (root / "docs/assurance/case.md").write_text("G1 rests on CL-1, PRV-99, `prop:b`, `prop:gone`, C900 and C777.\n") + problems = cc.run(root).problems + assert any("cites PRV-99" in p for p in problems) + assert any("cites prop:gone" in p for p in problems) + assert any("cites check C777" in p for p in problems) + assert not any("prop:b," in p or "C900" in p for p in problems) + + # Modules allowed to open network connections, and why (docs/security.md). NETWORK_ALLOWED = { "relsynth/service/auth.py": "OIDC discovery and keys of the identity provider the operator configures", diff --git a/tools/claims_check.py b/tools/claims_check.py index 132ecd5..e90cf31 100644 --- a/tools/claims_check.py +++ b/tools/claims_check.py @@ -273,6 +273,17 @@ def doc_text(name: str) -> str | None: for s in res.unclaimed.get(doc, []): res.problems.append(f"{doc}: unregistered claim: \"{s[:110]}\"") + ids = {c.get("id") for c in reg.get("claims", [])} + for path in sorted((root / REGISTER.parent).glob("*.md")): + rel = path.relative_to(root).as_posix() + text = path.read_text(encoding="utf-8") + for ref in sorted(set(re.findall(r"\b(?:PRV|DP|SEC|GOV|RM)-\d+\b", text)) - ids): + res.problems.append(f"{rel}: cites {ref}, which the register does not have") + for label in sorted(set(re.findall(r"`(prop:[\w-]+)`", text)) - set(props)): + res.problems.append(f"{rel}: cites {label}, which the proofs do not define") + for cid in sorted(set(re.findall(r"\bC\d{2,4}\b", text)) - checks): + res.problems.append(f"{rel}: cites check {cid}, which does not exist") + for path in sorted(root.glob("*.md")) + sorted((root / "docs").rglob("*.md")): rel = path.relative_to(root).as_posix() if rel.startswith(("docs/research/", "CHANGELOG")): From 038ee5c56570b2e91df905b8194d0887cd3efc98 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 19:04:29 +0000 Subject: [PATCH 4/8] docs(readme): link the compliance evidence in the documentation map Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PSVqwE13trs5MdpBoE4YyJ Signed-off-by: Claude --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 7837bdc..34a967a 100644 --- a/README.md +++ b/README.md @@ -419,6 +419,7 @@ What each test module covers is in [docs/testing.md](docs/testing.md). | [docs/datasets.md](docs/datasets.md) | Which datasets may be used, committed and where | | [CONTRIBUTING.md](CONTRIBUTING.md) | How to contribute: workflow, commits, sign-off, tests | | [SECURITY.md](SECURITY.md) | Reporting a vulnerability | +| [docs/assurance/](docs/assurance/README.md) | For privacy, legal and audit teams: the assurance case, the regulation mapping (GDPR and EDPB 02/2026, AI Act, HIPAA, NIST, ISO), a DPIA template, release guidance and the claim register | ## Licence From 74d5f95c3220a7b56255cd8d9823fa876b1aa9a7 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 19:05:08 +0000 Subject: [PATCH 5/8] fix(assurance): accept proposition numbers named in titles, flag backward ranges Sections may name a fixed number in the title ([...; Prop.~950]) while LaTeX prints the running number; references to the named number are valid. A reference such as 'Props. 951-902' is now reported. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PSVqwE13trs5MdpBoE4YyJ Signed-off-by: Claude --- tools/claims_check.py | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/tools/claims_check.py b/tools/claims_check.py index e90cf31..5cb71ab 100644 --- a/tools/claims_check.py +++ b/tools/claims_check.py @@ -113,25 +113,33 @@ def _expand(path: Path) -> str: return "\n".join(out) -def proposition_numbers(root: Path = ROOT) -> dict[str, int]: - """Label -> printed number of every proposition in the proofs document.""" +def proposition_numbers(root: Path = ROOT, aliases: dict[str, int] | None = None) -> dict[str, int]: + """Label -> printed number of every proposition in the proofs document. + + A section may also name a fixed number in the title, ``[Worker independence; + Prop.~950]``, while LaTeX prints the running number; such numbers are + put in ``aliases`` (label -> number) when it is given. + """ text = _expand(root / PROOFS / "relsynth-proofs.tex") token = re.compile( - r"\\begin\{proposition\}|\\setcounter\{(\w+)\}\{(?:\\value\{(\w+)\}|(-?\d+))\}|\\label\{(prop:[^}]+)\}" + r"\\begin\{proposition\}(\[[^\]]*\])?|\\setcounter\{(\w+)\}\{(?:\\value\{(\w+)\}|(-?\d+))\}|\\label\{(prop:[^}]+)\}" ) counter, saved, labels, pending = 0, {}, {}, False for m in token.finditer(text): if m.group(0).startswith("\\begin"): counter += 1 pending = True - elif m.group(1): - value = counter if m.group(2) == "proposition" else saved.get(m.group(2), 0) if m.group(2) else int(m.group(3)) - if m.group(1) == "proposition": + title = re.search(r"Prop\.~?\s*(\d{3,5})\]", m.group(1) or "") + elif m.group(2): + value = counter if m.group(3) == "proposition" else saved.get(m.group(3), 0) if m.group(3) else int(m.group(4)) + if m.group(2) == "proposition": counter = value else: - saved[m.group(1)] = value + saved[m.group(2)] = value elif pending: - labels[m.group(4)] = counter + labels[m.group(5)] = counter + if title and aliases is not None: + aliases[m.group(5)] = int(title.group(1)) pending = False return labels @@ -196,8 +204,9 @@ def _resolve(kind: str, value: str, root: Path, props: dict[str, int], checks: s def run(root: Path = ROOT) -> Result: res = Result() reg = yaml.safe_load((root / REGISTER).read_text(encoding="utf-8")) - res.props = props = proposition_numbers(root) - numbers = set(props.values()) + aliases: dict[str, int] = {} + res.props = props = proposition_numbers(root, aliases) + numbers = set(props.values()) | set(aliases.values()) checks = check_ids(root) sym_cache: dict = {} docs: dict[str, str] = {} @@ -246,6 +255,8 @@ def doc_text(name: str) -> str | None: item = {"kind": kind, "ref": value} if kind == "prop" and value in props: item["number"] = props[value] + if value in aliases: + item["named"] = aliases[value] resolved.append(item) if c.get("status") in ("supported", "partial", "limitation") and not set(kinds) & set(STRONG_KINDS): res.problems.append(f"{where}: a claim about relsynth needs a prop, check, test or report") @@ -291,7 +302,9 @@ def doc_text(name: str) -> str | None: for m in PROP_REF.finditer(path.read_text(encoding="utf-8")): for a, b in re.findall(r"(\d+)(?:\s*[–-]\s*(\d+))?", m.group(1)): lo, hi = int(a), int(b or a) - for n in (lo, hi): + if hi < lo: + res.problems.append(f"{rel}: '{m.group(0).strip()}' is a range that runs backwards") + for n in sorted({lo, hi}): if n not in numbers: ref = m.group(0).strip() res.problems.append(f"{rel}: '{ref}' names Proposition {n}, which the proofs do not define") From 49662bbbc963e301d06102a827f8de770d13bd31 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 21:23:09 +0000 Subject: [PATCH 6/8] docs(settings): regenerate the settings reference test_settings_reference_is_up_to_date fails on main since the runtime settings (#35) landed after the reference was generated (#39); the unit jobs of every PR are red until it is regenerated. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PSVqwE13trs5MdpBoE4YyJ Signed-off-by: Claude --- docs/settings-reference.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/settings-reference.md b/docs/settings-reference.md index 1967b40..7effb3a 100644 --- a/docs/settings-reference.md +++ b/docs/settings-reference.md @@ -105,6 +105,11 @@ Generated from `relsynth.settings` (`python -m relsynth.settings > docs/settings | `compute.preset` | str | `cpu` | | | cpu | gpu-auto | gpu-single | gpu-multi, or a preset of compute.file: the devices generator plug-ins may use. The default engine is CPU-only; see docs/compute.md. | | `compute.file` | str | `None` | | | A compute.yaml with more presets (relative to the spec). | | `compute.overrides` | map | `{}` | | | Expert settings that replace the preset's fields: device, gpus, devices, backend, precision, batch_size, memory_limit, threads, fallback. | +| `runtime.executor` | str | `none` | | | none | local | process | spark: none (default) samples in one process as relsynth 1.0; the others generate the facts in partitions (docs/runtime.md): local in this process, process in a pool of worker processes, spark as the tasks of a Spark job. The same seed gives the same data with any executor and number of workers. | +| `runtime.workers` | any | `None` | | | Worker processes (process) or tasks at once (spark); default: the CPUs (process) or the Spark context's default parallelism. | +| `runtime.partition_rows` | int | `1000000` | | | Fact rows per partition (about); part of the output's identity: change it and the draws change. | +| `runtime.start` | any | `None` | | | process executor: fork (default on Linux), spawn or forkserver. | +| `runtime.master` | any | `None` | | | spark executor: the master URL when no Spark session is active (default local[workers]). | ## `evaluation.thresholds` From 2e21a9b6c22128250dc16deb64dd4c295430871a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 31 Oct 2026 21:34:14 +0000 Subject: [PATCH 7/8] test(service): do not scan ciphertext for the source marker test_no_source_row_is_persisted searched every file under the data folder, including the encrypted artefacts, for a 3-byte marker. Random ciphertext contains it by chance (macOS unit job on #41); the artefacts are already checked decrypted. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PSVqwE13trs5MdpBoE4YyJ Signed-off-by: Claude --- tests/unit/service/test_service_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/service/test_service_api.py b/tests/unit/service/test_service_api.py index 51e276b..d7b1d58 100644 --- a/tests/unit/service/test_service_api.py +++ b/tests/unit/service/test_service_api.py @@ -116,7 +116,8 @@ def test_no_source_row_is_persisted(client, flow, settings, store): for a in store.artifacts("acme", flow["run"]["id"]): assert SECRET_MARK.encode() not in arts.read_bytes("acme", a), a["name"] for f in Path(settings.data_dir).rglob("*"): - if f.is_file(): + # Ciphertext is checked decrypted above: random bytes contain a short marker by chance + if f.is_file() and f.suffix != ".enc": assert SECRET_MARK.encode() not in f.read_bytes(), f assert not list((Path(settings.data_dir) / "work").rglob("*.csv")) # plaintext work files are gone dump = "\n".join(store.engine.raw_connection().driver_connection.iterdump()) From b0dc8f20e3255a5f7cb766c8b865eba35ca9c513 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 02:29:55 +0000 Subject: [PATCH 8/8] docs(assurance): register the README claims and fix the fixture encoding on Windows Put README.md in the claim register's scope with 36 entries (RM-01..RM-36). A README sentence that sums up a guide cites that guide's claim with the new {claim: ID} evidence kind, which is as strong as the claim it cites. README fixes found while registering: 'No row is copied' becomes 'Rows are drawn from a fitted model; only tables you declare mode: copy are copied'; 'Privacy checks run by default' becomes copy guards; C700-C706 do not exist, the 2.0 checks are C600-C606 and the generator propositions are unit-tested. The checker tests wrote their fixture files in the platform encoding, which broke on Windows (cp1252); they now write UTF-8. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PSVqwE13trs5MdpBoE4YyJ Signed-off-by: Claude --- CHANGELOG.md | 3 +- README.md | 6 +- docs/assurance/README.md | 2 +- docs/assurance/claims.yaml | 513 +++++++++++++++++++++++++++++++++++ docs/development/ci.md | 2 +- tests/unit/test_assurance.py | 41 ++- tools/claims_check.py | 19 +- 7 files changed, 571 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7608d9d..decf5b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,10 +7,11 @@ All notable user-visible changes are recorded here, following [Keep a Changelog ### Added - **Compliance evidence and assurance case** (`docs/assurance/`; roadmap T17): an assurance case for a release (claims, evidence, assumptions, defeaters), a mapping to EDPB Guidelines 02/2026 and *EDPS v SRB*, the AI Act (Art. 10 and the new Art. 4a), HIPAA expert determination, NIST SP 800-226 and SP 800-188 and ISO/IEC 27559 and 20889, a DPIA template for a synthetic release, and release guidance for internal, partner and public recipients. -- **Claim register** (`docs/assurance/claims.yaml`, `tools/claims_check.py`, CI job `lint`): every claim of the privacy, DP, security and governance guides traced to a proposition, check, test or measured report; CI fails when a claim changes, its evidence disappears, or a new claim is not registered. +- **Claim register** (`docs/assurance/claims.yaml`, `tools/claims_check.py`, CI job `lint`): every claim of the README and of the privacy, DP, security and governance guides (96 claims) traced to a proposition, check, test or measured report; a README sentence that sums up a guide cites that guide's claim. CI fails when a claim changes, its evidence disappears, or a new claim is not registered. ### Changed +- README: "No row is copied" now reads "Rows are drawn from a fitted model; only tables you declare `mode: copy` are copied"; "Privacy checks run by default" now says the copy guards run by default and are not a privacy check; the 2.0 science checks are C600–C606 (privacy attacks), not C700–C706, which do not exist (the generator propositions are tested in `tests/unit/test_generators.py`). - `docs/security.md` now describes 2.0: differential privacy is optional rather than absent, the distance guards are copy guards rather than privacy checks, seven core dependencies, four `subprocess` call sites, and the service's connections to the OIDC provider and Vault. ## 2.0.0 (2026-10-31) diff --git a/README.md b/README.md index 9972f77..dce9a62 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ relsynth run spec.yaml --target "sqlite:synthetic.db" --fail-under 0.98 - **Statistics.** Each table's marginals and dependence come from a Bayesian Gaussian copula over mixed types. Links between tables come from latent segments, so customers who buy more of one product kind keep doing so. Counts, participation and multiplicities follow fitted laws. - **Bayesian.** Each synthetic dataset is one draw from the posterior predictive distribution. Several datasets show the model's own uncertainty, and `posterior.mode: mean` gives stable test data. - **Scale.** You set the size per table: `--scale 0.1` for a small test database, or `--scale 10` for load tests. Facts follow the tables they hang from by the fitted rates. -- **Privacy.** No row is copied. Distance-to-closest-record checks compare synthetic rows with real ones, against the real data's own spacing. Text cannot reproduce rare source content, and memorisation guards are on by default. `relsynth privacy` attacks a release per person (membership, attribute inference, singling out, linkability) and writes a report per release context. +- **Privacy.** Rows are drawn from a fitted model; only tables you declare `mode: copy` are copied. Distance-to-closest-record checks compare synthetic rows with real ones, against the real data's own spacing. Text cannot reproduce rare source content, and memorisation guards are on by default. `relsynth privacy` attacks a release per person (membership, attribute inference, singling out, linkability) and writes a report per release context. - **Differential privacy (optional).** `relsynth run --epsilon 1` gives a user-level (ε, δ) guarantee for each person of the privacy unit and all their rows. The organisation sets ε; there is no default. Releases outside the organisation are refused without it. - **Governance.** Signed, hash-chained manifests of every run, a privacy budget ledger per domain, an organisation policy and human approvals before release (`relsynth gov`). - **Time.** Event sequences (a customer's orders) keep their gaps, life times and calendar; meter-style series keep their seasonal profiles and autocorrelation; `relsynth extend` adds new days to a synthetic dataset. @@ -358,7 +358,7 @@ The relational part of the model is unchanged: Proposition 100 proves it, and a - **SQL.** Every SQL statement is built with SQLAlchemy Core, with quoted identifiers and bound values. Table and column names cannot inject SQL; tests use hostile names on every database. - **Specs.** A spec is YAML read with a safe loader that also refuses aliases, so an alias bomb fails. Rule expressions run in a sandboxed evaluator: no attribute access, calls only to a fixed list of functions, and size and depth limits. Regular expressions with catastrophic backtracking are refused. - **Files.** Table names cannot leave the target folder (`..`, separators, absolute paths and symlinks are refused). -- **Data.** Rows never leave the source database for pushed-down facts. Bundles and profiles hold aggregates only, and `profile --top` must be asked for explicitly. Privacy checks run by default. +- **Data.** Rows never leave the source database for pushed-down facts. Bundles and profiles hold aggregates only, and `profile --top` must be asked for explicitly. Copy guards run by default; they are not a privacy check ([docs/privacy.md](docs/privacy.md) measures privacy). - **Supply chain.** The code is scanned with bandit and ruff's security rules, and dependencies with pip-audit, in CI. Details and the threat model: [docs/security.md](docs/security.md). @@ -373,7 +373,7 @@ Tuning knobs and measurements: [docs/pushdown.md](docs/pushdown.md#performance). ## Validation -**Release 2.0.** The statistical check suite now has C00–C82 plus the 2.0 checks C700–C706 (generators), C800–C803 (governance), C900–C906 (differential privacy) and C1200–C1206 (time series): 113 science tests, all run in CI on every pull request in four shards, next to 775 unit, end-to-end, security and live-server integration tests. A benchmark harness (`bench/`) reruns the real databases (bi/COREP, Chinook, AdventureWorks DW, Contoso, plus Berka, ACS PUMS and a doubled Chinook) on every change and flags drift against a five-seed baseline. Draws for the same seed change on databases with link tables, because degree laws are now the default (`synthesis.facts.link_degree: segment` restores the 1.0 draws); the [changelog](CHANGELOG.md) lists every change. +**Release 2.0.** The statistical check suite now has C00–C82 plus the 2.0 checks C600–C606 (privacy attacks), C800–C803 (governance), C900–C906 (differential privacy) and C1200–C1206 (time series), with the generator propositions 700–706 tested in tests/unit/test_generators.py: 113 science tests, all run in CI on every pull request in four shards, next to about 800 unit, end-to-end, security and live-server integration tests. A benchmark harness (`bench/`) reruns the real databases (bi/COREP, Chinook, AdventureWorks DW, Contoso, plus Berka, ACS PUMS and a doubled Chinook) on every change and flags drift against a five-seed baseline. Draws for the same seed change on databases with link tables, because degree laws are now the default (`synthesis.facts.link_degree: segment` restores the 1.0 draws); the [changelog](CHANGELOG.md) lists every change. relsynth 1.0.0 was checked at three levels. Every number below comes from the final code. diff --git a/docs/assurance/README.md b/docs/assurance/README.md index 13dc7e7..85177c3 100644 --- a/docs/assurance/README.md +++ b/docs/assurance/README.md @@ -13,7 +13,7 @@ This folder is for the privacy, legal and audit teams who decide whether a synth ## The claim register -Every sentence of the documents listed under `scope` in [claims.yaml](claims.yaml) that reads as a claim (it says *never*, *every*, *exact*, *guarantee*, *refused* and similar) has an entry. Each entry quotes the sentence and lists its evidence: a proposition of the [proofs](../proofs/relsynth-proofs.tex) by label, a numbered check (C902), a pytest test, a measured report in this repository, or the code that enforces it. A `status` says whether the evidence supports the claim fully, partly (with a note), or shows a limitation. +Every sentence of the documents listed under `scope` in [claims.yaml](claims.yaml) that reads as a claim (it says *never*, *every*, *exact*, *guarantee*, *refused* and similar) has an entry. Each entry quotes the sentence and lists its evidence: a proposition of the [proofs](../proofs/relsynth-proofs.tex) by label, a numbered check (C902), a pytest test, a measured report in this repository, the code that enforces it, or, for a sentence that sums up another registered claim (the README does this), that claim's id. A `status` says whether the evidence supports the claim fully, partly (with a note), or shows a limitation. `python tools/claims_check.py` checks the register, and CI runs it on every pull request. It fails when: diff --git a/docs/assurance/claims.yaml b/docs/assurance/claims.yaml index 5351de9..c6fa58e 100644 --- a/docs/assurance/claims.yaml +++ b/docs/assurance/claims.yaml @@ -12,6 +12,7 @@ # limitation the claim states a limit of relsynth; the evidence shows the limit is enforced or measured # scope says what a source (a regulation, a paper) says, not what relsynth does # evidence list of {prop: label} {check: Cnnn} {test: file::name} {report: path} {code: file::symbol} {source: text} +# {claim: ID} (a summary of another registered claim; as strong as that claim) # note optional: what the evidence does not show scope: @@ -19,6 +20,7 @@ scope: - docs/dp.md - docs/security.md - docs/governance/guide.md + - README.md claims: @@ -715,7 +717,518 @@ claims: - test: tests/unit/governance/test_governance.py::test_nondp_runs_are_recorded - test: tests/unit/governance/test_governance.py::test_ledger_reserve_touch_settle_refund + # ------------------------------------------------------------------ README.md (summary of the guides; registered after 2.0.0) + # Where a README sentence sums up a guide's claim it cites that claim, so the evidence is kept in one place. + + - id: RM-01 + doc: README.md + area: structure + quote: "Every primary and foreign key resolves, every rule holds, and the statistics match the source." + status: partial + note: Keys and rules are exact on every row; statistics match within calibrated limits, and on real databases not every check passes (AdventureWorks 1229/1286 in memory). + evidence: + - prop: prop:rules + - prop: prop:unique + - check: C14 + - check: C43 + - test: tests/unit/core/test_evaluate.py::test_cardinality_and_fk_integrity + - report: docs/pushdown.md#real-databases-in-memory-against-pushdown + + - id: RM-02 + doc: README.md + area: scope + quote: + - "Each statement the engine relies on is a numbered proposition with a proof (docs/proofs/relsynth-proofs.pdf) and a test that checks it." + - "relsynth-proofs.pdf: 152 propositions with proofs, each tied to a numbered check." + status: partial + note: The count is checked (test below), and checks name their propositions, but no tool verifies that every proposition has a check of its own. + evidence: + - test: tests/unit/test_assurance.py::test_proposition_numbers_follow_latex_counters + - test: tests/unit/test_assurance.py::test_claim_register_traces_every_claim + - code: tools/check_proofs.py + - report: docs/proofs/relsynth-proofs.tex + + - id: RM-03 + doc: README.md + area: privacy-formal + quote: + - "Release 2.0 adds an optional differential privacy mode with a per-person guarantee" + - "Differential privacy (optional). relsynth run --epsilon 1 gives a user-level (ε, δ) guarantee for each person of the privacy unit and all their rows." + - "The organisation sets ε; there is no default." + - "Stage 1 bounds each person's contribution and releases noisy integer statistics with OpenDP's discrete Gaussian mechanism; stage 2 fits the ordinary engine to them." + - "The guarantee is user-level (ε, δ) for one person and all their rows; δ is at most 10⁻⁵ and ε has no default." + - "Every run writes a DP statement in the NIST SP 800-226 style." + status: supported + evidence: + - claim: DP-01 + - claim: DP-03 + - claim: DP-06 + - claim: DP-07 + - claim: DP-08 + + - id: RM-04 + doc: README.md + area: structure + quote: + - "Primary keys are unique, foreign keys resolve, hierarchies have no cycles, and composite and unique keys hold." + - "In both modes, every key resolves, every unique key but one holds" + status: supported + evidence: + - prop: prop:unique + - prop: prop:gw + - check: C11 + - check: C14 + - check: C46 + - test: tests/unit/core/test_evaluate.py::test_cardinality_and_fk_integrity + - test: tests/unit/test_runtime.py::test_keys_and_links_hold_across_partitions + - report: docs/pushdown.md#real-databases-in-memory-against-pushdown + + - id: RM-05 + doc: README.md + area: structure + quote: + - "lookups, running balances and aggregates (an order total equal to its lines) hold on every row." + - "and every declared or found rule holds on every row." + - "Rules are imposed by exact conditioning: rejection, a redraw instead of a deletion, or calibrated link draws." + status: supported + evidence: + - prop: prop:rules + - prop: prop:k3 + - prop: prop:computed + - prop: prop:cumulative + - prop: prop:unique + - check: C15 + - check: C20 + - check: C23 + - check: C39 + - check: C51 + + - id: RM-06 + doc: README.md + area: privacy-measured + quote: "Rows are drawn from a fitted model; only tables you declare mode: copy are copied." + status: supported + note: Rows are new draws; a few may agree with a real row by chance on low-cardinality columns, which the test bounds on high-cardinality attributes. + evidence: + - test: tests/unit/test_security.py::test_synthetic_rows_are_not_copies + - check: C04 + - check: C66 + + - id: RM-07 + doc: README.md + area: privacy-measured + quote: + - "Text cannot reproduce rare source content, and memorisation guards are on by default." + - "relsynth privacy attacks a release per person (membership, attribute inference, singling out, linkability) and writes a report per release context." + status: supported + evidence: + - prop: prop:lm + - prop: prop:guard + - check: C13 + - test: tests/unit/test_security.py::test_privacy_checks_run_by_default + - claim: PRV-01 + + - id: RM-08 + doc: README.md + area: governance + quote: + - "Releases outside the organisation are refused without it." + - "A run with --context partner or public is refused before reading data unless it is differentially private." + status: supported + evidence: + - claim: DP-02 + - claim: GOV-05 + + - id: RM-09 + doc: README.md + area: governance + quote: + - "Signed, hash-chained manifests of every run, a privacy budget ledger per domain, an organisation policy and human approvals before release" + - "A registry of signed (Ed25519, DSSE / in-toto) manifests in a hash-chained log; an organisation policy with ε caps; a privacy budget ledger per domain that reserves before any data is read; approvals with quorum and separation of duties; signed release packages." + status: supported + evidence: + - claim: GOV-01 + - claim: GOV-02 + - claim: GOV-04 + - claim: GOV-07 + - claim: GOV-08 + - claim: GOV-09 + + - id: RM-10 + doc: README.md + area: security + quote: + - "Facts larger than memory are fitted from statistics computed inside the database (GROUP BY, window functions) plus a uniform sample. Rows never leave the database as a whole." + - "Rows never leave the source database for pushed-down facts." + status: supported + evidence: + - prop: prop:pdsuff + - prop: prop:hashsample + - test: tests/unit/test_pushdown.py::test_sampled_pushdown_keeps_quality_and_memory_bound + - test: tests/unit/test_pushdown.py::test_full_sample_pushdown_equals_in_memory + + - id: RM-11 + doc: README.md + area: statistics + quote: + - "A scorecard compares synthetic with real data, check by check, with calibrated limits. --fail-under 0.98 makes a CI job fail when quality drops." + - "The scorecard checks marginals, dependence, joins, cardinalities, trees, time, text, rules and privacy, each against a limit calibrated to the sampling noise of a posterior predictive draw." + - "It holds hundreds of pass/fail checks" + - "Families of checks are Šidák-corrected, and each check reports its significance level, so the number of false failures to expect is known." + status: supported + evidence: + - prop: prop:ppnull + - prop: prop:sidak + - check: C41 + - check: C76 + - check: C79 + - check: C80 + - test: tests/e2e/test_cli.py::test_quality_gate_exit_code + + - id: RM-12 + doc: README.md + area: structure + quote: "The file lists every table and column with its role, every link and every rule found, and a review: list of questions only you can answer." + status: supported + evidence: + - check: C40 + - check: C44 + - test: tests/unit/test_api.py::test_introspect_draft_is_valid_and_runnable + + - id: RM-13 + doc: README.md + area: structure + quote: "From release 2.0, introspect writes an IR 2.0 document by default (--format v1 keeps the 1.0 spec); every command reads both." + status: supported + evidence: + - prop: prop:irround + - test: tests/unit/test_ir.py::test_introspect_format_follows_the_release + - test: tests/unit/test_ir.py::test_cli_runs_an_ir_file + - test: tests/e2e/test_cli.py::test_doctor_and_tune_write_ir_when_given_ir + - test: tests/unit/test_ir.py::test_ir_document_gives_identical_synthetic_data + + - id: RM-14 + doc: README.md + area: security + quote: + - "Passwords are never accepted on the command line; they come from password_env or url_env." + - "Passwords are masked in every message, log, saved spec, diagnostics bundle and exported model." + - "(saved specs never contain passwords)" + status: supported + evidence: + - claim: SEC-01 + - claim: SEC-04 + - test: tests/e2e/test_cli.py::test_locations_from_environment + - test: tests/unit/test_spec.py::test_save_masks_passwords + + - id: RM-15 + doc: README.md + area: scope + quote: "Every command takes --json for machine-readable output and returns an exit code for scripts." + status: partial + note: The tests cover the exit codes 1, 2 and 3 and --json on run, check and the governed run; no test walks every command. + evidence: + - test: tests/e2e/test_cli.py::test_quality_gate_exit_code + - test: tests/e2e/test_cli.py::test_spec_errors_exit_2 + - test: tests/e2e/test_cli.py::test_connection_errors_exit_3_and_mask_passwords + + - id: RM-16 + doc: README.md + area: structure + quote: "and it is validated strictly: every option is documented in docs/spec-reference.md, and misspelt keys are refused with a suggestion." + status: supported + evidence: + - check: C67 + - check: C68 + - test: tests/unit/test_spec.py::test_invalid_settings_are_refused + + - id: RM-17 + doc: README.md + area: statistics + quote: + - "Every entity row (a customer, a product) gets a latent segment, fitted by Gibbs sampling over the whole schema." + - "Each table's columns follow a Bayesian Gaussian copula with exact marginals, per segment or pooled." + - "A dataset draws every parameter once from its posterior, then the rows, entities first and facts after." + status: supported + evidence: + - prop: prop:ident + - prop: prop:eta + - prop: prop:pit + - prop: prop:copula + - prop: prop:postpred + - check: C01 + - check: C02 + - check: C05 + - check: C09 + - check: C57 + + - id: RM-18 + doc: README.md + area: statistics + quote: "Tune engine settings toward your objectives over several seeds; constraints report for what cannot be met" + status: supported + evidence: + - test: tests/unit/test_autotune.py::test_loop_finds_the_planted_fix_and_explains_the_rest + - test: tests/unit/test_autotune.py::test_loop_never_accepts_a_hard_violation + + - id: RM-19 + doc: README.md + area: scope + quote: + - "The full reference with every option is in docs/cli.md." + - "The full reference, with every parameter, is in docs/api.md." + - "| docs/cli.md | Every command and option |" + - "| docs/api.md | Every public class and function |" + - "| docs/spec-reference.md | Every specification option, with recipes (template) |" + status: partial + note: The references are generated from the parser, the public names and the spec schema (docs/make_*_reference.py), but unlike settings-reference.md and ir-reference.md no test fails when they are stale. + evidence: + - code: docs/make_cli_reference.py + - code: docs/make_api_reference.py + - code: docs/make_spec_reference.py + - check: C68 + - test: tests/unit/test_api.py::test_public_names + + - id: RM-20 + doc: README.md + area: scope + quote: "| docs/ir-reference.md | Every IR field (generated from the model; JSON Schema in src/relsynth/ir/schema/) |" + status: supported + evidence: + - test: tests/unit/test_ir.py::test_schema_and_reference_are_generated + + - id: RM-21 + doc: README.md + area: security + quote: "The package's own CI is in .github/workflows/ci.yml: lint, security and licence scans, types, unit tests on Python 3.10–3.13 and three operating systems" + status: supported + evidence: + - report: .github/workflows/ci.yml + - claim: SEC-13 + + - id: RM-22 + doc: README.md + area: statistics + quote: + - "exact GROUP BY counts over the key tuples the model uses;" + - "Entities (dimensions) are always loaded, because each of their rows carries its own latent segment." + - "Proposition 100 proves it, and a run with a sample of every row gives exactly the in-memory result on all four databases." + - "The column models are an exact posterior given the sample (Proposition 104)." + status: supported + evidence: + - prop: prop:pdsuff + - prop: prop:pdexact + - prop: prop:pdsample + - prop: prop:pdeligible + - test: tests/unit/test_pushdown.py::test_full_sample_pushdown_equals_in_memory + - test: tests/unit/test_pushdown.py::test_ineligible_reasons + - test: tests/integration/test_servers.py::test_full_sample_pushdown_matches_sqlite_in_memory + - report: docs/pushdown.md#real-databases-in-memory-against-pushdown + + - id: RM-23 + doc: README.md + area: privacy-formal + quote: "On Contoso at ε = 1 the measured membership-inference rate stayed below the DP bound." + status: supported + evidence: + - claim: DP-14 + - report: docs/assurance/evidence/T07/audit_contoso_eps1.jsonl + + - id: RM-24 + doc: README.md + area: statistics + quote: + - "With default settings the output is unchanged byte for byte." + - "relsynth autotune separates systematic failures from noise over several seeds" + status: supported + evidence: + - test: tests/unit/test_settings.py::test_promoted_constants_keep_the_1_0_values + - test: tests/unit/test_runtime.py::test_default_runtime_keeps_the_1_0_sampler + - test: tests/unit/test_generators.py::test_copula_is_the_default_and_naming_it_changes_nothing + - test: tests/unit/test_autotune.py::test_systematic_versus_sporadic + - code: tools/identity_check.py + + - id: RM-25 + doc: README.md + area: security + quote: + - "Every SQL statement is built with SQLAlchemy Core, with quoted identifiers and bound values." + - "Table and column names cannot inject SQL; tests use hostile names on every database." + status: supported + evidence: + - claim: SEC-03 + - test: tests/unit/test_connectors.py::test_hostile_table_and_column_names_are_quoted + - test: tests/integration/test_servers.py::test_hostile_identifiers_on_server + + - id: RM-26 + doc: README.md + area: security + quote: + - "A spec is YAML read with a safe loader that also refuses aliases, so an alias bomb fails." + - "Regular expressions with catastrophic backtracking are refused." + - "Table names cannot leave the target folder (.., separators, absolute paths and symlinks are refused)." + status: supported + evidence: + - claim: SEC-05 + - claim: SEC-06 + - claim: SEC-07 + - claim: SEC-09 + + - id: RM-27 + doc: README.md + area: security + quote: "Bundles and profiles hold aggregates only, and profile --top must be asked for explicitly." + status: partial + note: As SEC-10, no test shows that a diagnostics bundle holds no rows. + evidence: + - claim: SEC-10 + + - id: RM-28 + doc: README.md + area: statistics + quote: "Fits use exact vectorised kernels, with numba kernels as an option (relsynth[speed])." + status: supported + evidence: + - prop: prop:kernels + - check: C69 + + - id: RM-29 + doc: README.md + area: scope + quote: + - "The statistical check suite now has C00–C82 plus the 2.0 checks C600–C606 (privacy attacks), C800–C803 (governance), C900–C906 (differential privacy) and C1200–C1206 (time series), with the generator propositions 700–706 tested in tests/unit/test_generators.py: 113 science tests" + - "A benchmark harness (bench/) reruns the real databases" + - "and flags drift against a five-seed baseline." + status: supported + note: The test counts are those of the 2.0.0 tree (pytest --collect-only -m science gives 113). + evidence: + - check: C82 + - check: C606 + - test: tests/unit/test_generators.py::test_reference_generators_keep_keys_links_and_rules + - check: C803 + - check: C906 + - check: C1206 + - report: .github/workflows/ci.yml + - report: bench/baseline.json + - test: tests/unit/test_bench_regress.py::test_r4_drift_of_a_tracked_value + + - id: RM-30 + doc: README.md + area: statistics + quote: "Draws for the same seed change on databases with link tables, because degree laws are now the default (synthesis.facts.link_degree: segment restores the 1.0 draws)" + status: limitation + evidence: + - prop: prop:linkdeg + - test: tests/unit/core/test_link_degree.py::test_link_degree_spec_values + + - id: RM-31 + doc: README.md + area: statistics + quote: + - "Every number below comes from the final code." + - "It then checks that each numbered proposition holds" + - "All 79 checks (C00–C78, more than 500 assertions) pass." + - "Four real databases, each from an unedited introspect draft, fitted in memory and with every eligible fact pushed down" + status: supported + note: These are the 1.0.0 measurements; on 2.0 the benchmark harness reruns the same databases. + evidence: + - check: C00 + - check: C78 + - report: docs/pushdown.md#real-databases-in-memory-against-pushdown + - report: bench/baseline.json + + - id: RM-32 + doc: README.md + area: structure + quote: "On every server, the same seed gives the same synthetic database as SQLite, value for value." + status: supported + evidence: + - test: tests/integration/test_servers.py::test_full_sample_pushdown_matches_sqlite_in_memory + - test: tests/integration/test_servers.py::test_introspect_fit_and_pushdown_run + - test: tests/integration/test_servers.py::test_statistics_match_the_in_memory_reference + + - id: RM-33 + doc: README.md + area: scope + quote: + - "the layer 1 algorithms, with complexity and proofs." + - "It covers the Bayesian Gaussian copula, latent segments from in-database sufficient statistics, the merge step and differential privacy on summaries." + - "gives a readable summary of the model and of the six v1.0 propositions, with the proofs in full." + status: supported + evidence: + - check: C00 + - report: docs/proofs/algorithms.pdf + - report: docs/theory.md + + - id: RM-34 + doc: README.md + area: scope + quote: + - "| Rules and keys | Rules by rejection; three-valued rules agree with the database; computed columns are exact; unique keys by exact conditioning; order headers (lossless split); running balances |" + - "| Evaluation | Calibrated limits under the posterior predictive null; Šidák over families; distances with an atom at zero |" + - "| In-database fitting (v1.0) | 100 sufficiency of the in-database statistics; 101 agreement with in-memory fitting; 102 Acklam's quantile bound; 103 hash sampling is Bernoulli; 104 the sample posterior is exact; 105 eligibility |" + - "| IR, generators and quality (v2.0) | 106 loss-free 1.0 conversion; 107–108 privacy-unit tracing; 109–113 calibrated quality report; 700–706 the skeleton is kept whatever the generator |" + - "| Privacy and governance (v2.0) | Per-person attack metrics; 800–803 the budget ledger bounds the domain's privacy loss, reserve before reading, tamper evidence; 900–906 discrete Gaussian, stable contribution bounding, exact partition selection, two-stage DP |" + - "| Time and scale (v2.0) | 1200–1206 time change, gamma renewal bridge, sequence posterior, calibrated limits, incremental runs; degree laws of link tables; 950–954 worker independence, partitions keep the law, keys across partitions, drift tests |" + status: supported + note: The table names propositions; each label below is one of them, and the numbers are checked against the LaTeX counters. + evidence: + - prop: prop:rules + - prop: prop:k3 + - prop: prop:computed + - prop: prop:unique + - prop: prop:header + - prop: prop:ppnull + - prop: prop:sidak + - prop: prop:dcr + - prop: prop:pdsuff + - prop: prop:pdeligible + - prop: prop:irround + - prop: prop:ratenorm + - prop: prop:genskeleton + - prop: prop:cartbb + - prop: prop:puctrl + - prop: prop:ledger + - prop: prop:govchain + - prop: prop:dgauss + - prop: prop:run + - prop: prop:tschange + - prop: prop:tsincr + - prop: prop:linkdeg + - prop: prop:rtdet + - prop: prop:rtdrift + + - id: RM-35 + doc: README.md + area: scope + quote: "Rules that hold on every source row but are not declared or found are reproduced statistically, not exactly." + status: limitation + evidence: + - prop: prop:rules + - check: C58 + + - id: RM-36 + doc: README.md + area: privacy-formal + quote: "Differential privacy mode releases one-way statistics, degrees and one correlation matrix per table, so its utility is nearly flat in ε" + status: limitation + evidence: + - claim: DP-13 + - claim: DP-15 + - report: docs/assurance/evidence/T07/utility_vs_epsilon.jsonl + not_claims: - {doc: docs/privacy.md, quote: "Every key is optional.", reason: "describes the configuration file format"} - {doc: docs/privacy.md, quote: "Viswanath, The composition theorem for differential privacy, ICML 2015.", reason: "bibliography entry"} - {doc: docs/security.md, quote: "Pass secrets as masked CI variables (see cicd.md), and use read-only database accounts for sources.", reason: "advice to the operator"} + - {doc: README.md, quote: "| A privacy officer or data protection officer |", reason: "reading guide: points to the documents"} + - {doc: README.md, quote: "| Reviewing the method |", reason: "reading guide: points to the documents"} + - {doc: README.md, quote: "- Privacy, differential privacy and governance", reason: "table of contents"} + - {doc: README.md, quote: "- Theory: propositions and proofs", reason: "table of contents"} + - {doc: README.md, quote: "Never put a password on the command line.", reason: "advice to the operator (the enforced part is RM-14)"} + - {doc: README.md, quote: "A job that refreshes a test database every night and fails when quality drops:", reason: "introduces an example pipeline"} + - {doc: README.md, quote: "Differential privacy (relsynth run --epsilon E, relsynth[dp], docs/dp.md).", reason: "bullet heading; the bullet's claims are RM-03 and RM-23"} + - {doc: README.md, quote: "| docs/dp.md | Differential privacy mode: guarantee, parameters, DP statement, limits |", reason: "documentation map: names a document's contents"} + - {doc: README.md, quote: "| docs/governance/guide.md | Registry, signed manifests, policy, budget ledger, approvals (design) |", reason: "documentation map: names a document's contents"} + - {doc: README.md, quote: "| docs/theory.md | The model in brief and the v1.0 propositions with proofs |", reason: "documentation map: names a document's contents"} + - {doc: README.md, quote: "| docs/proofs/ | The full proofs (PDF and LaTeX source) |", reason: "documentation map: names a document's contents"} diff --git a/docs/development/ci.md b/docs/development/ci.md index f6ad042..b556c0a 100644 --- a/docs/development/ci.md +++ b/docs/development/ci.md @@ -11,7 +11,7 @@ Every pull request and every push to `main` runs [`.github/workflows/ci.yml`](.. | `unit (os, python)` | `tests/unit` on Python 3.10–3.13 (Linux) and 3.12 on Windows and macOS, with branch coverage; the Linux 3.12 job writes coverage per module to the job summary | 3–6 min | | `e2e` | every CLI command as a pipeline runs it (`tests/e2e`) | 2 min | | `integration` | live PostgreSQL 16, SQL Server 2022 and Oracle 23 Free in service containers (`tests/integration`) | 4 min | -| `science (i/4)` | the statistical check suite (C00–C82, C700s, C800s, C900s, C1200s) in four shards of about equal run time | about 15 min each | +| `science (i/4)` | the statistical check suite (C00–C82, C600s, C800s, C900s, C1200s) in four shards of about equal run time | about 15 min each | | `build` | sdist and wheel, `twine check`, a clean-environment install with every extra, and a CycloneDX SBOM of that install | 1 min | | `sign` | dry run of the release signature: the wheel, sdist and SBOM are signed with Sigstore under this workflow's identity and verified (pull requests from this repository and pushes; not on forks) | 1 min | diff --git a/tests/unit/test_assurance.py b/tests/unit/test_assurance.py index e7e229c..ee79c18 100644 --- a/tests/unit/test_assurance.py +++ b/tests/unit/test_assurance.py @@ -36,7 +36,7 @@ def _checker(): def test_claim_register_traces_every_claim(): res = cc.run(ROOT) assert res.problems == [], "\n".join(res.problems) - assert len(res.claims) >= 50 + assert len(res.claims) >= 90 def test_proposition_numbers_follow_latex_counters(): @@ -61,12 +61,13 @@ def _mini_repo(tmp_path: Path, register: str, doc: str) -> Path: "\\newcounter{sv}\\setcounter{sv}{\\value{proposition}}\\setcounter{proposition}{899}\n" "\\begin{proposition}[T]\\label{prop:b}y\\end{proposition}\n" "\\setcounter{proposition}{\\value{sv}}\n" - "\\begin{proposition}\\label{prop:c}z\\end{proposition} % \\begin{proposition}\n") - (tmp_path / "legacy/checks/run_checks.py").write_text('@check("C01", "x", "Prop. 1")\n') + "\\begin{proposition}\\label{prop:c}z\\end{proposition} % \\begin{proposition}\n", + encoding="utf-8") + (tmp_path / "legacy/checks/run_checks.py").write_text('@check("C01", "x", "Prop. 1")\n', encoding="utf-8") (tmp_path / "tests/unit/test_x.py").write_text( - "def test_c900_b():\n pass\n\nclass TestK:\n def test_m(self):\n pass\n") - (tmp_path / "docs/guide.md").write_text(doc) - (tmp_path / "docs/assurance/claims.yaml").write_text(register) + "def test_c900_b():\n pass\n\nclass TestK:\n def test_m(self):\n pass\n", encoding="utf-8") + (tmp_path / "docs/guide.md").write_text(doc, encoding="utf-8") + (tmp_path / "docs/assurance/claims.yaml").write_text(register, encoding="utf-8") return tmp_path @@ -108,9 +109,35 @@ def test_checker_catches_drift(tmp_path, doc, register, expect): assert any(expect in p for p in res.problems), res.problems +SUMMARY = """ + - id: CL-2 + doc: docs/guide.md + quote: Keys resolve in the summary. + status: supported + evidence: [{claim: CL-1}] +""" + + +@pytest.mark.parametrize(("cited", "expect"), [ + ("CL-1", None), + ("CL-9", "cites claim CL-9, which the register does not have"), + ("CL-3", "cites claim CL-3, which has no prop, check, test or report of its own"), +]) +def test_a_summary_claim_is_as_strong_as_the_claim_it_cites(tmp_path, cited, expect): + weak = " - {id: CL-3, doc: docs/guide.md, quote: Every key resolves., status: scope, evidence: [{source: x}]}\n" + register = GOOD.replace("not_claims:", SUMMARY.replace("CL-1}", cited + "}").lstrip("\n") + weak + "not_claims:") + doc = "Every key resolves. Run it every night. Keys resolve in the summary.\n" + problems = cc.run(_mini_repo(tmp_path, register, doc)).problems + if expect is None: + assert problems == [] + else: + assert any(expect in p for p in problems), problems + + def test_checker_catches_bad_citations_in_the_assurance_documents(tmp_path): root = _mini_repo(tmp_path, GOOD, "Every key resolves. Run it every night.\n") - (root / "docs/assurance/case.md").write_text("G1 rests on CL-1, PRV-99, `prop:b`, `prop:gone`, C900 and C777.\n") + (root / "docs/assurance/case.md").write_text( + "G1 rests on CL-1, PRV-99, `prop:b`, `prop:gone`, C900 and C777.\n", encoding="utf-8") problems = cc.run(root).problems assert any("cites PRV-99" in p for p in problems) assert any("cites prop:gone" in p for p in problems) diff --git a/tools/claims_check.py b/tools/claims_check.py index 5cb71ab..8ff9aab 100644 --- a/tools/claims_check.py +++ b/tools/claims_check.py @@ -217,6 +217,12 @@ def doc_text(name: str) -> str | None: docs[name] = norm(p.read_text(encoding="utf-8")) if p.is_file() else None return docs[name] + # A claim that repeats another (the README summing up a guide) cites it as {claim: ID} + # and is as strong as the claim it cites. + direct = { + c.get("id"): {k for ev in c.get("evidence") or [] if isinstance(ev, dict) for k in ev} + for c in reg.get("claims", []) + } seen: set[str] = set() quotes: dict[str, list[str]] = {} for c in reg.get("claims", []): @@ -248,8 +254,17 @@ def doc_text(name: str) -> str | None: continue (kind, value), = ev.items() value = str(value) - kinds.append(kind) - why = _resolve(kind, value, root, props, checks, sym_cache) + if kind == "claim": + if value not in direct: + why = f"cites claim {value}, which the register does not have" + elif not direct[value] & set(STRONG_KINDS): + why = f"cites claim {value}, which has no prop, check, test or report of its own" + else: + why = None + kinds.extend(direct[value]) + else: + kinds.append(kind) + why = _resolve(kind, value, root, props, checks, sym_cache) if why: res.problems.append(f"{where}: {why}") item = {"kind": kind, "ref": value}