From 4b107ada36f37852ac75515c70e156dd8b511185 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 27 Aug 2026 18:10:42 -0400 Subject: [PATCH 1/2] fix(fixtures): bind synthetic provenance to the fixture byte determinants The synthetic fixture provenance recorded a sha256 of the whole uv.lock, and tests/test_synthetic_fixtures.py asserted it. That hash does not determine the fixture bytes. The generator reads the source spec, its own source, the repository's writer code, and SQLite plus SQLAlchemy. Every one of those is recorded already: the spec and generator by sha256, the two library versions by value, with require_exact_builder() refusing any other writer. Because the lock hash was written into the provenance, and the provenance is embedded in the recording config, the hash reached the sealed database bytes. That made it self-referential. Any lock rewrite forced a rewrite of two sealed binary fixtures, whether or not the rewrite touched anything the generator reads. Adding a platform-gated optional dependency did exactly that: it added two Linux-only wheels and migrated the lock file format, changed no resolved version of anything, and still broke the fixture suite. Drop lock_path and lock_sha256, and bump the provenance schema to v2 so a reader sees the narrower shape. The byte oracle is unchanged and remains the authority: check_generated() regenerates all five artifacts and compares them byte for byte, so a dependency change that really does move the bytes still fails. Two tests now prove that oracle bites: - a one-pixel edit to a source spec makes the committed fixtures stale - one flipped byte in any of the five generated artifacts is caught Co-Authored-By: Claude Opus 5 --- .../demo_new/capture-artifact-manifest.json | 2 +- .../captures/demo_new/capture-terminal.json | 2 +- examples/captures/demo_new/recording.db | Bin 249856 -> 245760 bytes .../demo_new/synthetic-provenance.json | 2 +- .../capture-artifact-manifest.json | 2 +- .../turn-off-nightshift/capture-terminal.json | 2 +- .../captures/turn-off-nightshift/recording.db | Bin 323584 -> 323584 bytes .../synthetic-provenance.json | 2 +- scripts/generate_synthetic_captures.py | 13 ++- tests/test_synthetic_fixtures.py | 74 ++++++++++++++++-- 10 files changed, 81 insertions(+), 18 deletions(-) diff --git a/examples/captures/demo_new/capture-artifact-manifest.json b/examples/captures/demo_new/capture-artifact-manifest.json index 0faafff..c4bcf7e 100644 --- a/examples/captures/demo_new/capture-artifact-manifest.json +++ b/examples/captures/demo_new/capture-artifact-manifest.json @@ -1 +1 @@ -{"artifacts":[{"path":"capture-state.json","sha256":"b96ed12c609334c76636598b336d664557d444a7060df2bdace4e25f7b368cfa","size_bytes":370},{"path":"recording.db","sha256":"24fa103e41c6c6d15b423fd7a3264f82df026d1ca1c449551becd2b49735aca2","size_bytes":249856},{"path":"synthetic-provenance.json","sha256":"5a3fb539cb3a702b310b53ac25bfb157a63dfc2f14bd3d645ac2d572e1b8fda5","size_bytes":3112}],"schema_version":"openadapt.capture-artifact-manifest/v1"} +{"artifacts":[{"path":"capture-state.json","sha256":"b96ed12c609334c76636598b336d664557d444a7060df2bdace4e25f7b368cfa","size_bytes":370},{"path":"recording.db","sha256":"bcd9d62065ab9bfca346a43a0fd347a6321dd2c2de69e44ef8e9e2475afa65b9","size_bytes":245760},{"path":"synthetic-provenance.json","sha256":"bd76448dfe409e02a51e7c23aaee75a7b74cebc020895bbd1eb47523c90ecd1b","size_bytes":3009}],"schema_version":"openadapt.capture-artifact-manifest/v1"} diff --git a/examples/captures/demo_new/capture-terminal.json b/examples/captures/demo_new/capture-terminal.json index f7c8b01..29968d3 100644 --- a/examples/captures/demo_new/capture-terminal.json +++ b/examples/captures/demo_new/capture-terminal.json @@ -1 +1 @@ -{"artifact_manifest_sha256":"03980cfda7efa77aca53d2890e70362712627d30563f387f42b82cae88634e93","artifact_manifest_size_bytes":448,"ended_at":"2026-01-01T00:00:13.000000Z","event_counts":{"action":14,"browser":0,"screen":14,"video":0,"window":0},"last_source_ordinal":28,"reason_code":"normal_stop","schema_version":"openadapt.capture-terminal/v2","source_capture_session_sha256":"1d64c69bbd4b28e980830c4dc446048b51fe9c6897f225870a2b9b2a7bfc0261","started_at":"2026-01-01T00:00:00.000000Z","state":"COMPLETE","terminal_sha256":"0b7ce6eed73b6b3e1a2a45cc77996775fb5ea2c1d40f15106ce43cd1c2c77855"} +{"artifact_manifest_sha256":"0b90d55a85b57b8d57d0ac5bcc0730553fee5f79027162a7b71d1b9dd492ce30","artifact_manifest_size_bytes":448,"ended_at":"2026-01-01T00:00:13.000000Z","event_counts":{"action":14,"browser":0,"screen":14,"video":0,"window":0},"last_source_ordinal":28,"reason_code":"normal_stop","schema_version":"openadapt.capture-terminal/v2","source_capture_session_sha256":"1d64c69bbd4b28e980830c4dc446048b51fe9c6897f225870a2b9b2a7bfc0261","started_at":"2026-01-01T00:00:00.000000Z","state":"COMPLETE","terminal_sha256":"a25eb0467508e2d36b9783d149d91a19640d818161b389635379e2183db11037"} diff --git a/examples/captures/demo_new/recording.db b/examples/captures/demo_new/recording.db index 52805bc0451b46e2a321d8c88d836b69b9a26ce1..c967a4121eb6899860d8f4ad799ec00a3bb075b3 100644 GIT binary patch delta 727 zcmYL_Pe>GD7{=%QX4YBFoHg@bo6c;xwq@q*?(D2<66+xnyqH)C1@1qsn_D-La1|m6 zYK(kN^qD8D%^8{ zM3Ze07a{jW2>Ps0O#5%ZZP9Xw6_yx_bDuETO0qB$;zha<2i8`KD?j15vHCunstNcY zgB_s>hX1ighv13?M{%_mN@>|RXf7zHEuUdbv>ahX8P~4zGZGl zyE)X6fpr;-EL zXsKZ|4bwej6K&hf`5%>MARk&)sHedo|3(HsUcAQF8uMF4hV5-?Uo&Nz3nZxBe@&*N qN&dGL>gZ64Zx<~(L!QM~J;aVSC#3dvQ?~oO#E5%SWE`F6TmJ$FkLR!e delta 868 zcmYL_O-vI}5P&;xTecu}7b`znb(bnw1$B43+ueegl4u~&L^MXd$+FuDsY>JD#SjV? z(L^EH!{cBgc;WyFu;B9|{#`U4)EM=EKh=Qo1T~&8G5U%z9^NExX7bIugBvlofg-K(BPIP`u)JWgRRHe zP_&d5>Pu*-@zb!Sh=$5Sa|OC1igzS_I?G7laefB-oUiNVL3SSpZ$d&g4c&;^9Z5N^ zY6^WZBPN@w6jhRH(u_-*YMG{G>4u!rQb|qGQ+iCcY*o@yhGOfQGgN-Hw_XgN>9*3R zGUmn3aDy1Wba{8-AO4rW&}k}KtWZgH#7$i{JLpi|>abI?Zdi(9D6$-lONK4$amlhZ z)iz^MIVwqtWJ}4EsmZpXs9uWv%_v(#hc=(z7>D~PNdO- z`)xAB0q1%j{uZDNzd8(?aoY@f@BYdaaj?$y=4pOC{=R^=IGOnw{4s}E@_~n14$6sF zgzFLD$zUsFSXg_v4f_00Mfy)d59Z467kwF8~D5r6muTb?oAuZV}cx`|>Q=)rQ^*u=Bqf@+*U=n>ymnqOl0H(!^wWo9&*EGl2K z`Hwuugvk@-vnR7`*z1sLWM*QSWNKz3H!=b-6A&|R|Gbff*A4)` CfiO$} delta 283 zcmXwyKTE?v9EOwrAxc0ZUB#4hm2$t!UGAb3#L2;rAjst|XG3Z$SY1>MICK^AB^+Zi zICbvmSLoLe9J~$=&+xu4Ja4%fl$$}_5BmPv@z?Esu=0bCk-O*gI-On54GZV{v^_r< zcU`E;@VUA=yr?egqxLu4d9UX6>I|gdBGPFF31!3*ix?!u5y@yK6U?ZRQYkLLFq1LD z4Ua)-ikT5ab5>11m#@b|G@YyB`c5t;=wyf<9!J}Z{`>ohBrM)8CQT&gGPS2%rP=@& ziV#5nq67;KJi$scs%0EOgfYPyXGSv6f>OX#p%?31k bool: + import sqlalchemy + + return (sqlite3.sqlite_version, sqlalchemy.__version__) == ( + REQUIRED_SQLITE_VERSION, + REQUIRED_SQLALCHEMY_VERSION, + ) + + +def _copy_committed_fixtures(destination: Path) -> Path: + for fixture_id in EXPECTED_COUNTS: + shutil.copytree(CAPTURE_ROOT / fixture_id, destination / fixture_id) + return destination + + def test_public_fixtures_match_the_source_generator_byte_for_byte() -> None: import sqlalchemy - assert (REPOSITORY / BUILDER_LOCK_PATH).is_file() if sys.version_info[:2] != (3, 12): actual = (sqlite3.sqlite_version, sqlalchemy.__version__) expected = (REQUIRED_SQLITE_VERSION, REQUIRED_SQLALCHEMY_VERSION) @@ -168,6 +181,57 @@ def test_public_fixtures_match_the_source_generator_byte_for_byte() -> None: ) +@pytest.mark.skipif( + not _builder_reproduces_the_committed_fixtures(), + reason="this builder cannot reproduce the committed fixture bytes", +) +def test_a_changed_source_spec_makes_the_committed_fixtures_stale( + tmp_path: Path, +) -> None: + """A source change reaches the fixture bytes, and the byte check reports it.""" + spec_root = tmp_path / "specs" + spec_root.mkdir() + for spec_path in (CAPTURE_ROOT / "specs").glob("*.json"): + spec = json.loads(spec_path.read_bytes()) + if spec_path.stem == "demo_new": + spec["actions"][0]["x"] += 1 + (spec_root / spec_path.name).write_bytes(canonical_json_bytes(spec)) + + with pytest.raises(RuntimeError, match="generated bytes are stale"): + check_generated( + spec_root=spec_root, + output_root=CAPTURE_ROOT, + generator_path=GENERATOR, + ) + + +@pytest.mark.skipif( + not _builder_reproduces_the_committed_fixtures(), + reason="this builder cannot reproduce the committed fixture bytes", +) +@pytest.mark.parametrize( + "artifact_name", + sorted(GENERATED_FILENAMES), +) +def test_every_committed_fixture_artifact_is_byte_checked( + tmp_path: Path, + artifact_name: str, +) -> None: + """One flipped byte in any generated artifact fails the byte check.""" + output_root = _copy_committed_fixtures(tmp_path / "captures") + target = output_root / "demo_new" / artifact_name + raw = bytearray(target.read_bytes()) + raw[-1] ^= 0xFF + target.write_bytes(bytes(raw)) + + with pytest.raises(RuntimeError, match="generated bytes are stale"): + check_generated( + spec_root=CAPTURE_ROOT / "specs", + output_root=output_root, + generator_path=GENERATOR, + ) + + def test_reconstructed_legacy_fixture_is_marked_unsealed_and_ineligible( tmp_path: Path, ) -> None: From da1bd4e4dc7c626274e30a540393a6e53b4ff2b5 Mon Sep 17 00:00:00 2001 From: abrichr Date: Thu, 27 Aug 2026 18:14:28 -0400 Subject: [PATCH 2/2] test(fixtures): run the tamper checks only on the reference builder The two new tamper tests assert that check_generated() raises 'generated bytes are stale'. On a builder that does not reproduce the committed bytes exactly, that message could report the builder instead of the tamper, and the assertion would hold for the wrong reason. Python 3.12 with the required SQLite and SQLAlchemy is the environment that test_public_fixtures_match_the_source_generator_byte_for_byte proves reproduces every committed byte, so it supplies the control. Restrict both tamper tests to it. Co-Authored-By: Claude Opus 5 --- tests/test_synthetic_fixtures.py | 34 ++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/tests/test_synthetic_fixtures.py b/tests/test_synthetic_fixtures.py index b625227..dc8375a 100644 --- a/tests/test_synthetic_fixtures.py +++ b/tests/test_synthetic_fixtures.py @@ -145,13 +145,27 @@ def test_public_fixture_is_synthetic_sealed_and_exactly_bound( assert all(token not in lowered for token in PRIVATE_SOURCE_TOKENS) -def _builder_reproduces_the_committed_fixtures() -> bool: +def _is_the_reference_builder() -> bool: + """Report whether this interpreter regenerates the committed bytes exactly. + + Python 3.12 with the required SQLite and SQLAlchemy is the one environment + that `test_public_fixtures_match_the_source_generator_byte_for_byte` proves + reproduces every committed byte. A tamper test is only meaningful there: on + any other builder a raised `generated bytes are stale` could report the + builder rather than the tamper. + """ import sqlalchemy - return (sqlite3.sqlite_version, sqlalchemy.__version__) == ( - REQUIRED_SQLITE_VERSION, - REQUIRED_SQLALCHEMY_VERSION, - ) + return sys.version_info[:2] == (3, 12) and ( + sqlite3.sqlite_version, + sqlalchemy.__version__, + ) == (REQUIRED_SQLITE_VERSION, REQUIRED_SQLALCHEMY_VERSION) + + +REFERENCE_BUILDER_ONLY = pytest.mark.skipif( + not _is_the_reference_builder(), + reason="only the reference builder regenerates the committed bytes exactly", +) def _copy_committed_fixtures(destination: Path) -> Path: @@ -181,10 +195,7 @@ def test_public_fixtures_match_the_source_generator_byte_for_byte() -> None: ) -@pytest.mark.skipif( - not _builder_reproduces_the_committed_fixtures(), - reason="this builder cannot reproduce the committed fixture bytes", -) +@REFERENCE_BUILDER_ONLY def test_a_changed_source_spec_makes_the_committed_fixtures_stale( tmp_path: Path, ) -> None: @@ -205,10 +216,7 @@ def test_a_changed_source_spec_makes_the_committed_fixtures_stale( ) -@pytest.mark.skipif( - not _builder_reproduces_the_committed_fixtures(), - reason="this builder cannot reproduce the committed fixture bytes", -) +@REFERENCE_BUILDER_ONLY @pytest.mark.parametrize( "artifact_name", sorted(GENERATED_FILENAMES),