From 0c9e07b276ac85b0cdbb56c814e49908ab286090 Mon Sep 17 00:00:00 2001 From: "Prashant Rane (ALLYIS INC)" Date: Tue, 3 Nov 2026 04:19:06 -0600 Subject: [PATCH 1/2] tools: retain chronological stability evidence without admission Describe every chronological window in complete retained rounds, binding input hashes and image roles. Preserve the original assessor, samples, limits and four actual current-image failures. Add six numerical, actual-CLI and expected-error contracts; focused suite 64 pass, zero skip. No protocol, warmup, sample-count, application or dependency change. Design accepted: extend the existing assessor, without a new framework or deployment authority. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 14 ++ Session_starter.md | 12 ++ docs/CONTEXT_INSTRUCTIONS.md | 11 ++ docs/ENDPOINT_SNAPSHOT_REVISION.md | 59 ++++++++ .../ENGINEERING_LESSONS_AND_PATTERNS.md | 9 ++ scripts/performance/assess-membership.cjs | 68 ++++++++- scripts/test/membership-stability.test.cjs | 134 ++++++++++++++++++ 7 files changed, 303 insertions(+), 4 deletions(-) create mode 100644 scripts/test/membership-stability.test.cjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e1355ee..02e5ac9b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### 0.55.67 descriptive stability evidence (2026-11-03, proposed) + +- Add a non-admitting, hash-bound report over all chronological windows in + retained complete qualification rounds. Reuse original validation and + nearest-rank percentiles; preserve all samples and original failed checks. +- Add six numerical/CLI/negative regression contracts. Focused tooling: + 58 to 64 PASS, zero skips or failures. Negative tests require expected + validation errors rather than accepting unrelated exceptions. +- Actual current-image Linux qualification `41615647439` fails four first + same-image checks. Functional 66, privacy/logging 200 and exact cleanup + pass independently. No later block, protocol change or unchanged retry. + Existing application source, dependencies, version and lockfiles unchanged; + no new application count or deployment admission claimed. + ### 0.55.67 exact-checkout qualification image (2026-11-03, proposed) - Build and bind the actual qualification checkout rather than testing only diff --git a/Session_starter.md b/Session_starter.md index 641f86ff..a2a4d671 100644 --- a/Session_starter.md +++ b/Session_starter.md @@ -1,5 +1,17 @@ ## SCIMServer - Condensed Session Memory +**2026-11-03 calibration evidence:** branch +`fix/qualification-calibration-20261103` is based on reviewed merge6f846cf8. +Actual source-bound Linux run41615647439 fails4/12 same-image checks; +functional66/privacy200/exact cleanup PASS, later blocks not run. New +non-admitting descriptive collector retains all16,000 samples in64 windows, +hash-binds inputs and reproduces all original checks exactly. Focused tests +64/64 PASS after semantic RED; no protocol, warmup, sample or limit changed. +PR251 is draft again to avoid unchanged automatic retries while a prospective +stability design is prepared. Operator unavailable is not hardware, gate- +waiver or deployment approval. Latest failed evidence is protected separately +in qualification-failure-20261103T1011035881166Z with verified roundtrip. + **Last verified:** 2026-11-03 ### Exact-checkout qualification implementation (2026-11-03) diff --git a/docs/CONTEXT_INSTRUCTIONS.md b/docs/CONTEXT_INSTRUCTIONS.md index 1b8d2680..1562ba6f 100644 --- a/docs/CONTEXT_INSTRUCTIONS.md +++ b/docs/CONTEXT_INSTRUCTIONS.md @@ -9,6 +9,17 @@ ## Active Delivery Process +**2026-11-03 measured calibration gap:** #263 merged into #251 at6f846cf8; +ordinary exact-tip CI passes. Actual source-bound Linux run41615647439 fails +four same-image comparisons, while functional66/privacy200/exact cleanup +pass. No old/current comparison runs. #251 is draft again, not admitted. +The new descriptive collector retains all64 chronological windows/16,000 +samples and hash-binds inputs without changing any original assessment, +protocol or limit. Focused tooling64/64 PASS after RED. Numerical and actual +CLI tests ensure the report cannot claim convergence or deployment admission. +Review/prospective stabilization design remains separate; operator unavailable +does not authorize hardware, gate waiver or deployment. + **2026-11-03 exact-checkout qualification:** isolated implementation binds the actual .67 checkout image rather than treating frozen .58 performance as new artifact admission. Historical protocol/results and all latency/resource/sample diff --git a/docs/ENDPOINT_SNAPSHOT_REVISION.md b/docs/ENDPOINT_SNAPSHOT_REVISION.md index 006ad3a4..bc0a1fb6 100644 --- a/docs/ENDPOINT_SNAPSHOT_REVISION.md +++ b/docs/ENDPOINT_SNAPSHOT_REVISION.md @@ -7,6 +7,65 @@ Performance admission and the independent release gates remain open. ## Current-checkout image binding (2026-11-03, not released) +### Descriptive calibration prerequisite + +The new `--describe-stability` mode reuses the actual original assessor and +nearest-rank estimator. It describes all 64 chronological 250-sample windows +in a complete four-round block, retains both visits and every measured +sample, and records the source image roles plus SHA-256 hashes of the exact +protocol/round files. It cannot change a protocol, overwrite a report or +admit a release. Its output explicitly says uncertainty and warmup convergence +are not established. Window ratios are descriptive, not new acceptance +tests on smaller sample sets. + +Focused RED reports the missing collector assertion before implementation; +GREEN is **64/64**, comprising 58 existing contracts and six new cases. +The actual current-image report reproduces every original failed comparison +exactly and retains all 16,000 observations. In the baseline's second visit, +populated-read p95 across its four windows is 4.574270, 30.201782, +22.543207 and 16.514300 ms. The candidate's second visit is 2.465563, +3.469971, 3.396606 and 2.432962 ms. This is not explained simply by +discarding the first cold window. No extra benchmark was run. + +**Test-correctness RCA (Low):** during RED, a broad negative `assert.throws` +could succeed because the missing collector itself threw TypeError. Require +the expected receipt-validation assertion messages, alongside the positive +collector and real CLI contracts. This closes a false-green assertion class +at the earliest focused test stage. + +**Next boundary:** a reviewed prospective study must distinguish within-visit +and across-visit instability before selecting any changed warmup or +measurement policy. The operator was unavailable for the design question; +that is not approval for paid resources, relaxed bounds or deployment. +The pragmatic continuation is non-admitting analysis on existing evidence, +under the requested instruction to diagnose/fix issues. + +**Self-improvement applied:** test numerical windows and exact input/output +identity, not the mere presence of an analysis file or a thrown exception. +**Design disposition: accepted.** A small descriptive collector and shared +nearest-rank helper extend the existing assessor. No new framework, +production dependency, resource lifecycle or admission strategy is added. + +**Hosted follow-up:** reviewed prerequisite #263 is merged into #251 at +`6f846cf8`. Current-image Linux run +[41615647439](https://github.com/pranems/SCIMServer/actions/runs/41615647439) +successfully builds and binds actual checkout `bc60f7f3` and the immutable +0.55.67 image starting `92cd5024`. Its first same-image block fails four +of twelve checks: populated-read p95 ratio 0.571817, remove p95 1.123033, +empty-read p95 0.440720 and empty-read p99 0.844341. Faster out-of-range +controls also fail repeatability. All 17,628 checked requests, 16,000 retained +timings, 66 functional checks, 200 private-log checks, capture and exact +cleanup complete. The opposite control and both old/current blocks do not +run. No unchanged retry or new performance admission is claimed. + +The fixed 100 warmups do not establish stable observed timings: first-visit +populated-read p95 falls from 6.355253/6.801990 ms in the first 250 measured +samples to 2.652742/2.667220 ms in the last 250. App-cgroup CPU changes across +visits while SQL counts stay 25,591-25,593. This demonstrates a measurement +stability gap, not its unique cause. The next design requires prospective +calibration and review, not speculative production-code removal or +post-hoc selection of a favorable warmup count. + The existing qualification still selected a pinned 0.55.58 candidate even after source/dependency reconciliation proposed 0.55.67. A passing historical image comparison could not qualify that new artifact. The new prospective diff --git a/docs/strategy/ENGINEERING_LESSONS_AND_PATTERNS.md b/docs/strategy/ENGINEERING_LESSONS_AND_PATTERNS.md index 2067d0e4..fb498ccf 100644 --- a/docs/strategy/ENGINEERING_LESSONS_AND_PATTERNS.md +++ b/docs/strategy/ENGINEERING_LESSONS_AND_PATTERNS.md @@ -235,6 +235,15 @@ properties and runtime before adopting them. The narrow Linux qualification reuses the original lifecycle instead of building a new measurement framework. See [protocol and RCA](../ENDPOINT_SNAPSHOT_REVISION.md#prospective-linux-performance-qualification). +A fixed warmup count is not evidence that observations have stabilized. +Preserve chronological windows and distinguish within-visit drift from +across-visit differences before attributing a failed identical-image control +to production code. A descriptive report must preserve all original samples +and assessments, bind its inputs, and explicitly avoid causal attribution, +false-failure estimates or new admission criteria. Smaller windows are not +independent replications and do not inherit pooled-sample limits. See the +[current-image calibration prerequisite](../ENDPOINT_SNAPSHOT_REVISION.md#descriptive-calibration-prerequisite). + ### Failure evidence needs its own completion budget and contract A failed measurement is not the same thing as an absent receipt. Preserve diff --git a/scripts/performance/assess-membership.cjs b/scripts/performance/assess-membership.cjs index 77589788..0c431fbf 100644 --- a/scripts/performance/assess-membership.cjs +++ b/scripts/performance/assess-membership.cjs @@ -52,6 +52,10 @@ function rolesFor(protocol, block) { return { baseline: block.sameImage ? protocol.candidate : protocol.baseline, candidate: protocol.candidate }; } +function sortedPercentile(values, percentile) { + return values[Math.ceil(values.length * percentile / 100) - 1]; +} + function assessBlock(rows, block, roles) { assert.ok(Object.hasOwn(schedules, block.order), 'Unknown round order'); assert.equal(typeof block.sameImage, 'boolean'); @@ -78,9 +82,8 @@ function assessBlock(rows, block, roles) { const baseline = values('baseline'); const candidate = values('candidate'); return [50, 95, 99].map(percentile => { - const index = Math.ceil(2000 * percentile / 100) - 1; - const baselineMs = baseline[index]; - const candidateMs = candidate[index]; + const baselineMs = sortedPercentile(baseline, percentile); + const candidateMs = sortedPercentile(candidate, percentile); const ratio = candidateMs / baselineMs; return { operation, percentile, samplesPerRole: 2000, baselineMs, candidateMs, ratio, @@ -91,7 +94,49 @@ function assessBlock(rows, block, roles) { return { block: block.name, sameImage: block.sameImage, passed: checks.every(check => check.passed), checks }; } -module.exports = { assessBlock, checkoutProtocol, rolesFor, validateProtocol }; +function describeStability(rows, block, roles) { + const qualificationAssessment = assessBlock(rows, block, roles); + const visits = { baseline: 0, candidate: 0 }; + const rounds = rows.map((row, index) => ({ + round: index + 1, + role: row.role, + visit: ++visits[row.role], + operations: Object.fromEntries(operations.map(operation => { + const windows = Array.from({ length: 4 }, (_, window) => { + const startCycle = window * 250; + const values = row.samples[operation].slice(startCycle, startCycle + 250).sort((a, b) => a - b); + return { + startCycle, endCycleExclusive: startCycle + 250, sampleCount: values.length, + p50: sortedPercentile(values, 50), + p95: sortedPercentile(values, 95), + p99: sortedPercentile(values, 99), + }; + }); + return [operation, { + windows, + lastToFirstP95Ratio: windows[3].p95 / windows[0].p95, + }]; + })), + })); + return { + diagnosticOnly: true, + admissionEligible: false, + uncertaintyEstimated: false, + warmupConvergenceEstablished: false, + images: structuredClone(roles), + notes: [ + 'Chronological windows describe retained observations, not independent replications.', + 'Cycle bounds are zero-based measured cycles after the protocol warmup.', + 'Window ratios are not subject to the original pooled 2000-sample admission bounds.', + 'No causal attribution, false-failure probability or acceptable warmup is inferred.', + 'The original assessment and every sample are preserved without changing the protocol.', + ], + qualificationAssessment, + rounds, + }; +} + +module.exports = { assessBlock, checkoutProtocol, describeStability, rolesFor, validateProtocol }; if (require.main === module) { try { @@ -103,6 +148,21 @@ if (require.main === module) { JSON.parse(fs.readFileSync(identityPath, 'utf8')), sourceCommit, version, ); fs.writeFileSync(output, `${JSON.stringify(current, null, 2)}\n`, { flag: 'wx' }); + } else if (process.argv[2] === '--describe-stability') { + assert.equal(process.argv.length, 7, 'Explicit protocol, block, receipts and new output required'); + const [protocolPath, blockName, rowsPath, output] = process.argv.slice(3); + const protocol = JSON.parse(fs.readFileSync(protocolPath, 'utf8')); + const block = protocol.blocks.find(item => item.name === blockName); + assert.ok(block, 'Unknown preregistered block'); + const report = describeStability( + JSON.parse(fs.readFileSync(rowsPath, 'utf8')), block, rolesFor(protocol, block), + ); + const { createHash } = require('node:crypto'); + report.inputHashes = Object.fromEntries([ + ['protocol', protocolPath], ['rounds', rowsPath], + ].map(([name, file]) => [name, createHash('sha256').update(fs.readFileSync(file)).digest('hex')])); + fs.writeFileSync(output, `${JSON.stringify(report, null, 2)}\n`, { flag: 'wx' }); + console.log('Descriptive analysis complete; no stability or deployment admission.'); } else { const [protocolPath, blockName, rowsPath, output] = process.argv.slice(2); assert.ok(protocolPath && blockName && rowsPath && output, 'Explicit protocol, block, receipts and output required'); diff --git a/scripts/test/membership-stability.test.cjs b/scripts/test/membership-stability.test.cjs new file mode 100644 index 00000000..ee3e8833 --- /dev/null +++ b/scripts/test/membership-stability.test.cjs @@ -0,0 +1,134 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const assessor = require('../performance/assess-membership.cjs'); +const protocol = require('../performance/membership-protocol.json'); + +function fixture() { + const block = protocol.blocks[0]; + const roles = assessor.rolesFor(protocol, block); + const rows = ['candidate', 'baseline', 'baseline', 'candidate'].map(role => ({ + role, + version: roles[role].version, + image: roles[role].image, + status: 'PASS', + complete: true, + cpuLimit: 0.5, + memoryLimit: 1073741824, + warmupIterations: 100, + requestCount: 4407, + samples: Object.fromEntries( + ['add', 'populatedRead', 'remove', 'emptyRead'].map(operation => [ + operation, Array.from({ length: 1000 }, (_, index) => index + 1), + ]), + ), + })); + return { block, roles, rows }; +} + +test('descriptive stability analysis exists without creating admission', () => { + assert.equal(typeof assessor.describeStability, 'function', + 'Missing descriptive stability collector'); + const { rows, block, roles } = fixture(); + const before = structuredClone(rows); + const report = assessor.describeStability(rows, block, roles); + assert.equal(report.diagnosticOnly, true); + assert.equal(report.admissionEligible, false); + assert.equal(report.uncertaintyEstimated, false); + assert.equal(report.warmupConvergenceEstablished, false); + assert.deepEqual(report.qualificationAssessment, assessor.assessBlock(rows, block, roles)); + assert.deepEqual(rows, before, 'Retained samples must not be sorted or rewritten in place'); +}); + +test('all chronological windows retain exact boundaries and nearest-rank percentiles', () => { + const { rows, block, roles } = fixture(); + const report = assessor.describeStability(rows, block, roles); + assert.equal(report.rounds.length, 4); + assert.deepEqual(report.rounds.map(round => round.visit), [1, 1, 2, 2]); + const operation = report.rounds[0].operations.add; + assert.equal(operation.windows.length, 4); + assert.deepEqual(operation.windows[0], { + startCycle: 0, endCycleExclusive: 250, sampleCount: 250, + p50: 125, p95: 238, p99: 248, + }); + assert.deepEqual(operation.windows[3], { + startCycle: 750, endCycleExclusive: 1000, sampleCount: 250, + p50: 875, p95: 988, p99: 998, + }); + assert.equal(operation.lastToFirstP95Ratio, 988 / 238); + for (const round of report.rounds) { + for (const entry of Object.values(round.operations)) { + assert.equal(entry.windows.reduce((sum, window) => sum + window.sampleCount, 0), 1000); + } + } +}); + +test('pooled same-image acceptance is not evidence of within-round convergence', () => { + const { rows, block, roles } = fixture(); + const report = assessor.describeStability(rows, block, roles); + assert.equal(report.qualificationAssessment.passed, true); + assert.ok(report.rounds[0].operations.emptyRead.lastToFirstP95Ratio > 4); + assert.equal(report.admissionEligible, false); + assert.equal(report.warmupConvergenceEstablished, false); +}); + +test('descriptive analysis retains failure instead of converting it to a pass', () => { + const { rows, block, roles } = fixture(); + for (const row of rows.filter(row => row.role === 'candidate')) { + row.samples.remove = row.samples.remove.map(value => value * 2); + } + const report = assessor.describeStability(rows, block, roles); + assert.equal(report.qualificationAssessment.passed, false); + assert.equal(report.admissionEligible, false); +}); + +test('partial, malformed and instrumented receipts remain explicit failures', () => { + for (const mutate of [ + rows => { rows[0].complete = false; }, + rows => { rows[0].samples.add.pop(); }, + rows => { rows[0].samples.add[0] = NaN; }, + rows => { rows[0].diagnosticOnly = true; }, + rows => { rows.reverse(); rows[0].role = 'baseline'; }, + ]) { + const { rows, block, roles } = fixture(); + mutate(rows); + assert.throws(() => assessor.describeStability(rows, block, roles), + /Partial receipts|Every measured sample|Invalid latency sample|Diagnostic receipts|Actual role order/); + } +}); + +test('actual descriptive CLI writes a non-admitting report and refuses overwrite', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'membership-stability-')); + try { + const { rows, block } = fixture(); + const protocolPath = path.join(directory, 'protocol.json'); + const rowsPath = path.join(directory, 'rounds.json'); + const output = path.join(directory, 'stability.json'); + fs.writeFileSync(protocolPath, JSON.stringify(protocol)); + fs.writeFileSync(rowsPath, JSON.stringify(rows)); + const args = [ + path.resolve(__dirname, '../performance/assess-membership.cjs'), + '--describe-stability', protocolPath, block.name, rowsPath, output, + ]; + const result = spawnSync(process.execPath, args, { encoding: 'utf8', timeout: 10000 }); + assert.equal(result.status, 0, result.stderr); + const report = JSON.parse(fs.readFileSync(output, 'utf8')); + assert.equal(report.admissionEligible, false); + assert.equal(report.rounds.length, 4); + const { createHash } = require('node:crypto'); + assert.equal(report.inputHashes.rounds, + createHash('sha256').update(fs.readFileSync(rowsPath)).digest('hex')); + assert.equal(report.inputHashes.protocol, + createHash('sha256').update(fs.readFileSync(protocolPath)).digest('hex')); + const original = fs.readFileSync(output); + const repeated = spawnSync(process.execPath, args, { encoding: 'utf8', timeout: 10000 }); + assert.equal(repeated.status, 1); + assert.match(repeated.stderr, /EEXIST/); + assert.deepEqual(fs.readFileSync(output), original); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } +}); From d374a3d00736d65143b623dddbc5876f98b90ecd Mon Sep 17 00:00:00 2001 From: "Prashant Rane (ALLYIS INC)" Date: Tue, 3 Nov 2026 04:54:28 -0600 Subject: [PATCH 2/2] fix(perf): retain raw observations and bind parsed evidence bytes Reproduce and close both independent-review evidence-integrity findings. Preserve all original qualification checks and failed results. Focused tooling passes 66 of 66; full protected validation remains the push gate. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 9 ++- Session_starter.md | 6 +- docs/CONTEXT_INSTRUCTIONS.md | 5 +- docs/ENDPOINT_SNAPSHOT_REVISION.md | 11 +++- .../ENGINEERING_LESSONS_AND_PATTERNS.md | 8 +++ scripts/performance/assess-membership.cjs | 14 +++-- scripts/test/membership-stability.test.cjs | 56 +++++++++++++++++++ 7 files changed, 99 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02e5ac9b..859968e4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,9 +12,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add a non-admitting, hash-bound report over all chronological windows in retained complete qualification rounds. Reuse original validation and nearest-rank percentiles; preserve all samples and original failed checks. -- Add six numerical/CLI/negative regression contracts. Focused tooling: - 58 to 64 PASS, zero skips or failures. Negative tests require expected +- Add eight numerical/CLI/negative regression contracts. Focused tooling: + 58 to 66 PASS, zero skips or failures. Negative tests require expected validation errors rather than accepting unrelated exceptions. +- Close two independent-review findings after reproducing both: retain the + actual unsorted observations, not only summaries, and hash the same + once-read buffers used for parsing. Reconstruct all 16,000 real observations + and preserve all twelve original checks/four failures in a separate + corrected report. The CLI regression changes both inputs after reading. - Actual current-image Linux qualification `41615647439` fails four first same-image checks. Functional 66, privacy/logging 200 and exact cleanup pass independently. No later block, protocol change or unchanged retry. diff --git a/Session_starter.md b/Session_starter.md index a2a4d671..5bfca1f8 100644 --- a/Session_starter.md +++ b/Session_starter.md @@ -6,7 +6,11 @@ Actual source-bound Linux run41615647439 fails4/12 same-image checks; functional66/privacy200/exact cleanup PASS, later blocks not run. New non-admitting descriptive collector retains all16,000 samples in64 windows, hash-binds inputs and reproduces all original checks exactly. Focused tests -64/64 PASS after semantic RED; no protocol, warmup, sample or limit changed. +66/66 PASS after semantic RED, including review regressions for missing raw +observations and a second-read hash race. Corrected output includes all +unsorted samples and hashes exactly the once-read parsed buffers; all12 +original checks/four failures remain identical. No protocol, warmup, sample +or limit changed. PR251 is draft again to avoid unchanged automatic retries while a prospective stability design is prepared. Operator unavailable is not hardware, gate- waiver or deployment approval. Latest failed evidence is protected separately diff --git a/docs/CONTEXT_INSTRUCTIONS.md b/docs/CONTEXT_INSTRUCTIONS.md index 1562ba6f..69837fdf 100644 --- a/docs/CONTEXT_INSTRUCTIONS.md +++ b/docs/CONTEXT_INSTRUCTIONS.md @@ -15,7 +15,10 @@ four same-image comparisons, while functional66/privacy200/exact cleanup pass. No old/current comparison runs. #251 is draft again, not admitted. The new descriptive collector retains all64 chronological windows/16,000 samples and hash-binds inputs without changing any original assessment, -protocol or limit. Focused tooling64/64 PASS after RED. Numerical and actual +protocol or limit. Focused tooling66/66 PASS after RED. Review exposed and +closed missing raw observations and a second-read provenance race: output +now reconstructs every original sample in order, and hashes share exactly +the parsed input buffers. Numerical, adversarial input-replacement and actual CLI tests ensure the report cannot claim convergence or deployment admission. Review/prospective stabilization design remains separate; operator unavailable does not authorize hardware, gate waiver or deployment. diff --git a/docs/ENDPOINT_SNAPSHOT_REVISION.md b/docs/ENDPOINT_SNAPSHOT_REVISION.md index bc0a1fb6..0ee520a8 100644 --- a/docs/ENDPOINT_SNAPSHOT_REVISION.md +++ b/docs/ENDPOINT_SNAPSHOT_REVISION.md @@ -20,7 +20,14 @@ tests on smaller sample sets. Focused RED reports the missing collector assertion before implementation; GREEN is **64/64**, comprising 58 existing contracts and six new cases. -The actual current-image report reproduces every original failed comparison +Independent review then found that summaries did not actually include the +raw observations claimed above, and input hashes used a second pathname +read. The corrected report includes each window's unsorted observations; +percentiles use a separate sorted copy. Parsing and hashing now share the +same once-read buffers. Both defects were reproduced before correction. +Focused GREEN is now **66/66**, including chronological reconstruction and +an actual CLI test that changes both input paths after their first read. +The separately generated corrected current-image report reproduces every original failed comparison exactly and retains all 16,000 observations. In the baseline's second visit, populated-read p95 across its four windows is 4.574270, 30.201782, 22.543207 and 16.514300 ms. The candidate's second visit is 2.465563, @@ -86,6 +93,8 @@ resource limits, operation order and latency bounds are not modified. | Missing isolated root/web tooling / Low | The first chosen web build and documentation check failed on missing vite/jsdom. Restore each worktree-owned dependency set using npm ci only after its observed missing-dependency failure. The protected hook owns consolidated validation. | Prerequisite build/documentation checks; no shared dependency mutation or lock regeneration. | | Missing generated Prisma model types / Low | API dependencies were deliberately installed with scripts disabled, leaving the generated client absent. The API build exposed model-type errors; explicit Prisma generation succeeds with a codegen-only local URL and no database connection. The protected hook owns the subsequent full build/test verification. | First API build; normal code-generation prerequisite, not a production source defect. | | Stale mutable local base / Medium | The first local build reused Node 24.16.0 despite newer historical images. Require docker build --pull, retain the first image identity, and verify the refreshed runtime separately. | Actual built-image runtime readback; earliest possible gate is a fresh-base build plus runtime readback. | +| Raw-observation retention claim / Medium | Window summaries retained counts and percentiles but omitted raw observations. Include the unsorted samples and prove all 16,000 can be reconstructed in original order from the report alone. Existing immutable reports remain unchanged; corrected output uses a new path. | Independent review; earliest possible focused test previously checked counts rather than reconstructability. The new outcome assertion closes that escape. | +| Input-byte provenance race / Medium | Hashing reread mutable paths after parsing, allowing different bytes to identify the analysis. Read each input once and use that same buffer for parsing and hashing. An adversarial real CLI preload changes both files immediately after each read and verifies the hashes still identify the parsed bytes. | Independent review; earliest possible focused CLI test previously hashed only unchanged files. The deterministic replacement test closes that escape. | The initial source-binding test failed before implementation. The complete focused suite then passed 58/58, up from 51, with no skips. A separate real diff --git a/docs/strategy/ENGINEERING_LESSONS_AND_PATTERNS.md b/docs/strategy/ENGINEERING_LESSONS_AND_PATTERNS.md index fb498ccf..75d3deb4 100644 --- a/docs/strategy/ENGINEERING_LESSONS_AND_PATTERNS.md +++ b/docs/strategy/ENGINEERING_LESSONS_AND_PATTERNS.md @@ -244,6 +244,14 @@ false-failure estimates or new admission criteria. Smaller windows are not independent replications and do not inherit pooled-sample limits. See the [current-image calibration prerequisite](../ENDPOINT_SNAPSHOT_REVISION.md#descriptive-calibration-prerequisite). +Counts and percentiles are not retained observations. Reconstruct the complete +chronological input from the report in a regression test, including unsorted +values. Bind provenance to the exact buffers parsed, not a second read of +the same pathname. A deterministic CLI test must replace or update the inputs +after their first read and still obtain hashes of the analyzed bytes. +Independent review caught both gaps in the descriptive collector; they are +now closed in-place without changing any benchmark or acceptance result. + ### Failure evidence needs its own completion budget and contract A failed measurement is not the same thing as an absent receipt. Preserve diff --git a/scripts/performance/assess-membership.cjs b/scripts/performance/assess-membership.cjs index 0c431fbf..2c41c44d 100644 --- a/scripts/performance/assess-membership.cjs +++ b/scripts/performance/assess-membership.cjs @@ -104,9 +104,11 @@ function describeStability(rows, block, roles) { operations: Object.fromEntries(operations.map(operation => { const windows = Array.from({ length: 4 }, (_, window) => { const startCycle = window * 250; - const values = row.samples[operation].slice(startCycle, startCycle + 250).sort((a, b) => a - b); + const samples = row.samples[operation].slice(startCycle, startCycle + 250); + const values = [...samples].sort((a, b) => a - b); return { startCycle, endCycleExclusive: startCycle + 250, sampleCount: values.length, + samples, p50: sortedPercentile(values, 50), p95: sortedPercentile(values, 95), p99: sortedPercentile(values, 99), @@ -151,16 +153,18 @@ if (require.main === module) { } else if (process.argv[2] === '--describe-stability') { assert.equal(process.argv.length, 7, 'Explicit protocol, block, receipts and new output required'); const [protocolPath, blockName, rowsPath, output] = process.argv.slice(3); - const protocol = JSON.parse(fs.readFileSync(protocolPath, 'utf8')); + const protocolBytes = fs.readFileSync(protocolPath); + const rowsBytes = fs.readFileSync(rowsPath); + const protocol = JSON.parse(protocolBytes.toString('utf8')); const block = protocol.blocks.find(item => item.name === blockName); assert.ok(block, 'Unknown preregistered block'); const report = describeStability( - JSON.parse(fs.readFileSync(rowsPath, 'utf8')), block, rolesFor(protocol, block), + JSON.parse(rowsBytes.toString('utf8')), block, rolesFor(protocol, block), ); const { createHash } = require('node:crypto'); report.inputHashes = Object.fromEntries([ - ['protocol', protocolPath], ['rounds', rowsPath], - ].map(([name, file]) => [name, createHash('sha256').update(fs.readFileSync(file)).digest('hex')])); + ['protocol', protocolBytes], ['rounds', rowsBytes], + ].map(([name, bytes]) => [name, createHash('sha256').update(bytes).digest('hex')])); fs.writeFileSync(output, `${JSON.stringify(report, null, 2)}\n`, { flag: 'wx' }); console.log('Descriptive analysis complete; no stability or deployment admission.'); } else { diff --git a/scripts/test/membership-stability.test.cjs b/scripts/test/membership-stability.test.cjs index ee3e8833..492b8ae3 100644 --- a/scripts/test/membership-stability.test.cjs +++ b/scripts/test/membership-stability.test.cjs @@ -52,10 +52,12 @@ test('all chronological windows retain exact boundaries and nearest-rank percent assert.equal(operation.windows.length, 4); assert.deepEqual(operation.windows[0], { startCycle: 0, endCycleExclusive: 250, sampleCount: 250, + samples: Array.from({ length: 250 }, (_, index) => index + 1), p50: 125, p95: 238, p99: 248, }); assert.deepEqual(operation.windows[3], { startCycle: 750, endCycleExclusive: 1000, sampleCount: 250, + samples: Array.from({ length: 250 }, (_, index) => index + 751), p50: 875, p95: 988, p99: 998, }); assert.equal(operation.lastToFirstP95Ratio, 988 / 238); @@ -66,6 +68,18 @@ test('all chronological windows retain exact boundaries and nearest-rank percent } }); +test('the report independently retains every raw observation in its original chronological order', () => { + const { rows, block, roles } = fixture(); + for (const row of rows) row.samples.add.reverse(); + const report = assessor.describeStability(rows, block, roles); + for (const [index, round] of report.rounds.entries()) { + for (const [operation, entry] of Object.entries(round.operations)) { + assert.deepEqual(entry.windows.flatMap(window => window.samples), rows[index].samples[operation], + 'The report must contain all raw observations, not only summaries or their sorted order'); + } + } +}); + test('pooled same-image acceptance is not evidence of within-round convergence', () => { const { rows, block, roles } = fixture(); const report = assessor.describeStability(rows, block, roles); @@ -132,3 +146,45 @@ test('actual descriptive CLI writes a non-admitting report and refuses overwrite fs.rmSync(directory, { recursive: true, force: true }); } }); + +test('actual CLI hashes exactly the input bytes it parsed even if the input paths change after reading', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'membership-stability-input-race-')); + try { + const { rows, block } = fixture(); + const protocolPath = path.join(directory, 'protocol.json'); + const rowsPath = path.join(directory, 'rounds.json'); + const output = path.join(directory, 'stability.json'); + const preload = path.join(directory, 'replace-after-read.cjs'); + const originalProtocol = Buffer.from(JSON.stringify(protocol)); + const originalRows = Buffer.from(JSON.stringify(rows)); + fs.writeFileSync(protocolPath, originalProtocol); + fs.writeFileSync(rowsPath, originalRows); + fs.writeFileSync(preload, ` + const fs = require('node:fs'); + const read = fs.readFileSync; + const targets = new Set([process.env.STABILITY_TEST_PROTOCOL, process.env.STABILITY_TEST_ROUNDS]); + fs.readFileSync = function(file, ...args) { + const content = read.call(this, file, ...args); + if (targets.has(file)) { + fs.writeFileSync(file, Buffer.concat([Buffer.from(content), Buffer.from('\\n')])); + } + return content; + }; + `); + const result = spawnSync(process.execPath, [ + '--require', preload, path.resolve(__dirname, '../performance/assess-membership.cjs'), + '--describe-stability', protocolPath, block.name, rowsPath, output, + ], { encoding: 'utf8', timeout: 10000, env: { ...process.env, + STABILITY_TEST_PROTOCOL: protocolPath, STABILITY_TEST_ROUNDS: rowsPath } }); + assert.equal(result.status, 0, result.stderr); + assert.notDeepEqual(fs.readFileSync(protocolPath), originalProtocol, 'Protocol mutation actually executed'); + assert.notDeepEqual(fs.readFileSync(rowsPath), originalRows, 'Rounds mutation actually executed'); + const report = JSON.parse(fs.readFileSync(output, 'utf8')); + const { createHash } = require('node:crypto'); + assert.equal(report.inputHashes.protocol, createHash('sha256').update(originalProtocol).digest('hex'), + 'Protocol hash must identify the parsed buffer, not a later pathname read'); + assert.equal(report.inputHashes.rounds, createHash('sha256').update(originalRows).digest('hex'), + 'Rounds hash must identify the parsed buffer, not a later pathname read'); + assert.deepEqual(report.qualificationAssessment, assessor.assessBlock(rows, block, assessor.rolesFor(protocol, block))); + } finally { fs.rmSync(directory, { recursive: true }); } +});