From f83eecfc3f4e4dff27026b28ecc83ccad6af249d Mon Sep 17 00:00:00 2001 From: "Prashant Rane (ALLYIS INC)" Date: Tue, 3 Nov 2026 03:20:11 -0600 Subject: [PATCH 1/2] ci: qualify the exact checkout image Bind source, package version and immutable local image identity before qualification. Preserve frozen historical results, resource limits and latency bounds. Refresh mutable bases and reserve build/cleanup/upload time within the existing job cap. Focused contracts: 58 pass; complete local tooling: 227 pass, 8 explicit skips. No performance or deployment admission claimed. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../workflows/performance-qualification.yml | 32 ++++- CHANGELOG.md | 18 +++ Session_starter.md | 19 ++- docs/CONTEXT_INSTRUCTIONS.md | 11 ++ ...LOYMENT_INFRASTRUCTURE_AND_FORM_FACTORS.md | 39 +++++- docs/ENDPOINT_SNAPSHOT_REVISION.md | 38 ++++++ scripts/performance/assess-membership.cjs | 51 ++++++-- .../run-membership-qualification.ps1 | 33 ++++- scripts/test/membership-performance.test.cjs | 113 +++++++++++++++++- 9 files changed, 330 insertions(+), 24 deletions(-) diff --git a/.github/workflows/performance-qualification.yml b/.github/workflows/performance-qualification.yml index a61f4c54..c354e7b6 100644 --- a/.github/workflows/performance-qualification.yml +++ b/.github/workflows/performance-qualification.yml @@ -32,11 +32,37 @@ jobs: timeout-minutes: 3 run: node --test scripts/test/membership-performance.test.cjs + - name: Build and bind exact checkout image + timeout-minutes: 10 + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $source = (git rev-parse HEAD).Trim() + if ($LASTEXITCODE -ne 0) { throw 'Exact checkout source unavailable.' } + git diff --exit-code + if ($LASTEXITCODE -ne 0) { throw 'Tracked checkout inputs were modified.' } + $directory = Join-Path $env:GITHUB_WORKSPACE 'test-results/performance-image' + New-Item -ItemType Directory -Path $directory -ErrorAction Stop | Out-Null + $version = (Get-Content api/package.json -Raw | ConvertFrom-Json).version + docker build --pull --label "org.opencontainers.image.revision=$source" ` + --build-arg "IMAGE_TAG=$source" --iidfile (Join-Path $directory 'image-id.txt') . + if ($LASTEXITCODE -ne 0) { throw 'Exact-checkout image build failed.' } + $image = (Get-Content (Join-Path $directory 'image-id.txt') -Raw).Trim() + docker image inspect --format ` + '{"id":{{json .Id}},"os":{{json .Os}},"architecture":{{json .Architecture}},"sourceCommit":{{json (index .Config.Labels "org.opencontainers.image.revision")}}}' ` + $image | Set-Content (Join-Path $directory 'identity.json') + if ($LASTEXITCODE -ne 0) { throw 'Exact-checkout image identity unavailable.' } + node scripts/performance/assess-membership.cjs --checkout-image ` + scripts/performance/membership-protocol.json (Join-Path $directory 'identity.json') ` + $source $version (Join-Path $directory 'protocol.json') + if ($LASTEXITCODE -ne 0) { throw 'Current-image protocol binding failed.' } + - name: Run fixed same-image controls before old/current comparisons - timeout-minutes: 65 + timeout-minutes: 55 shell: pwsh run: | ./scripts/performance/run-membership-qualification.ps1 ` + -ProtocolPath (Join-Path $env:GITHUB_WORKSPACE 'test-results/performance-image/protocol.json') ` -OutputDirectory (Join-Path $env:GITHUB_WORKSPACE 'test-results/performance-qualification') - name: Retain all completed and failed evidence @@ -45,6 +71,8 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: membership-qualification-${{ github.run_id }}-${{ github.run_attempt }} - path: test-results/performance-qualification/ + path: | + test-results/performance-qualification/ + test-results/performance-image/ if-no-files-found: error retention-days: 14 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f5f763f..3e1355ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### 0.55.67 exact-checkout qualification image (2026-11-03, proposed) + +- Build and bind the actual qualification checkout rather than testing only + the frozen 0.55.58 artifact. Preserve the historical manifest/results; + generate a separate immutable candidate protocol with exact source, + package version, Linux/x64 architecture and revision verification. +- Refresh mutable Docker bases, retain identity/protocol artifacts, reject + unrelated sources before allocation and reserve build/cleanup/upload time + within the existing 75-minute job cap. No latency/resource/sample limit, + serving source, dependency or lockfile change. +- Qualification contract coverage increases 51 to 58 passing cases, zero + skips in the focused run. Complete local tooling: 227 PASS / 8 explicit + retained-fixture or opt-in skips / 0 FAIL. The refreshed image reports + 0.55.67 on Node24.22.2 and generates a real source-bound protocol. + Hosted execution and release admission remain separate pending evidence. + Existing application test counts are not rerun or relabelled by this + tooling change. No deployment or performance PASS is claimed. + ### 0.55.67 Hono-master performance reconciliation (2026-10-30, proposed) - Normally integrate reviewed master `eb733f61` after Hono #262 merged at diff --git a/Session_starter.md b/Session_starter.md index 2a8a7d35..641f86ff 100644 --- a/Session_starter.md +++ b/Session_starter.md @@ -1,6 +1,23 @@ ## SCIMServer - Condensed Session Memory -**Last verified:** 2026-10-30 +**Last verified:** 2026-11-03 + +### Exact-checkout qualification implementation (2026-11-03) + +Isolated branch fix/qualification-image-binding-20261103 preserves the clean +abdef720 candidate and its dependencies. The qualification workflow now builds +the actual checkout, generates a separate immutable local-image protocol and +rejects wrong source/version/revision before benchmark allocation. Original +historical protocol, latency limits and failed receipts remain unchanged. +Focused contracts pass 58/58 after semantic RED. Broad tooling initially +required missing pg in this new worktree; isolated API npm ci restored it +without changing any manifest/lock. Full tooling now227 PASS /8 explicit +retained-fixture or opt-in skips /0 FAIL. First local .67 image revealed +cached Node24.16.0; --pull rebuild passes and reports0.55.67/Node24.22.2. +Actual image identity generates a source-bound protocol successfully. +No shared dependencies, release, target +deployment, paid host or admission waiver. Actual hosted qualification, +reviewed publication and complete dev/canary target gates remain required. ### Current Hono-master performance reconciliation (2026-10-30) diff --git a/docs/CONTEXT_INSTRUCTIONS.md b/docs/CONTEXT_INSTRUCTIONS.md index 218a6121..1b8d2680 100644 --- a/docs/CONTEXT_INSTRUCTIONS.md +++ b/docs/CONTEXT_INSTRUCTIONS.md @@ -9,6 +9,17 @@ ## Active Delivery Process +**2026-11-03 exact-checkout qualification:** isolated implementation binds the +actual .67 checkout image rather than treating frozen .58 performance as new +artifact admission. Historical protocol/results and all latency/resource/sample +bounds remain unchanged. Current source/version/revision and Linux/x64 +identity are verified before allocation; mutable bases are refreshed. +Focused contracts pass58/58; full local tooling227 PASS /8 explicit skips +/0 FAIL. Refreshed local image reports0.55.67/Node24.22.2 and a real +source-bound protocol is generated. Hosted execution and normal reviewed +publication remain independent. No performance PASS, +paid capacity or target deployment is inferred. + **2026-10-30 current performance reconciliation:** Hono #262 merged at `eb733f61e7f25d1d642fe746075e266c3b19ad60`, master 0.55.66. Proposed 0.55.67 normally integrates it into held performance `ff2d2377`, preserving both diff --git a/docs/DEPLOYMENT_INFRASTRUCTURE_AND_FORM_FACTORS.md b/docs/DEPLOYMENT_INFRASTRUCTURE_AND_FORM_FACTORS.md index 8dd6709f..3da3a6e1 100644 --- a/docs/DEPLOYMENT_INFRASTRUCTURE_AND_FORM_FACTORS.md +++ b/docs/DEPLOYMENT_INFRASTRUCTURE_AND_FORM_FACTORS.md @@ -7,20 +7,49 @@ --- +## Exact-checkout qualification image (2026-11-03 source change) + +The qualification workflow now builds its actual checkout, including current +dependency fixes, instead of measuring the frozen 0.55.58 image and implying +that the result qualifies 0.55.67. The original manifest and failed receipts +remain unchanged. A separate generated protocol binds the candidate's immutable +local image ID, exact checkout commit and package version. Image OS, +architecture and revision are verified; unrelated checkout/version/revision +values fail before benchmark resource allocation. The baseline remains the +original immutable public image. No package is published by this workflow. + +The Docker build uses `--pull`: the first local build otherwise reused a +cached Node 24.16.0 base, older than the recorded 0.55.20 image runtime. +Building the requested source alone does not establish a current base. +The refreshed base and actual final runtime must be measured independently. +The source-bound lane reserves ten minutes for the image build, uses a +45-minute cooperative measurement cap and a 55-minute measurement-step cap, +and preserves the 75-minute job cap and four-minute artifact upload. +Historical standalone protocol execution retains its original 60-minute +work cap. Resource limits, samples, warmups, all twelve latency bounds and +the four fail-stop blocks are unchanged. Generated identity/protocol evidence +is uploaded alongside completed or failed measurement evidence. + +This is source-level workflow verification, not new Azure estate evidence +or a passing performance result. Existing public GitHub-hosted Linux capacity +is used; no dedicated Azure host, new paid resource or deployment is introduced. + ## Prospective performance qualification runner (2026-10-22 source change) [performance-qualification.yml](../.github/workflows/performance-qualification.yml) uses the repository's existing public GitHub Actions service with an `ubuntu-24.04` runner. It provisions no Azure resource or new paid host, grants only `contents: read`, serializes qualification campaigns and permits no -deployment or package publication. A 60-minute cooperative work deadline, -65-minute qualification-step cap and 75-minute job cap reserve time for -cleanup/upload. Checkout, contracts and upload have 2/3/4-minute caps. +deployment or package publication. The current source-bound lane uses the +45/55/75-minute work/step/job limits described above; the original frozen +standalone campaign retains its 60-minute work limit. Checkout, contracts +and upload have 2/3/4-minute caps. Completed/failed evidence is retained for 14 days. The workflow starts on relevant pull requests or explicit dispatch, not every application change. -The [fixed manifest](../scripts/performance/membership-protocol.json) pins -actual application/PostgreSQL images and the original app resource limits. +The [fixed historical manifest](../scripts/performance/membership-protocol.json) +pins the original application/PostgreSQL images and resource limits. Current +checkout qualification generates a separate source-bound candidate protocol. Separate owned app/database processes are created per four-round block; each role's app/database is reused for its second visit, preserving the existing within-block history. Two A/A blocks must pass before the two A/B diff --git a/docs/ENDPOINT_SNAPSHOT_REVISION.md b/docs/ENDPOINT_SNAPSHOT_REVISION.md index 63e9285b..08f32bbe 100644 --- a/docs/ENDPOINT_SNAPSHOT_REVISION.md +++ b/docs/ENDPOINT_SNAPSHOT_REVISION.md @@ -5,6 +5,44 @@ **Status:** Implemented and tested locally, not released or deployed. Performance admission and the independent release gates remain open. +## Current-checkout image binding (2026-11-03, not released) + +The existing qualification still selected a pinned 0.55.58 candidate even +after source/dependency reconciliation proposed 0.55.67. A passing historical +image comparison could not qualify that new artifact. The new prospective +workflow builds its exact checkout and generates a separate candidate protocol +from the immutable local image ID, revision label and package version. +The frozen historical manifest and every failed result remain unchanged. +Current API/schema/browser source, dependencies, warmups, sample counts, +resource limits, operation order and latency bounds are not modified. + +### Execution issues and prevention + +| Issue | Root cause, resolution and prevention | Detection and escape | +|---|---|---| +| Stale candidate image / High | Source revision changed while the protocol stayed pinned to 0.55.58. Bind the real checkout build and reject wrong source/version/revision before allocation. New contracts exercise actual preflight and both same-image/old-current role selection. | Release investigation; earliest possible check was a checkout-to-image preflight, now added. | +| CLI argument count / Low | New flag plus five values was initially counted as nine process arguments rather than eight. The real CLI test failed, then passed after correction; it also rejects overwriting retained protocol evidence. | Focused actual CLI contract; no escape beyond its earliest gate. | +| Build and cleanup budget / Medium | Adding a build inside the measurement step could consume reserved cleanup time. A separate ten-minute build step and 45/55-minute work/step caps preserve the 75-minute job budget. Actual deadline tests cover both historical and source-bound caps. | Focused workflow/deadline contract; corrected before workflow execution. | +| Missing isolated-worktree pg dependency / Low | The broader Node tooling suite requires the API dependency set. Restore dependencies only in the new isolated worktree after MODULE_NOT_FOUND; no shared node_modules mutation or lock regeneration. | First complete tooling run; not a benchmark/application failure. | +| Stale mutable local base / Medium | The first local build reused Node 24.16.0 despite newer historical images. Require docker build --pull, retain the first image identity, and verify the refreshed runtime separately. | Actual built-image runtime readback; earliest possible gate is a fresh-base build plus runtime readback. | + +The initial source-binding test failed before implementation. The complete +focused suite then passed 58/58, up from 51, with no skips. A separate real +workflow assertion reproduced missing `--pull` as RED before that correction. +Complete local tooling passes227 cases with8 explicit retained-fixture/opt-in +skips and0 failures. The refreshed exact-source image reports0.55.67 on +Node24.22.2; its actual identity generates the separate immutable protocol. +Manifest/lock bytes remain unchanged. No hosted qualification, final release +or new performance PASS is claimed here. + +**Self-improvement applied:** release qualification must bind source, +dependency-bearing image and actual runtime, not merely a historical version +string. **Design disposition: accepted.** Extend the existing protocol +validator and runner with one small pure binding helper; no serving service, +new framework, new paid host or speculative measurement strategy is added. +Provenance is this continuation's exact-source build and focused RED/GREEN +output, not a reconstruction of earlier full transcripts. + ## Current reviewed-source union (2026-10-29) The preserved 0.55.65 performance candidate now normally integrates reviewed diff --git a/scripts/performance/assess-membership.cjs b/scripts/performance/assess-membership.cjs index bb41dd4c..77589788 100644 --- a/scripts/performance/assess-membership.cjs +++ b/scripts/performance/assess-membership.cjs @@ -21,12 +21,31 @@ function validateProtocol(protocol) { { name: 'ab-abba', sameImage: false, order: 'ABBA' }, ]); for (const role of ['baseline', 'candidate']) { - assert.match(protocol[role].image, /^ghcr\.io\/pranems\/scimserver@sha256:[a-f0-9]{64}$/); + if (role === 'candidate' && protocol.candidate.sourceCommit !== undefined) { + assert.match(protocol.candidate.sourceCommit, /^[a-f0-9]{40}$/); + assert.match(protocol.candidate.image, /^sha256:[a-f0-9]{64}$/); + } else { + assert.match(protocol[role].image, /^ghcr\.io\/pranems\/scimserver@sha256:[a-f0-9]{64}$/); + } assert.match(protocol[role].version, /^\d+\.\d+\.\d+$/); } assert.match(protocol.postgres, /^postgres@sha256:[a-f0-9]{64}$/); } +function checkoutProtocol(protocol, identity, sourceCommit, version) { + validateProtocol(protocol); + assert.match(sourceCommit, /^[a-f0-9]{40}$/, 'Exact checkout commit required'); + assert.match(version, /^\d+\.\d+\.\d+$/, 'Exact package version required'); + assert.match(identity.id, /^sha256:[a-f0-9]{64}$/, 'Immutable local image ID required'); + assert.equal(identity.os, 'linux', 'Linux image required'); + assert.equal(identity.architecture, 'amd64', 'Original x64 measurement architecture required'); + assert.equal(identity.sourceCommit, sourceCommit, 'Image revision must match the checkout'); + const current = structuredClone(protocol); + current.candidate = { image: identity.id, version, sourceCommit }; + validateProtocol(current); + return current; +} + function rolesFor(protocol, block) { validateProtocol(protocol); assert.deepEqual(protocol.blocks.find(item => item.name === block.name), block); @@ -72,19 +91,29 @@ function assessBlock(rows, block, roles) { return { block: block.name, sameImage: block.sameImage, passed: checks.every(check => check.passed), checks }; } -module.exports = { assessBlock, rolesFor, validateProtocol }; +module.exports = { assessBlock, checkoutProtocol, rolesFor, validateProtocol }; if (require.main === module) { try { - const [protocolPath, blockName, rowsPath, output] = process.argv.slice(2); - assert.ok(protocolPath && blockName && rowsPath && output, 'Explicit protocol, block, receipts and output required'); - const protocol = JSON.parse(fs.readFileSync(protocolPath, 'utf8')); - const block = protocol.blocks.find(item => item.name === blockName); - assert.ok(block, 'Unknown preregistered block'); - const result = assessBlock(JSON.parse(fs.readFileSync(rowsPath, 'utf8')), block, rolesFor(protocol, block)); - fs.writeFileSync(output, `${JSON.stringify(result, null, 2)}\n`); - console.table(result.checks); - if (!result.passed) process.exitCode = 1; + if (process.argv[2] === '--checkout-image') { + const [, protocolPath, identityPath, sourceCommit, version, output] = process.argv.slice(2); + assert.equal(process.argv.length, 8, 'Explicit protocol, identity, commit, version and new output required'); + const current = checkoutProtocol( + JSON.parse(fs.readFileSync(protocolPath, 'utf8')), + JSON.parse(fs.readFileSync(identityPath, 'utf8')), sourceCommit, version, + ); + fs.writeFileSync(output, `${JSON.stringify(current, null, 2)}\n`, { flag: 'wx' }); + } else { + const [protocolPath, blockName, rowsPath, output] = process.argv.slice(2); + assert.ok(protocolPath && blockName && rowsPath && output, 'Explicit protocol, block, receipts and output required'); + const protocol = JSON.parse(fs.readFileSync(protocolPath, 'utf8')); + const block = protocol.blocks.find(item => item.name === blockName); + assert.ok(block, 'Unknown preregistered block'); + const result = assessBlock(JSON.parse(fs.readFileSync(rowsPath, 'utf8')), block, rolesFor(protocol, block)); + fs.writeFileSync(output, `${JSON.stringify(result, null, 2)}\n`); + console.table(result.checks); + if (!result.passed) process.exitCode = 1; + } } catch (error) { console.error(error.message); process.exitCode = 1; diff --git a/scripts/performance/run-membership-qualification.ps1 b/scripts/performance/run-membership-qualification.ps1 index 6165e95a..a51a9618 100644 --- a/scripts/performance/run-membership-qualification.ps1 +++ b/scripts/performance/run-membership-qualification.ps1 @@ -1,12 +1,14 @@ param( [string]$OutputDirectory, + [string]$ProtocolPath, [switch]$TraceDiagnostic, [switch]$ProfileDiagnostic ) $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $false $root = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) -$protocolPath = Join-Path $PSScriptRoot 'membership-protocol.json' +if (!$ProtocolPath) { $ProtocolPath = Join-Path $PSScriptRoot 'membership-protocol.json' } +$ProtocolPath = (Resolve-Path -LiteralPath $ProtocolPath).Path $evaluator = Join-Path $PSScriptRoot 'assess-membership.cjs' $clientScript = Join-Path $PSScriptRoot 'benchmark-membership.cjs' $protocol = Get-Content -LiteralPath $protocolPath -Raw | ConvertFrom-Json @@ -37,7 +39,8 @@ $summary = [ordered]@{ AdmissionEligible = $false } $campaignClock = [System.Diagnostics.Stopwatch]::StartNew() -$workMinutes = if ($TraceDiagnostic -or $ProfileDiagnostic) { 15 } else { 60 } +$workMinutes = if ($TraceDiagnostic -or $ProfileDiagnostic) { 15 } + elseif ($protocol.candidate.sourceCommit) { 45 } else { 60 } $workDeadlineUtc = [DateTimeOffset]::UtcNow.AddMinutes($workMinutes).ToUnixTimeMilliseconds() $summary['WorkDeadlineUtc'] = [DateTimeOffset]::FromUnixTimeMilliseconds($workDeadlineUtc).ToString('o') $envNames = @('POSTGRES_PASSWORD', 'SCIM_SHARED_SECRET', 'JWT_SECRET', 'OAUTH_CLIENT_SECRET', @@ -47,7 +50,8 @@ $priorEnvironment = @{} foreach ($key in $envNames) { $priorEnvironment[$key] = [Environment]::GetEnvironmentVariable($key) } function Assert-WithinDeadline { - $limit = if ($TraceDiagnostic -or $ProfileDiagnostic) { 15 } else { 60 } + $limit = if ($TraceDiagnostic -or $ProfileDiagnostic) { 15 } + elseif ($protocol.candidate.sourceCommit) { 45 } else { 60 } if ($campaignClock.Elapsed.TotalMinutes -ge $limit) { throw 'Campaign work deadline reached; cleanup and upload time is reserved, no admission.' } @@ -59,6 +63,21 @@ function Invoke-Docker { return $value } +function Assert-CheckoutImage { + if (!$protocol.candidate.sourceCommit) { return } + $checkout = (& git -C $root rev-parse HEAD).Trim() + if ($LASTEXITCODE -ne 0 -or $checkout -cne $protocol.candidate.sourceCommit) { + throw 'Candidate source differs from the exact checkout.' + } + $package = Get-Content -LiteralPath (Join-Path $root 'api\package.json') -Raw | ConvertFrom-Json + if ($package.version -cne $protocol.candidate.version) { + throw 'Candidate version differs from the checkout package.' + } + $revision = (Invoke-Docker image inspect --format ` + '{{index .Config.Labels "org.opencontainers.image.revision"}}' $protocol.candidate.image).Trim() + if ($revision -cne $checkout) { throw 'Candidate image revision differs from the checkout.' } +} + function Assert-Owned { param([string]$Id, [switch]$Network) $labels = if ($Network) { @@ -514,9 +533,15 @@ try { if ($LASTEXITCODE -ne 0) { throw 'Host CPU identity unavailable.' } $null = Invoke-Docker info --format '{{json .}}' | Set-Content (Join-Path $OutputDirectory 'docker-info.json') + Assert-CheckoutImage foreach ($image in @($protocol.baseline.image, $protocol.candidate.image, $protocol.postgres)) { Assert-WithinDeadline - $null = Invoke-Docker pull $image + if ($image -eq $protocol.candidate.image -and $protocol.candidate.sourceCommit) { + $actualId = (Invoke-Docker image inspect --format '{{.Id}}' $image).Trim() + if ($actualId -cne $image) { throw 'Current candidate immutable image ID mismatch.' } + } else { + $null = Invoke-Docker pull $image + } } $identities = foreach ($role in @('baseline', 'candidate')) { $image = $protocol.$role.image diff --git a/scripts/test/membership-performance.test.cjs b/scripts/test/membership-performance.test.cjs index 13b892a3..b0b1fb7e 100644 --- a/scripts/test/membership-performance.test.cjs +++ b/scripts/test/membership-performance.test.cjs @@ -376,6 +376,106 @@ test('preregistered protocol rejects budget and schedule changes', () => { assert.throws(() => load().validateProtocol({ ...protocol, blocks: [...protocol.blocks].reverse() })); }); +test('checkout qualification binds the actual image to the exact source without changing the frozen protocol', () => { + const protocol = require('../performance/membership-protocol.json'); + const original = JSON.stringify(protocol); + const source = 'a'.repeat(40); + const identity = { id: `sha256:${'b'.repeat(64)}`, os: 'linux', architecture: 'amd64', sourceCommit: source }; + const current = load().checkoutProtocol(protocol, identity, source, '0.55.67'); + assert.equal(JSON.stringify(protocol), original, 'Historical protocol must not be modified'); + assert.deepEqual(current, { + ...protocol, + candidate: { image: identity.id, version: '0.55.67', sourceCommit: source }, + }); + load().validateProtocol(current); + assert.equal(load().rolesFor(current, current.blocks[0]).baseline.image, identity.id); + assert.equal(load().rolesFor(current, current.blocks[2]).baseline.image, protocol.baseline.image); + for (const [field, value] of [ + ['id', 'candidate:latest'], ['id', 'sha256:abc'], ['os', 'windows'], + ['architecture', 'arm64'], ['sourceCommit', 'c'.repeat(40)], + ]) { + assert.throws(() => load().checkoutProtocol(protocol, { ...identity, [field]: value }, source, '0.55.67')); + } + assert.throws(() => load().checkoutProtocol(protocol, identity, 'short', '0.55.67')); + assert.throws(() => load().checkoutProtocol(protocol, identity, source, 'latest')); + const unbound = structuredClone(current); + delete unbound.candidate.sourceCommit; + assert.throws(() => load().validateProtocol(unbound), 'Local images require explicit source identity'); + assert.throws(() => load().validateProtocol({ + ...current, baseline: current.candidate, + }), 'The published baseline must not become an arbitrary local image'); +}); + +test('checkout protocol CLI retains a new source-bound receipt and refuses to overwrite it', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'scim-checkout-protocol-')); + try { + const source = 'a'.repeat(40); + const identity = path.join(directory, 'identity.json'); + const output = path.join(directory, 'protocol.json'); + fs.writeFileSync(identity, JSON.stringify({ + id: `sha256:${'b'.repeat(64)}`, os: 'linux', architecture: 'amd64', sourceCommit: source, + })); + const args = [implementation, '--checkout-image', + path.resolve(__dirname, '../performance/membership-protocol.json'), identity, source, '0.55.67', output]; + const result = spawnSync(process.execPath, args, { encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + const receipt = fs.readFileSync(output, 'utf8'); + assert.equal(JSON.parse(receipt).candidate.sourceCommit, source); + const repeated = spawnSync(process.execPath, args, { encoding: 'utf8' }); + assert.notEqual(repeated.status, 0, 'Existing protocol evidence must not be overwritten'); + assert.equal(fs.readFileSync(output, 'utf8'), receipt); + } finally { fs.rmSync(directory, { recursive: true, force: true }); } +}); + +for (const scenario of ['success', 'wrong-source', 'git-failure', 'wrong-version', 'wrong-label']) { + test(`actual checkout preflight rejects unrelated runtime before allocation: ${scenario}`, () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'scim-checkout-guard-')); + try { + fs.mkdirSync(path.join(directory, 'api')); + fs.writeFileSync(path.join(directory, 'api', 'package.json'), + JSON.stringify({ version: scenario === 'wrong-version' ? '0.55.58' : '0.55.67' })); + const runner = path.resolve(__dirname, '../performance/run-membership-qualification.ps1'); + const result = spawnSync('pwsh', ['-NoProfile', '-Command', ` + $ErrorActionPreference = 'Stop' + $root = '${directory.replaceAll("'", "''")}' + $scenario = '${scenario}' + $protocol = @{ candidate = @{ sourceCommit = ('a' * 40); version = '0.55.67'; image = ('sha256:' + ('b' * 64)) } } + $script:inspected = $false + function git { + if (($args -join ' ') -cne "-C $root rev-parse HEAD") { throw 'Unexpected source query' } + $global:LASTEXITCODE = if ($scenario -eq 'git-failure') { 1 } else { 0 } + if ($scenario -eq 'wrong-source') { return ('c' * 40) } + return ('a' * 40) + } + function Invoke-Docker { + if ($args[0] -cne 'image' -or $args[1] -cne 'inspect' -or + $args[-1] -cne $protocol.candidate.image) { throw 'Unexpected image operation' } + $script:inspected = $true + if ($scenario -eq 'wrong-label') { return ('c' * 40) } + return ('a' * 40) + } + $ast = [System.Management.Automation.Language.Parser]::ParseFile( + '${runner.replaceAll("'", "''")}', [ref]$null, [ref]$null) + $guard = $ast.Find({ param($item) + $item -is [System.Management.Automation.Language.FunctionDefinitionAst] -and + $item.Name -eq 'Assert-CheckoutImage' + }, $true) + if (!$guard) { throw 'Missing actual checkout preflight' } + . ([scriptblock]::Create($guard.Extent.Text)) + $failure = $null + try { Assert-CheckoutImage } catch { $failure = $_.Exception.Message } + if ($scenario -eq 'success' -and ($failure -or !$script:inspected)) { throw "Valid checkout rejected: $failure" } + if ($scenario -ne 'success' -and !$failure) { throw 'Unrelated checkout was accepted' } + if ($scenario -in @('wrong-source', 'git-failure', 'wrong-version') -and $script:inspected) { + throw 'Unrelated source reached Docker' + } + if ($failure -and $failure -notlike 'Candidate * differs from *') { throw $failure } + `], { encoding: 'utf8' }); + assert.equal(result.status, 0, result.error?.message ?? result.stdout + result.stderr); + } finally { fs.rmSync(directory, { recursive: true, force: true }); } + }); +} + for (const scenario of ['success', 'corrupt-read', 'cleanup-failure', 'deadline-expired']) { for (const traced of ['success', 'corrupt-read'].includes(scenario) ? [false, true] : [false]) { test(`HTTP client retains explicit complete/partial evidence for ${scenario}${traced ? ' with diagnostic trace' : ''}`, async () => { @@ -535,7 +635,11 @@ test('internal work deadline and shorter step cap reserve cleanup/upload before const runner = path.resolve(__dirname, '..', 'performance', 'run-membership-qualification.ps1'); const workflow = fs.readFileSync(path.resolve(__dirname, '..', '..', '.github', 'workflows', 'performance-qualification.yml'), 'utf8'); assert.match(workflow, /^ timeout-minutes: 75$/m); - assert.match(workflow, /^ - name: Run fixed same-image controls before old\/current comparisons\r?\n timeout-minutes: 65$/m); + assert.match(workflow, /^ - name: Build and bind exact checkout image\r?\n timeout-minutes: 10$/m); + assert.match(workflow, /docker build --pull --label "org\.opencontainers\.image\.revision=\$source"/, + 'Current-source qualification must refresh the mutable base instead of silently using an older cached runtime'); + assert.match(workflow, /^ - name: Run fixed same-image controls before old\/current comparisons\r?\n timeout-minutes: 55$/m); + assert.match(workflow, /-ProtocolPath \(Join-Path \$env:GITHUB_WORKSPACE 'test-results\/performance-image\/protocol\.json'\)/); const result = spawnSync('pwsh', ['-NoProfile', '-Command', ` $ErrorActionPreference = 'Stop' $ast = [System.Management.Automation.Language.Parser]::ParseFile( @@ -545,11 +649,18 @@ test('internal work deadline and shorter step cap reserve cleanup/upload before }, $true) if (!$guard) { throw 'Missing actual internal deadline guard' } . ([scriptblock]::Create($guard.Extent.Text)) + $protocol = @{ candidate = @{} } $campaignClock = @{ Elapsed = [timespan]::FromMinutes(59) } Assert-WithinDeadline $campaignClock = @{ Elapsed = [timespan]::FromMinutes(60) } try { Assert-WithinDeadline; throw 'Expired work budget was accepted' } catch { if ($_.Exception.Message -notlike '*work deadline*') { throw } } + $protocol.candidate.sourceCommit = 'a' * 40 + $campaignClock = @{ Elapsed = [timespan]::FromMinutes(44) } + Assert-WithinDeadline + $campaignClock = @{ Elapsed = [timespan]::FromMinutes(45) } + try { Assert-WithinDeadline; throw 'Expired checkout work budget was accepted' } + catch { if ($_.Exception.Message -notlike '*work deadline*') { throw } } exit 0 `], { encoding: 'utf8' }); assert.equal(result.status, 0, result.error?.message ?? result.stdout + result.stderr); From ba99530c459d6b559fdd5a6693f5cde773a10aa4 Mon Sep 17 00:00:00 2001 From: "Prashant Rane (ALLYIS INC)" Date: Tue, 3 Nov 2026 03:25:16 -0600 Subject: [PATCH 2/2] docs: record qualification prerequisite recovery and historical rejection Record isolated dependency and Prisma generation prerequisites. Confirm the new assessor preserves the original control assessment exactly, including all three failures. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/ENDPOINT_SNAPSHOT_REVISION.md | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/ENDPOINT_SNAPSHOT_REVISION.md b/docs/ENDPOINT_SNAPSHOT_REVISION.md index 08f32bbe..006ad3a4 100644 --- a/docs/ENDPOINT_SNAPSHOT_REVISION.md +++ b/docs/ENDPOINT_SNAPSHOT_REVISION.md @@ -24,17 +24,24 @@ resource limits, operation order and latency bounds are not modified. | CLI argument count / Low | New flag plus five values was initially counted as nine process arguments rather than eight. The real CLI test failed, then passed after correction; it also rejects overwriting retained protocol evidence. | Focused actual CLI contract; no escape beyond its earliest gate. | | Build and cleanup budget / Medium | Adding a build inside the measurement step could consume reserved cleanup time. A separate ten-minute build step and 45/55-minute work/step caps preserve the 75-minute job budget. Actual deadline tests cover both historical and source-bound caps. | Focused workflow/deadline contract; corrected before workflow execution. | | Missing isolated-worktree pg dependency / Low | The broader Node tooling suite requires the API dependency set. Restore dependencies only in the new isolated worktree after MODULE_NOT_FOUND; no shared node_modules mutation or lock regeneration. | First complete tooling run; not a benchmark/application failure. | +| Missing isolated root/web tooling / Low | The first chosen web build and documentation check failed on missing vite/jsdom. Restore each worktree-owned dependency set using npm ci only after its observed missing-dependency failure. The protected hook owns consolidated validation. | Prerequisite build/documentation checks; no shared dependency mutation or lock regeneration. | +| Missing generated Prisma model types / Low | API dependencies were deliberately installed with scripts disabled, leaving the generated client absent. The API build exposed model-type errors; explicit Prisma generation succeeds with a codegen-only local URL and no database connection. The protected hook owns the subsequent full build/test verification. | First API build; normal code-generation prerequisite, not a production source defect. | | Stale mutable local base / Medium | The first local build reused Node 24.16.0 despite newer historical images. Require docker build --pull, retain the first image identity, and verify the refreshed runtime separately. | Actual built-image runtime readback; earliest possible gate is a fresh-base build plus runtime readback. | The initial source-binding test failed before implementation. The complete focused suite then passed 58/58, up from 51, with no skips. A separate real workflow assertion reproduced missing `--pull` as RED before that correction. -Complete local tooling passes227 cases with8 explicit retained-fixture/opt-in -skips and0 failures. The refreshed exact-source image reports0.55.67 on -Node24.22.2; its actual identity generates the separate immutable protocol. +Complete local tooling passes 227 cases with 8 explicit retained-fixture/opt-in +skips and 0 failures. The refreshed exact-source image reports 0.55.67 on +Node 24.22.2; its actual identity generates the separate immutable protocol. Manifest/lock bytes remain unchanged. No hosted qualification, final release or new performance PASS is claimed here. +Reassessing the retained October 29 control with the updated evaluator +produces an assessment structurally identical to the original: all 12 checks +match, including the same three failures. This verifies preserved rejection +of the historical evidence; it does not rerun or qualify the new image. + **Self-improvement applied:** release qualification must bind source, dependency-bearing image and actual runtime, not merely a historical version string. **Design disposition: accepted.** Extend the existing protocol