diff --git a/.github/workflows/performance-qualification.yml b/.github/workflows/performance-qualification.yml index a61f4c54..c354e7b6 100644 --- a/.github/workflows/performance-qualification.yml +++ b/.github/workflows/performance-qualification.yml @@ -32,11 +32,37 @@ jobs: timeout-minutes: 3 run: node --test scripts/test/membership-performance.test.cjs + - name: Build and bind exact checkout image + timeout-minutes: 10 + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $source = (git rev-parse HEAD).Trim() + if ($LASTEXITCODE -ne 0) { throw 'Exact checkout source unavailable.' } + git diff --exit-code + if ($LASTEXITCODE -ne 0) { throw 'Tracked checkout inputs were modified.' } + $directory = Join-Path $env:GITHUB_WORKSPACE 'test-results/performance-image' + New-Item -ItemType Directory -Path $directory -ErrorAction Stop | Out-Null + $version = (Get-Content api/package.json -Raw | ConvertFrom-Json).version + docker build --pull --label "org.opencontainers.image.revision=$source" ` + --build-arg "IMAGE_TAG=$source" --iidfile (Join-Path $directory 'image-id.txt') . + if ($LASTEXITCODE -ne 0) { throw 'Exact-checkout image build failed.' } + $image = (Get-Content (Join-Path $directory 'image-id.txt') -Raw).Trim() + docker image inspect --format ` + '{"id":{{json .Id}},"os":{{json .Os}},"architecture":{{json .Architecture}},"sourceCommit":{{json (index .Config.Labels "org.opencontainers.image.revision")}}}' ` + $image | Set-Content (Join-Path $directory 'identity.json') + if ($LASTEXITCODE -ne 0) { throw 'Exact-checkout image identity unavailable.' } + node scripts/performance/assess-membership.cjs --checkout-image ` + scripts/performance/membership-protocol.json (Join-Path $directory 'identity.json') ` + $source $version (Join-Path $directory 'protocol.json') + if ($LASTEXITCODE -ne 0) { throw 'Current-image protocol binding failed.' } + - name: Run fixed same-image controls before old/current comparisons - timeout-minutes: 65 + timeout-minutes: 55 shell: pwsh run: | ./scripts/performance/run-membership-qualification.ps1 ` + -ProtocolPath (Join-Path $env:GITHUB_WORKSPACE 'test-results/performance-image/protocol.json') ` -OutputDirectory (Join-Path $env:GITHUB_WORKSPACE 'test-results/performance-qualification') - name: Retain all completed and failed evidence @@ -45,6 +71,8 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: membership-qualification-${{ github.run_id }}-${{ github.run_attempt }} - path: test-results/performance-qualification/ + path: | + test-results/performance-qualification/ + test-results/performance-image/ if-no-files-found: error retention-days: 14 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f5f763f..3e1355ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### 0.55.67 exact-checkout qualification image (2026-11-03, proposed) + +- Build and bind the actual qualification checkout rather than testing only + the frozen 0.55.58 artifact. Preserve the historical manifest/results; + generate a separate immutable candidate protocol with exact source, + package version, Linux/x64 architecture and revision verification. +- Refresh mutable Docker bases, retain identity/protocol artifacts, reject + unrelated sources before allocation and reserve build/cleanup/upload time + within the existing 75-minute job cap. No latency/resource/sample limit, + serving source, dependency or lockfile change. +- Qualification contract coverage increases 51 to 58 passing cases, zero + skips in the focused run. Complete local tooling: 227 PASS / 8 explicit + retained-fixture or opt-in skips / 0 FAIL. The refreshed image reports + 0.55.67 on Node24.22.2 and generates a real source-bound protocol. + Hosted execution and release admission remain separate pending evidence. + Existing application test counts are not rerun or relabelled by this + tooling change. No deployment or performance PASS is claimed. + ### 0.55.67 Hono-master performance reconciliation (2026-10-30, proposed) - Normally integrate reviewed master `eb733f61` after Hono #262 merged at diff --git a/Session_starter.md b/Session_starter.md index 2a8a7d35..641f86ff 100644 --- a/Session_starter.md +++ b/Session_starter.md @@ -1,6 +1,23 @@ ## SCIMServer - Condensed Session Memory -**Last verified:** 2026-10-30 +**Last verified:** 2026-11-03 + +### Exact-checkout qualification implementation (2026-11-03) + +Isolated branch fix/qualification-image-binding-20261103 preserves the clean +abdef720 candidate and its dependencies. The qualification workflow now builds +the actual checkout, generates a separate immutable local-image protocol and +rejects wrong source/version/revision before benchmark allocation. Original +historical protocol, latency limits and failed receipts remain unchanged. +Focused contracts pass 58/58 after semantic RED. Broad tooling initially +required missing pg in this new worktree; isolated API npm ci restored it +without changing any manifest/lock. Full tooling now227 PASS /8 explicit +retained-fixture or opt-in skips /0 FAIL. First local .67 image revealed +cached Node24.16.0; --pull rebuild passes and reports0.55.67/Node24.22.2. +Actual image identity generates a source-bound protocol successfully. +No shared dependencies, release, target +deployment, paid host or admission waiver. Actual hosted qualification, +reviewed publication and complete dev/canary target gates remain required. ### Current Hono-master performance reconciliation (2026-10-30) diff --git a/docs/CONTEXT_INSTRUCTIONS.md b/docs/CONTEXT_INSTRUCTIONS.md index 218a6121..1b8d2680 100644 --- a/docs/CONTEXT_INSTRUCTIONS.md +++ b/docs/CONTEXT_INSTRUCTIONS.md @@ -9,6 +9,17 @@ ## Active Delivery Process +**2026-11-03 exact-checkout qualification:** isolated implementation binds the +actual .67 checkout image rather than treating frozen .58 performance as new +artifact admission. Historical protocol/results and all latency/resource/sample +bounds remain unchanged. Current source/version/revision and Linux/x64 +identity are verified before allocation; mutable bases are refreshed. +Focused contracts pass58/58; full local tooling227 PASS /8 explicit skips +/0 FAIL. Refreshed local image reports0.55.67/Node24.22.2 and a real +source-bound protocol is generated. Hosted execution and normal reviewed +publication remain independent. No performance PASS, +paid capacity or target deployment is inferred. + **2026-10-30 current performance reconciliation:** Hono #262 merged at `eb733f61e7f25d1d642fe746075e266c3b19ad60`, master 0.55.66. Proposed 0.55.67 normally integrates it into held performance `ff2d2377`, preserving both diff --git a/docs/DEPLOYMENT_INFRASTRUCTURE_AND_FORM_FACTORS.md b/docs/DEPLOYMENT_INFRASTRUCTURE_AND_FORM_FACTORS.md index 8dd6709f..3da3a6e1 100644 --- a/docs/DEPLOYMENT_INFRASTRUCTURE_AND_FORM_FACTORS.md +++ b/docs/DEPLOYMENT_INFRASTRUCTURE_AND_FORM_FACTORS.md @@ -7,20 +7,49 @@ --- +## Exact-checkout qualification image (2026-11-03 source change) + +The qualification workflow now builds its actual checkout, including current +dependency fixes, instead of measuring the frozen 0.55.58 image and implying +that the result qualifies 0.55.67. The original manifest and failed receipts +remain unchanged. A separate generated protocol binds the candidate's immutable +local image ID, exact checkout commit and package version. Image OS, +architecture and revision are verified; unrelated checkout/version/revision +values fail before benchmark resource allocation. The baseline remains the +original immutable public image. No package is published by this workflow. + +The Docker build uses `--pull`: the first local build otherwise reused a +cached Node 24.16.0 base, older than the recorded 0.55.20 image runtime. +Building the requested source alone does not establish a current base. +The refreshed base and actual final runtime must be measured independently. +The source-bound lane reserves ten minutes for the image build, uses a +45-minute cooperative measurement cap and a 55-minute measurement-step cap, +and preserves the 75-minute job cap and four-minute artifact upload. +Historical standalone protocol execution retains its original 60-minute +work cap. Resource limits, samples, warmups, all twelve latency bounds and +the four fail-stop blocks are unchanged. Generated identity/protocol evidence +is uploaded alongside completed or failed measurement evidence. + +This is source-level workflow verification, not new Azure estate evidence +or a passing performance result. Existing public GitHub-hosted Linux capacity +is used; no dedicated Azure host, new paid resource or deployment is introduced. + ## Prospective performance qualification runner (2026-10-22 source change) [performance-qualification.yml](../.github/workflows/performance-qualification.yml) uses the repository's existing public GitHub Actions service with an `ubuntu-24.04` runner. It provisions no Azure resource or new paid host, grants only `contents: read`, serializes qualification campaigns and permits no -deployment or package publication. A 60-minute cooperative work deadline, -65-minute qualification-step cap and 75-minute job cap reserve time for -cleanup/upload. Checkout, contracts and upload have 2/3/4-minute caps. +deployment or package publication. The current source-bound lane uses the +45/55/75-minute work/step/job limits described above; the original frozen +standalone campaign retains its 60-minute work limit. Checkout, contracts +and upload have 2/3/4-minute caps. Completed/failed evidence is retained for 14 days. The workflow starts on relevant pull requests or explicit dispatch, not every application change. -The [fixed manifest](../scripts/performance/membership-protocol.json) pins -actual application/PostgreSQL images and the original app resource limits. +The [fixed historical manifest](../scripts/performance/membership-protocol.json) +pins the original application/PostgreSQL images and resource limits. Current +checkout qualification generates a separate source-bound candidate protocol. Separate owned app/database processes are created per four-round block; each role's app/database is reused for its second visit, preserving the existing within-block history. Two A/A blocks must pass before the two A/B diff --git a/docs/ENDPOINT_SNAPSHOT_REVISION.md b/docs/ENDPOINT_SNAPSHOT_REVISION.md index 63e9285b..006ad3a4 100644 --- a/docs/ENDPOINT_SNAPSHOT_REVISION.md +++ b/docs/ENDPOINT_SNAPSHOT_REVISION.md @@ -5,6 +5,51 @@ **Status:** Implemented and tested locally, not released or deployed. Performance admission and the independent release gates remain open. +## Current-checkout image binding (2026-11-03, not released) + +The existing qualification still selected a pinned 0.55.58 candidate even +after source/dependency reconciliation proposed 0.55.67. A passing historical +image comparison could not qualify that new artifact. The new prospective +workflow builds its exact checkout and generates a separate candidate protocol +from the immutable local image ID, revision label and package version. +The frozen historical manifest and every failed result remain unchanged. +Current API/schema/browser source, dependencies, warmups, sample counts, +resource limits, operation order and latency bounds are not modified. + +### Execution issues and prevention + +| Issue | Root cause, resolution and prevention | Detection and escape | +|---|---|---| +| Stale candidate image / High | Source revision changed while the protocol stayed pinned to 0.55.58. Bind the real checkout build and reject wrong source/version/revision before allocation. New contracts exercise actual preflight and both same-image/old-current role selection. | Release investigation; earliest possible check was a checkout-to-image preflight, now added. | +| CLI argument count / Low | New flag plus five values was initially counted as nine process arguments rather than eight. The real CLI test failed, then passed after correction; it also rejects overwriting retained protocol evidence. | Focused actual CLI contract; no escape beyond its earliest gate. | +| Build and cleanup budget / Medium | Adding a build inside the measurement step could consume reserved cleanup time. A separate ten-minute build step and 45/55-minute work/step caps preserve the 75-minute job budget. Actual deadline tests cover both historical and source-bound caps. | Focused workflow/deadline contract; corrected before workflow execution. | +| Missing isolated-worktree pg dependency / Low | The broader Node tooling suite requires the API dependency set. Restore dependencies only in the new isolated worktree after MODULE_NOT_FOUND; no shared node_modules mutation or lock regeneration. | First complete tooling run; not a benchmark/application failure. | +| Missing isolated root/web tooling / Low | The first chosen web build and documentation check failed on missing vite/jsdom. Restore each worktree-owned dependency set using npm ci only after its observed missing-dependency failure. The protected hook owns consolidated validation. | Prerequisite build/documentation checks; no shared dependency mutation or lock regeneration. | +| Missing generated Prisma model types / Low | API dependencies were deliberately installed with scripts disabled, leaving the generated client absent. The API build exposed model-type errors; explicit Prisma generation succeeds with a codegen-only local URL and no database connection. The protected hook owns the subsequent full build/test verification. | First API build; normal code-generation prerequisite, not a production source defect. | +| Stale mutable local base / Medium | The first local build reused Node 24.16.0 despite newer historical images. Require docker build --pull, retain the first image identity, and verify the refreshed runtime separately. | Actual built-image runtime readback; earliest possible gate is a fresh-base build plus runtime readback. | + +The initial source-binding test failed before implementation. The complete +focused suite then passed 58/58, up from 51, with no skips. A separate real +workflow assertion reproduced missing `--pull` as RED before that correction. +Complete local tooling passes 227 cases with 8 explicit retained-fixture/opt-in +skips and 0 failures. The refreshed exact-source image reports 0.55.67 on +Node 24.22.2; its actual identity generates the separate immutable protocol. +Manifest/lock bytes remain unchanged. No hosted qualification, final release +or new performance PASS is claimed here. + +Reassessing the retained October 29 control with the updated evaluator +produces an assessment structurally identical to the original: all 12 checks +match, including the same three failures. This verifies preserved rejection +of the historical evidence; it does not rerun or qualify the new image. + +**Self-improvement applied:** release qualification must bind source, +dependency-bearing image and actual runtime, not merely a historical version +string. **Design disposition: accepted.** Extend the existing protocol +validator and runner with one small pure binding helper; no serving service, +new framework, new paid host or speculative measurement strategy is added. +Provenance is this continuation's exact-source build and focused RED/GREEN +output, not a reconstruction of earlier full transcripts. + ## Current reviewed-source union (2026-10-29) The preserved 0.55.65 performance candidate now normally integrates reviewed diff --git a/scripts/performance/assess-membership.cjs b/scripts/performance/assess-membership.cjs index bb41dd4c..77589788 100644 --- a/scripts/performance/assess-membership.cjs +++ b/scripts/performance/assess-membership.cjs @@ -21,12 +21,31 @@ function validateProtocol(protocol) { { name: 'ab-abba', sameImage: false, order: 'ABBA' }, ]); for (const role of ['baseline', 'candidate']) { - assert.match(protocol[role].image, /^ghcr\.io\/pranems\/scimserver@sha256:[a-f0-9]{64}$/); + if (role === 'candidate' && protocol.candidate.sourceCommit !== undefined) { + assert.match(protocol.candidate.sourceCommit, /^[a-f0-9]{40}$/); + assert.match(protocol.candidate.image, /^sha256:[a-f0-9]{64}$/); + } else { + assert.match(protocol[role].image, /^ghcr\.io\/pranems\/scimserver@sha256:[a-f0-9]{64}$/); + } assert.match(protocol[role].version, /^\d+\.\d+\.\d+$/); } assert.match(protocol.postgres, /^postgres@sha256:[a-f0-9]{64}$/); } +function checkoutProtocol(protocol, identity, sourceCommit, version) { + validateProtocol(protocol); + assert.match(sourceCommit, /^[a-f0-9]{40}$/, 'Exact checkout commit required'); + assert.match(version, /^\d+\.\d+\.\d+$/, 'Exact package version required'); + assert.match(identity.id, /^sha256:[a-f0-9]{64}$/, 'Immutable local image ID required'); + assert.equal(identity.os, 'linux', 'Linux image required'); + assert.equal(identity.architecture, 'amd64', 'Original x64 measurement architecture required'); + assert.equal(identity.sourceCommit, sourceCommit, 'Image revision must match the checkout'); + const current = structuredClone(protocol); + current.candidate = { image: identity.id, version, sourceCommit }; + validateProtocol(current); + return current; +} + function rolesFor(protocol, block) { validateProtocol(protocol); assert.deepEqual(protocol.blocks.find(item => item.name === block.name), block); @@ -72,19 +91,29 @@ function assessBlock(rows, block, roles) { return { block: block.name, sameImage: block.sameImage, passed: checks.every(check => check.passed), checks }; } -module.exports = { assessBlock, rolesFor, validateProtocol }; +module.exports = { assessBlock, checkoutProtocol, rolesFor, validateProtocol }; if (require.main === module) { try { - const [protocolPath, blockName, rowsPath, output] = process.argv.slice(2); - assert.ok(protocolPath && blockName && rowsPath && output, 'Explicit protocol, block, receipts and output required'); - const protocol = JSON.parse(fs.readFileSync(protocolPath, 'utf8')); - const block = protocol.blocks.find(item => item.name === blockName); - assert.ok(block, 'Unknown preregistered block'); - const result = assessBlock(JSON.parse(fs.readFileSync(rowsPath, 'utf8')), block, rolesFor(protocol, block)); - fs.writeFileSync(output, `${JSON.stringify(result, null, 2)}\n`); - console.table(result.checks); - if (!result.passed) process.exitCode = 1; + if (process.argv[2] === '--checkout-image') { + const [, protocolPath, identityPath, sourceCommit, version, output] = process.argv.slice(2); + assert.equal(process.argv.length, 8, 'Explicit protocol, identity, commit, version and new output required'); + const current = checkoutProtocol( + JSON.parse(fs.readFileSync(protocolPath, 'utf8')), + JSON.parse(fs.readFileSync(identityPath, 'utf8')), sourceCommit, version, + ); + fs.writeFileSync(output, `${JSON.stringify(current, null, 2)}\n`, { flag: 'wx' }); + } else { + const [protocolPath, blockName, rowsPath, output] = process.argv.slice(2); + assert.ok(protocolPath && blockName && rowsPath && output, 'Explicit protocol, block, receipts and output required'); + const protocol = JSON.parse(fs.readFileSync(protocolPath, 'utf8')); + const block = protocol.blocks.find(item => item.name === blockName); + assert.ok(block, 'Unknown preregistered block'); + const result = assessBlock(JSON.parse(fs.readFileSync(rowsPath, 'utf8')), block, rolesFor(protocol, block)); + fs.writeFileSync(output, `${JSON.stringify(result, null, 2)}\n`); + console.table(result.checks); + if (!result.passed) process.exitCode = 1; + } } catch (error) { console.error(error.message); process.exitCode = 1; diff --git a/scripts/performance/run-membership-qualification.ps1 b/scripts/performance/run-membership-qualification.ps1 index 6165e95a..a51a9618 100644 --- a/scripts/performance/run-membership-qualification.ps1 +++ b/scripts/performance/run-membership-qualification.ps1 @@ -1,12 +1,14 @@ param( [string]$OutputDirectory, + [string]$ProtocolPath, [switch]$TraceDiagnostic, [switch]$ProfileDiagnostic ) $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $false $root = Split-Path -Parent (Split-Path -Parent $PSScriptRoot) -$protocolPath = Join-Path $PSScriptRoot 'membership-protocol.json' +if (!$ProtocolPath) { $ProtocolPath = Join-Path $PSScriptRoot 'membership-protocol.json' } +$ProtocolPath = (Resolve-Path -LiteralPath $ProtocolPath).Path $evaluator = Join-Path $PSScriptRoot 'assess-membership.cjs' $clientScript = Join-Path $PSScriptRoot 'benchmark-membership.cjs' $protocol = Get-Content -LiteralPath $protocolPath -Raw | ConvertFrom-Json @@ -37,7 +39,8 @@ $summary = [ordered]@{ AdmissionEligible = $false } $campaignClock = [System.Diagnostics.Stopwatch]::StartNew() -$workMinutes = if ($TraceDiagnostic -or $ProfileDiagnostic) { 15 } else { 60 } +$workMinutes = if ($TraceDiagnostic -or $ProfileDiagnostic) { 15 } + elseif ($protocol.candidate.sourceCommit) { 45 } else { 60 } $workDeadlineUtc = [DateTimeOffset]::UtcNow.AddMinutes($workMinutes).ToUnixTimeMilliseconds() $summary['WorkDeadlineUtc'] = [DateTimeOffset]::FromUnixTimeMilliseconds($workDeadlineUtc).ToString('o') $envNames = @('POSTGRES_PASSWORD', 'SCIM_SHARED_SECRET', 'JWT_SECRET', 'OAUTH_CLIENT_SECRET', @@ -47,7 +50,8 @@ $priorEnvironment = @{} foreach ($key in $envNames) { $priorEnvironment[$key] = [Environment]::GetEnvironmentVariable($key) } function Assert-WithinDeadline { - $limit = if ($TraceDiagnostic -or $ProfileDiagnostic) { 15 } else { 60 } + $limit = if ($TraceDiagnostic -or $ProfileDiagnostic) { 15 } + elseif ($protocol.candidate.sourceCommit) { 45 } else { 60 } if ($campaignClock.Elapsed.TotalMinutes -ge $limit) { throw 'Campaign work deadline reached; cleanup and upload time is reserved, no admission.' } @@ -59,6 +63,21 @@ function Invoke-Docker { return $value } +function Assert-CheckoutImage { + if (!$protocol.candidate.sourceCommit) { return } + $checkout = (& git -C $root rev-parse HEAD).Trim() + if ($LASTEXITCODE -ne 0 -or $checkout -cne $protocol.candidate.sourceCommit) { + throw 'Candidate source differs from the exact checkout.' + } + $package = Get-Content -LiteralPath (Join-Path $root 'api\package.json') -Raw | ConvertFrom-Json + if ($package.version -cne $protocol.candidate.version) { + throw 'Candidate version differs from the checkout package.' + } + $revision = (Invoke-Docker image inspect --format ` + '{{index .Config.Labels "org.opencontainers.image.revision"}}' $protocol.candidate.image).Trim() + if ($revision -cne $checkout) { throw 'Candidate image revision differs from the checkout.' } +} + function Assert-Owned { param([string]$Id, [switch]$Network) $labels = if ($Network) { @@ -514,9 +533,15 @@ try { if ($LASTEXITCODE -ne 0) { throw 'Host CPU identity unavailable.' } $null = Invoke-Docker info --format '{{json .}}' | Set-Content (Join-Path $OutputDirectory 'docker-info.json') + Assert-CheckoutImage foreach ($image in @($protocol.baseline.image, $protocol.candidate.image, $protocol.postgres)) { Assert-WithinDeadline - $null = Invoke-Docker pull $image + if ($image -eq $protocol.candidate.image -and $protocol.candidate.sourceCommit) { + $actualId = (Invoke-Docker image inspect --format '{{.Id}}' $image).Trim() + if ($actualId -cne $image) { throw 'Current candidate immutable image ID mismatch.' } + } else { + $null = Invoke-Docker pull $image + } } $identities = foreach ($role in @('baseline', 'candidate')) { $image = $protocol.$role.image diff --git a/scripts/test/membership-performance.test.cjs b/scripts/test/membership-performance.test.cjs index 13b892a3..b0b1fb7e 100644 --- a/scripts/test/membership-performance.test.cjs +++ b/scripts/test/membership-performance.test.cjs @@ -376,6 +376,106 @@ test('preregistered protocol rejects budget and schedule changes', () => { assert.throws(() => load().validateProtocol({ ...protocol, blocks: [...protocol.blocks].reverse() })); }); +test('checkout qualification binds the actual image to the exact source without changing the frozen protocol', () => { + const protocol = require('../performance/membership-protocol.json'); + const original = JSON.stringify(protocol); + const source = 'a'.repeat(40); + const identity = { id: `sha256:${'b'.repeat(64)}`, os: 'linux', architecture: 'amd64', sourceCommit: source }; + const current = load().checkoutProtocol(protocol, identity, source, '0.55.67'); + assert.equal(JSON.stringify(protocol), original, 'Historical protocol must not be modified'); + assert.deepEqual(current, { + ...protocol, + candidate: { image: identity.id, version: '0.55.67', sourceCommit: source }, + }); + load().validateProtocol(current); + assert.equal(load().rolesFor(current, current.blocks[0]).baseline.image, identity.id); + assert.equal(load().rolesFor(current, current.blocks[2]).baseline.image, protocol.baseline.image); + for (const [field, value] of [ + ['id', 'candidate:latest'], ['id', 'sha256:abc'], ['os', 'windows'], + ['architecture', 'arm64'], ['sourceCommit', 'c'.repeat(40)], + ]) { + assert.throws(() => load().checkoutProtocol(protocol, { ...identity, [field]: value }, source, '0.55.67')); + } + assert.throws(() => load().checkoutProtocol(protocol, identity, 'short', '0.55.67')); + assert.throws(() => load().checkoutProtocol(protocol, identity, source, 'latest')); + const unbound = structuredClone(current); + delete unbound.candidate.sourceCommit; + assert.throws(() => load().validateProtocol(unbound), 'Local images require explicit source identity'); + assert.throws(() => load().validateProtocol({ + ...current, baseline: current.candidate, + }), 'The published baseline must not become an arbitrary local image'); +}); + +test('checkout protocol CLI retains a new source-bound receipt and refuses to overwrite it', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'scim-checkout-protocol-')); + try { + const source = 'a'.repeat(40); + const identity = path.join(directory, 'identity.json'); + const output = path.join(directory, 'protocol.json'); + fs.writeFileSync(identity, JSON.stringify({ + id: `sha256:${'b'.repeat(64)}`, os: 'linux', architecture: 'amd64', sourceCommit: source, + })); + const args = [implementation, '--checkout-image', + path.resolve(__dirname, '../performance/membership-protocol.json'), identity, source, '0.55.67', output]; + const result = spawnSync(process.execPath, args, { encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + const receipt = fs.readFileSync(output, 'utf8'); + assert.equal(JSON.parse(receipt).candidate.sourceCommit, source); + const repeated = spawnSync(process.execPath, args, { encoding: 'utf8' }); + assert.notEqual(repeated.status, 0, 'Existing protocol evidence must not be overwritten'); + assert.equal(fs.readFileSync(output, 'utf8'), receipt); + } finally { fs.rmSync(directory, { recursive: true, force: true }); } +}); + +for (const scenario of ['success', 'wrong-source', 'git-failure', 'wrong-version', 'wrong-label']) { + test(`actual checkout preflight rejects unrelated runtime before allocation: ${scenario}`, () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'scim-checkout-guard-')); + try { + fs.mkdirSync(path.join(directory, 'api')); + fs.writeFileSync(path.join(directory, 'api', 'package.json'), + JSON.stringify({ version: scenario === 'wrong-version' ? '0.55.58' : '0.55.67' })); + const runner = path.resolve(__dirname, '../performance/run-membership-qualification.ps1'); + const result = spawnSync('pwsh', ['-NoProfile', '-Command', ` + $ErrorActionPreference = 'Stop' + $root = '${directory.replaceAll("'", "''")}' + $scenario = '${scenario}' + $protocol = @{ candidate = @{ sourceCommit = ('a' * 40); version = '0.55.67'; image = ('sha256:' + ('b' * 64)) } } + $script:inspected = $false + function git { + if (($args -join ' ') -cne "-C $root rev-parse HEAD") { throw 'Unexpected source query' } + $global:LASTEXITCODE = if ($scenario -eq 'git-failure') { 1 } else { 0 } + if ($scenario -eq 'wrong-source') { return ('c' * 40) } + return ('a' * 40) + } + function Invoke-Docker { + if ($args[0] -cne 'image' -or $args[1] -cne 'inspect' -or + $args[-1] -cne $protocol.candidate.image) { throw 'Unexpected image operation' } + $script:inspected = $true + if ($scenario -eq 'wrong-label') { return ('c' * 40) } + return ('a' * 40) + } + $ast = [System.Management.Automation.Language.Parser]::ParseFile( + '${runner.replaceAll("'", "''")}', [ref]$null, [ref]$null) + $guard = $ast.Find({ param($item) + $item -is [System.Management.Automation.Language.FunctionDefinitionAst] -and + $item.Name -eq 'Assert-CheckoutImage' + }, $true) + if (!$guard) { throw 'Missing actual checkout preflight' } + . ([scriptblock]::Create($guard.Extent.Text)) + $failure = $null + try { Assert-CheckoutImage } catch { $failure = $_.Exception.Message } + if ($scenario -eq 'success' -and ($failure -or !$script:inspected)) { throw "Valid checkout rejected: $failure" } + if ($scenario -ne 'success' -and !$failure) { throw 'Unrelated checkout was accepted' } + if ($scenario -in @('wrong-source', 'git-failure', 'wrong-version') -and $script:inspected) { + throw 'Unrelated source reached Docker' + } + if ($failure -and $failure -notlike 'Candidate * differs from *') { throw $failure } + `], { encoding: 'utf8' }); + assert.equal(result.status, 0, result.error?.message ?? result.stdout + result.stderr); + } finally { fs.rmSync(directory, { recursive: true, force: true }); } + }); +} + for (const scenario of ['success', 'corrupt-read', 'cleanup-failure', 'deadline-expired']) { for (const traced of ['success', 'corrupt-read'].includes(scenario) ? [false, true] : [false]) { test(`HTTP client retains explicit complete/partial evidence for ${scenario}${traced ? ' with diagnostic trace' : ''}`, async () => { @@ -535,7 +635,11 @@ test('internal work deadline and shorter step cap reserve cleanup/upload before const runner = path.resolve(__dirname, '..', 'performance', 'run-membership-qualification.ps1'); const workflow = fs.readFileSync(path.resolve(__dirname, '..', '..', '.github', 'workflows', 'performance-qualification.yml'), 'utf8'); assert.match(workflow, /^ timeout-minutes: 75$/m); - assert.match(workflow, /^ - name: Run fixed same-image controls before old\/current comparisons\r?\n timeout-minutes: 65$/m); + assert.match(workflow, /^ - name: Build and bind exact checkout image\r?\n timeout-minutes: 10$/m); + assert.match(workflow, /docker build --pull --label "org\.opencontainers\.image\.revision=\$source"/, + 'Current-source qualification must refresh the mutable base instead of silently using an older cached runtime'); + assert.match(workflow, /^ - name: Run fixed same-image controls before old\/current comparisons\r?\n timeout-minutes: 55$/m); + assert.match(workflow, /-ProtocolPath \(Join-Path \$env:GITHUB_WORKSPACE 'test-results\/performance-image\/protocol\.json'\)/); const result = spawnSync('pwsh', ['-NoProfile', '-Command', ` $ErrorActionPreference = 'Stop' $ast = [System.Management.Automation.Language.Parser]::ParseFile( @@ -545,11 +649,18 @@ test('internal work deadline and shorter step cap reserve cleanup/upload before }, $true) if (!$guard) { throw 'Missing actual internal deadline guard' } . ([scriptblock]::Create($guard.Extent.Text)) + $protocol = @{ candidate = @{} } $campaignClock = @{ Elapsed = [timespan]::FromMinutes(59) } Assert-WithinDeadline $campaignClock = @{ Elapsed = [timespan]::FromMinutes(60) } try { Assert-WithinDeadline; throw 'Expired work budget was accepted' } catch { if ($_.Exception.Message -notlike '*work deadline*') { throw } } + $protocol.candidate.sourceCommit = 'a' * 40 + $campaignClock = @{ Elapsed = [timespan]::FromMinutes(44) } + Assert-WithinDeadline + $campaignClock = @{ Elapsed = [timespan]::FromMinutes(45) } + try { Assert-WithinDeadline; throw 'Expired checkout work budget was accepted' } + catch { if ($_.Exception.Message -notlike '*work deadline*') { throw } } exit 0 `], { encoding: 'utf8' }); assert.equal(result.status, 0, result.error?.message ?? result.stdout + result.stderr);