diff --git a/experiments/g01-scaleset/livecanary/ack_command_fixture_test.go b/experiments/g01-scaleset/livecanary/ack_command_fixture_test.go index 9511aefd..45d82db3 100644 --- a/experiments/g01-scaleset/livecanary/ack_command_fixture_test.go +++ b/experiments/g01-scaleset/livecanary/ack_command_fixture_test.go @@ -301,7 +301,7 @@ func (f *ackCommandFixture) brokerWire(r *http.Request) { v3 := f.base.a.GroupPolicy.Version == 2 primary := []string{"GET " + workflowRunFixturePath(f.transport), "GET /app", "GET /orgs/fixture-org/installation", "GET " + workflowRunFixturePath(f.transport), "POST /app/installations/12/access_tokens", "GET /installation/repositories", "GET /repos/fixture-org/canary", "GET /orgs/fixture-org/actions/runner-groups/3", "GET /orgs/fixture-org/actions/runner-groups/3/repositories"} if v3 { - primary = []string{"GET " + workflowRunFixturePath(f.transport), "GET /app", "GET /orgs/fixture-org/installation", "POST /app/installations/12/access_tokens", "GET /installation/repositories", "GET /repos/fixture-org/canary", "GET /orgs/fixture-org/actions/runner-groups/3", "GET /orgs/fixture-org/actions/runner-groups/3/repositories", "DELETE /installation/token", "GET " + workflowRunFixturePath(f.transport), "POST /app/installations/12/access_tokens", "GET /installation/repositories", "GET /repos/fixture-org/canary", "GET /orgs/fixture-org/actions/runner-groups/3", "GET /orgs/fixture-org/actions/runner-groups/3/repositories", "GET " + workflowRunFixturePath(f.transport)} + primary = []string{"GET " + workflowRunFixturePath(f.transport), "GET /app", "GET /orgs/fixture-org/installation", "POST /app/installations/12/access_tokens", "GET /installation/repositories", "GET /repos/fixture-org/canary", "GET /orgs/fixture-org/actions/runner-groups/3", "GET /orgs/fixture-org/actions/runner-groups/3/repositories", "DELETE /installation/token", "GET " + workflowRunFixturePath(f.transport), "POST /app/installations/12/access_tokens", "GET /installation/repositories", "GET /repos/fixture-org/canary", "GET /orgs/fixture-org/actions/runner-groups/3", "GET /orgs/fixture-org/actions/runner-groups/3/repositories", "GET " + workflowRunFixturePath(f.transport), "DELETE /installation/token"} } position := len(f.brokerAttempts) - 1 // shared lock held; already counted before drain fresh := f.transport.attempt == f.attempt+"-fresh" @@ -359,6 +359,9 @@ func (f *ackCommandFixture) brokerWire(r *http.Request) { token = "synthetic-process-read-installation-token" } else if v3 && r.URL.Path == "/installation/token" { token = "synthetic-process-read-installation-token" + if position == len(primary)-1 { + token = "synthetic-process-installation-token" + } } if auth != "Bearer "+token { t.Error("ACK broker distinct observation/verification/installation authority differs") @@ -602,7 +605,7 @@ func (f *ackCommandFixture) assertBroker() { want := []string{"GET " + workflowRunFixturePath(f.transport), "GET /app", "GET /orgs/fixture-org/installation", "GET " + workflowRunFixturePath(f.transport), "POST /app/installations/12/access_tokens", "GET /installation/repositories", "GET /repos/fixture-org/canary", "GET /orgs/fixture-org/actions/runner-groups/3", "GET /orgs/fixture-org/actions/runner-groups/3/repositories"} v3 := f.base.a.GroupPolicy.Version == 2 if v3 { - want = []string{"GET " + workflowRunFixturePath(f.transport), "GET /app", "GET /orgs/fixture-org/installation", "POST /app/installations/12/access_tokens", "GET /installation/repositories", "GET /repos/fixture-org/canary", "GET /orgs/fixture-org/actions/runner-groups/3", "GET /orgs/fixture-org/actions/runner-groups/3/repositories", "DELETE /installation/token", "GET " + workflowRunFixturePath(f.transport), "POST /app/installations/12/access_tokens", "GET /installation/repositories", "GET /repos/fixture-org/canary", "GET /orgs/fixture-org/actions/runner-groups/3", "GET /orgs/fixture-org/actions/runner-groups/3/repositories", "GET " + workflowRunFixturePath(f.transport)} + want = []string{"GET " + workflowRunFixturePath(f.transport), "GET /app", "GET /orgs/fixture-org/installation", "POST /app/installations/12/access_tokens", "GET /installation/repositories", "GET /repos/fixture-org/canary", "GET /orgs/fixture-org/actions/runner-groups/3", "GET /orgs/fixture-org/actions/runner-groups/3/repositories", "DELETE /installation/token", "GET " + workflowRunFixturePath(f.transport), "POST /app/installations/12/access_tokens", "GET /installation/repositories", "GET /repos/fixture-org/canary", "GET /orgs/fixture-org/actions/runner-groups/3", "GET /orgs/fixture-org/actions/runner-groups/3/repositories", "GET " + workflowRunFixturePath(f.transport), "DELETE /installation/token"} } if !reflect.DeepEqual(f.brokerAttempts, f.transport.calls) { f.t.Error("ACK broker server-entry counts differ from dispatched calls") @@ -611,7 +614,7 @@ func (f *ackCommandFixture) assertBroker() { f.t.Errorf("ACK actual broker exact trace differs: calls=%v want=%v", f.transport.calls, want) } if v3 { - if f.transport.mints != 2 || f.transport.readOnlyMints != 1 || f.transport.managementMints != 1 || f.transport.revocations != 1 { + if f.transport.mints != 2 || f.transport.readOnlyMints != 1 || f.transport.managementMints != 1 || f.transport.revocations != 2 || f.transport.readOnlyRevocations != 1 || f.transport.managementRevocations != 1 { f.t.Errorf("ACK actual broker issuance lifecycle differs: mints=%d read-only=%d management=%d revocations=%d", f.transport.mints, f.transport.readOnlyMints, f.transport.managementMints, f.transport.revocations) } } else if f.transport.mints != 1 { @@ -670,11 +673,11 @@ func (f *ackCommandFixture) assertBroker() { f.t.Errorf("ACK workflow provenance lifecycle events=%d, want none or a start/confirmation pair", workflowEvents) } if v3 { - wantLedger += 2 // read-token lifecycle start and confirmation surround completion. + wantLedger += 4 // read and management lifecycle starts and confirmations surround completion. } completionIndex := claimIndex + 1 if v3 { - completionIndex += 2 + completionIndex += 4 } if len(ledger) != wantLedger || claimIndex >= len(ledger) { f.t.Errorf("ACK exact broker claim/completion count differs: events=%d want=%d claim-index=%d", len(ledger), wantLedger, claimIndex) @@ -707,6 +710,10 @@ func (f *ackCommandFixture) assertBroker() { if json.Unmarshal(ledger[readTokenStartIndex], &started) != nil || json.Unmarshal(ledger[readTokenStartIndex+1], &confirmed) != nil || started.Kind != "read_token_lifecycle_started" || confirmed.Kind != "read_token_lifecycle_confirmed" || started.Slot != claimIdentity.Slot || confirmed.Slot != claimIdentity.Slot || !bytes.Equal(started.Attempt, claimIdentity.Attempt) || !bytes.Equal(started.Journal, claimIdentity.Journal) || !bytes.Equal(confirmed.Attempt, claimIdentity.Attempt) || !bytes.Equal(confirmed.Journal, claimIdentity.Journal) { f.t.Error("ACK read-token lifecycle claim identity differs") } + managementTokenStartIndex := claimIndex + 3 + if json.Unmarshal(ledger[managementTokenStartIndex], &started) != nil || json.Unmarshal(ledger[managementTokenStartIndex+1], &confirmed) != nil || started.Kind != "management_token_lifecycle_started" || confirmed.Kind != "management_token_lifecycle_confirmed" || started.Slot != claimIdentity.Slot || confirmed.Slot != claimIdentity.Slot || !bytes.Equal(started.Attempt, claimIdentity.Attempt) || !bytes.Equal(started.Journal, claimIdentity.Journal) || !bytes.Equal(confirmed.Attempt, claimIdentity.Attempt) || !bytes.Equal(confirmed.Journal, claimIdentity.Journal) { + f.t.Error("ACK management-token lifecycle claim identity differs") + } } if wantComplete == 1 { var claim, completion map[string]any diff --git a/experiments/g01-scaleset/livecanary/acquire_loss_command_fixture_test.go b/experiments/g01-scaleset/livecanary/acquire_loss_command_fixture_test.go index f14788f1..eb4ac9a4 100644 --- a/experiments/g01-scaleset/livecanary/acquire_loss_command_fixture_test.go +++ b/experiments/g01-scaleset/livecanary/acquire_loss_command_fixture_test.go @@ -464,7 +464,11 @@ func newCommandQualificationFixture(t *testing.T, controller, broker, variant, p if os.WriteFile(f.brokerApprovalPath, raw, 0600) != nil { t.Fatal("broker approval write") } - f.transport = &processBrokerHTTP{t: t, attempt: f.attempt, approval: &f.base.a} + verificationAuthorization := "Bearer " + f.base.api.credentials.VerificationToken + f.transport = &processBrokerHTTP{ + t: t, attempt: f.attempt, approval: &f.base.a, + expectedWorkflowAuthorizations: []string{"Bearer synthetic-observation-credential", verificationAuthorization, verificationAuthorization}, + } server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { f.brokerMu.Lock() defer f.brokerMu.Unlock() @@ -791,8 +795,9 @@ func (f *acquireLossCommandFixture) guardControlsForPhase(events []Event, phase f.brokerMu.Lock() replayCalls, replayMints := len(f.transport.calls), f.transport.mints replayReadOnlyMints, replayManagementMints, replayRevocations := f.transport.readOnlyMints, f.transport.managementMints, f.transport.revocations + replayReadOnlyRevocations, replayManagementRevocations := f.transport.readOnlyRevocations, f.transport.managementRevocations f.brokerMu.Unlock() - if !bytes.Equal(brokerBefore, brokerAfter) || replayCalls != originalCalls || replayMints != 2 || replayReadOnlyMints != 1 || replayManagementMints != 1 || replayRevocations != 1 { + if !bytes.Equal(brokerBefore, brokerAfter) || replayCalls != originalCalls || replayMints != 2 || replayReadOnlyMints != 1 || replayManagementMints != 1 || replayRevocations != 2 || replayReadOnlyRevocations != 1 || replayManagementRevocations != 1 { t.Error("exact broker-attempt replay changed history or crossed API/mint boundary") } claimPath := filepath.Join(f.admission, "broker-admission.jsonl") @@ -845,6 +850,8 @@ func (f *acquireLossCommandFixture) guardControlsForPhase(events []Event, phase readOnlyMintsBefore := f.transport.readOnlyMints managementMintsBefore := f.transport.managementMints revocationsBefore := f.transport.revocations + readOnlyRevocationsBefore := f.transport.readOnlyRevocations + managementRevocationsBefore := f.transport.managementRevocations f.transport.attempt = fresh f.brokerMu.Unlock() f.run(f.broker, f.brokerArgs(fresh), inputs...) @@ -854,12 +861,14 @@ func (f *acquireLossCommandFixture) guardControlsForPhase(events []Event, phase readOnlyMintsAfter := f.transport.readOnlyMints managementMintsAfter := f.transport.managementMints revocationsAfter := f.transport.revocations + readOnlyRevocationsAfter := f.transport.readOnlyRevocations + managementRevocationsAfter := f.transport.managementRevocations f.brokerMu.Unlock() wantCalls := []string{"GET " + workflowRunFixturePath(f.transport)} if workflowGETAccounted { wantCalls = nil } - if !reflect.DeepEqual(calls, wantCalls) || mintsAfter != mintsBefore || readOnlyMintsAfter != readOnlyMintsBefore || managementMintsAfter != managementMintsBefore || revocationsAfter != revocationsBefore { + if !reflect.DeepEqual(calls, wantCalls) || mintsAfter != mintsBefore || readOnlyMintsAfter != readOnlyMintsBefore || managementMintsAfter != managementMintsBefore || revocationsAfter != revocationsBefore || readOnlyRevocationsAfter != readOnlyRevocationsBefore || managementRevocationsAfter != managementRevocationsBefore { t.Error("fresh occupied-phase broker attempt crossed the expected provider or token boundary") } snapshotPath := filepath.Join(fresh, "controller-approval.json") @@ -900,12 +909,14 @@ func TestExperimentalAcquireLossCommand(t *testing.T) { managementMints := f.transport.managementMints readOnlyMints := f.transport.readOnlyMints revocations := f.transport.revocations + readOnlyRevocations := f.transport.readOnlyRevocations + managementRevocations := f.transport.managementRevocations calls := append([]string(nil), f.transport.calls...) f.brokerMu.Unlock() if managementMints != 1 { t.Fatalf("actual broker management issuance did not mint exactly once: total=%d read-only=%d management=%d revocations=%d calls=%v", mints, readOnlyMints, managementMints, revocations, calls) } - if mints != 2 || readOnlyMints != 1 || revocations != 1 { + if mints != 2 || readOnlyMints != 1 || revocations != 2 || readOnlyRevocations != 1 || managementRevocations != 1 { t.Fatalf("actual broker issuance lifecycle differs: total=%d read-only=%d management=%d revocations=%d calls=%v", mints, readOnlyMints, managementMints, revocations, calls) } events := f.assertRetained(variant) diff --git a/experiments/g01-scaleset/livecanary/baseline_pair_fixture_test.go b/experiments/g01-scaleset/livecanary/baseline_pair_fixture_test.go index 2fc49cb5..8ab8d5d2 100644 --- a/experiments/g01-scaleset/livecanary/baseline_pair_fixture_test.go +++ b/experiments/g01-scaleset/livecanary/baseline_pair_fixture_test.go @@ -20,6 +20,7 @@ import ( "time" "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/liveworker" + "github.com/1XP-AI/gh-runnerd/experiments/g02-auth/handoff" ) const pairFixtureJIT = "c3ludGhldGljLXByaXZhdGUtaml0LWNvbmZpZw==" @@ -39,10 +40,11 @@ type pairedIntegrationFixture struct { } type pairedFixtureConfiguration struct { - controllerCreate func(*FileJournal, Approval) - workerPhases []string - controllerPhases []string - controllerResponse func(string, any) any + controllerCreate func(*FileJournal, Approval) + controllerGroupPolicy *handoff.GroupPolicy + workerPhases []string + controllerPhases []string + controllerResponse func(string, any) any } func newPairedIntegrationFixture(t *testing.T) *pairedIntegrationFixture { @@ -52,6 +54,9 @@ func newPairedIntegrationFixtureConfigured(t *testing.T, config *pairedFixtureCo t.Helper() f := &pairedIntegrationFixture{} a := approval() + if config != nil && config.controllerGroupPolicy != nil { + a.GroupPolicy = *config.controllerGroupPolicy + } if config != nil && config.controllerPhases != nil { a.Phases = append([]string(nil), config.controllerPhases...) } diff --git a/experiments/g01-scaleset/livecanary/baseline_terminal_fixture_test.go b/experiments/g01-scaleset/livecanary/baseline_terminal_fixture_test.go index d873ec78..00632190 100644 --- a/experiments/g01-scaleset/livecanary/baseline_terminal_fixture_test.go +++ b/experiments/g01-scaleset/livecanary/baseline_terminal_fixture_test.go @@ -17,6 +17,7 @@ import ( "testing" "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/liveworker" + "github.com/1XP-AI/gh-runnerd/experiments/g02-auth/handoff" ) type terminalFixture struct { @@ -46,6 +47,10 @@ func newTerminalFixtureWithControllerPhases(t *testing.T, phases, controllerPhas return newTerminalFixtureWithCreate(t, phases, controllerPhases, nil) } func newTerminalFixtureWithCreate(t *testing.T, phases, controllerPhases []string, create func(*FileJournal, Approval)) *terminalFixture { + return newTerminalFixtureWithControllerGroupPolicy(t, phases, controllerPhases, create, nil) +} + +func newTerminalFixtureWithControllerGroupPolicy(t *testing.T, phases, controllerPhases []string, create func(*FileJournal, Approval), groupPolicy *handoff.GroupPolicy) *terminalFixture { t.Helper() f := &terminalFixture{setETag: `"fixture-v1"`} config := &pairedFixtureConfiguration{controllerResponse: func(stage string, value any) any { @@ -61,6 +66,7 @@ func newTerminalFixtureWithCreate(t *testing.T, phases, controllerPhases []strin } return value }} + config.controllerGroupPolicy = groupPolicy config.controllerCreate = create config.workerPhases = phases config.controllerPhases = controllerPhases diff --git a/experiments/g01-scaleset/livecanary/controller_handoff_observed_test.go b/experiments/g01-scaleset/livecanary/controller_handoff_observed_test.go index c6a2c2e6..e2b9e8ee 100644 --- a/experiments/g01-scaleset/livecanary/controller_handoff_observed_test.go +++ b/experiments/g01-scaleset/livecanary/controller_handoff_observed_test.go @@ -30,7 +30,7 @@ func observedHandoffInputs(t *testing.T, f controllerHandoffFixture) (enrollment grant := enrollment.WorkflowGrantForFixture{Request: request, OrganizationID: 101, RepositoryID: f.approval.RepositoryID, ExpiresAt: f.now.Add(time.Hour)} repo := enrollment.WorkflowRepositoryForFixture{ID: grant.RepositoryID, FullName: request.Repository, Private: true} repo.Owner.ID, repo.Owner.Login = grant.OrganizationID, f.approval.Organization - run := enrollment.WorkflowRunForFixture{ID: request.WorkflowRunID, HeadSHA: request.WorkflowSHA, HeadBranch: "main", Event: "workflow_dispatch", Path: request.WorkflowPath, RunAttempt: 1, Repository: repo, HeadRepository: repo} + run := enrollment.WorkflowRunForFixture{ID: request.WorkflowRunID, HeadSHA: request.WorkflowSHA, HeadBranch: "main", Event: "workflow_dispatch", Status: "in_progress", Path: request.WorkflowPath, RunAttempt: 1, Repository: repo, HeadRepository: repo} return grant, run } diff --git a/experiments/g01-scaleset/livecanary/controller_process_broker_fixture_test.go b/experiments/g01-scaleset/livecanary/controller_process_broker_fixture_test.go index 01094359..c047e1fa 100644 --- a/experiments/g01-scaleset/livecanary/controller_process_broker_fixture_test.go +++ b/experiments/g01-scaleset/livecanary/controller_process_broker_fixture_test.go @@ -5,6 +5,7 @@ package livecanary import ( "bytes" "context" + "encoding/base64" "encoding/json" "fmt" "io" @@ -12,6 +13,7 @@ import ( "os" "path/filepath" "reflect" + "strconv" "strings" "testing" "time" @@ -21,19 +23,311 @@ import ( ) type processBrokerHTTP struct { - approval *Approval - t *testing.T - attempt string - calls []string - mints int - readOnlyMints int - managementMints int - revocations int - failPreflight bool - groupFault bool + approval *Approval + t *testing.T + attempt string + calls []string + expectedWorkflowAuthorizations []string + workflowAuthorizationsChecked int + mints int + readOnlyMints int + managementMints int + revocations int + readOnlyRevocations int + managementRevocations int + failPreflight bool + groupFault bool +} + +func TestProcessBrokerHTTPRejectsUnrecognizedAuthorization(t *testing.T) { + attempt := t.TempDir() + if err := os.WriteFile(filepath.Join(attempt, "broker.jsonl"), []byte("attempt\n"), 0600); err != nil { + t.Fatal(err) + } + approval := &Approval{AppID: 11, WorkflowRunID: 7, Organization: "fixture-org", Repository: "canary"} + now := time.Now() + wrongClaims, _ := json.Marshal(struct { + Issuer string `json:"iss"` + IssuedAt int64 `json:"iat"` + ExpiresAt int64 `json:"exp"` + }{"12", now.Add(-time.Minute).Unix(), now.Add(9 * time.Minute).Unix()}) + wrongAppJWT := "Bearer " + base64.RawURLEncoding.EncodeToString([]byte(`{"alg":"RS256","typ":"JWT"}`)) + "." + base64.RawURLEncoding.EncodeToString(wrongClaims) + ".c2ln" + cases := []struct { + name string + path string + method string + authorization string + }{ + {name: "app JWT identity", path: "/app", method: http.MethodGet, authorization: wrongAppJWT}, + {name: "mint JWT identity", path: "/app/installations/12/access_tokens", method: http.MethodPost, authorization: wrongAppJWT}, + {name: "workflow observation", path: workflowRunFixturePath(&processBrokerHTTP{approval: approval}), method: http.MethodGet, authorization: "Bearer synthetic-process-verification-token"}, + {name: "read-only preflight", path: "/installation/repositories", method: http.MethodGet, authorization: "Bearer synthetic-process-installation-token"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + transport := &processBrokerHTTP{approval: approval, t: t, attempt: attempt, readOnlyMints: 1, expectedWorkflowAuthorizations: []string{"Bearer synthetic-observation-token"}} + request, err := http.NewRequest(tc.method, "https://api.github.com"+tc.path, nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Authorization", tc.authorization) + response, err := transport.RoundTrip(request) + if response != nil { + response.Body.Close() + } + if err == nil { + t.Fatal("unrecognized request authority was accepted") + } + if len(transport.calls) != 0 || transport.mints != 0 || transport.revocations != 0 { + t.Fatalf("rejected authority crossed fixture effect boundary: calls=%v mints=%d revocations=%d", transport.calls, transport.mints, transport.revocations) + } + }) + } +} + +func (f *processBrokerHTTP) validateAuthorization(r *http.Request) error { + if f.approval == nil { + return nil + } + authorization := r.Header.Get("Authorization") + switch { + case r.URL.Path == workflowRunFixturePath(f): + if r.Method != http.MethodGet || f.workflowAuthorizationsChecked >= len(f.expectedWorkflowAuthorizations) || authorization != f.expectedWorkflowAuthorizations[f.workflowAuthorizationsChecked] { + return fmt.Errorf("unexpected synthetic workflow authority") + } + f.workflowAuthorizationsChecked++ + return nil + case r.URL.Path == "/app" || r.URL.Path == "/orgs/fixture-org/installation" || r.URL.Path == "/app/installations/12/access_tokens": + if !processAppJWTMatches(authorization, f.approval.AppID, time.Now()) { + return fmt.Errorf("unexpected synthetic App authority") + } + return nil + case r.URL.Path == "/installation/token": + if r.Method != http.MethodDelete { + return fmt.Errorf("unexpected synthetic revocation method") + } + switch authorization { + case "Bearer synthetic-process-read-installation-token": + if f.readOnlyRevocations >= f.readOnlyMints || f.managementRevocations != f.managementMints { + return fmt.Errorf("unexpected synthetic read-token revocation order") + } + case "Bearer synthetic-process-installation-token": + if f.managementRevocations >= f.managementMints { + return fmt.Errorf("unexpected synthetic management-token revocation order") + } + default: + return fmt.Errorf("unexpected synthetic revocation authority") + } + return nil + case isProcessBrokerPreflightPath(r.URL.Path): + expected := "" + if f.managementMints > f.managementRevocations { + expected = "Bearer synthetic-process-installation-token" + } else if f.readOnlyMints > f.readOnlyRevocations { + expected = "Bearer synthetic-process-read-installation-token" + } + if r.Method != http.MethodGet || expected == "" || authorization != expected { + return fmt.Errorf("unexpected synthetic preflight authority") + } + return nil + default: + return nil + } +} + +func isProcessBrokerPreflightPath(path string) bool { + switch path { + case "/installation/repositories", "/repos/fixture-org/canary", "/orgs/fixture-org/actions/runner-groups/3", "/orgs/fixture-org/actions/runner-groups/3/repositories": + return true + default: + return false + } +} + +func expectedProcessBrokerV3Trace(approval Approval, readGroupReject bool) []string { + workflowGet := "GET " + workflowRunFixturePath(&processBrokerHTTP{approval: &approval}) + trace := []string{workflowGet, "GET /app", "GET /orgs/fixture-org/installation", "POST /app/installations/12/access_tokens"} + appendPreflight := func(rejectGroup bool) { + if approval.ManagementScope != "organization-only" { + trace = append(trace, "GET /installation/repositories", "GET /repos/fixture-org/canary") + } + trace = append(trace, "GET /orgs/fixture-org/actions/runner-groups/3") + if !rejectGroup { + trace = append(trace, "GET /orgs/fixture-org/actions/runner-groups/3/repositories") + } + } + appendPreflight(readGroupReject) + trace = append(trace, "DELETE /installation/token") + if readGroupReject { + return trace + } + trace = append(trace, workflowGet, "POST /app/installations/12/access_tokens") + appendPreflight(false) + trace = append(trace, workflowGet, "DELETE /installation/token") + return trace +} + +func assertProcessBrokerTrace(t *testing.T, transport *processBrokerHTTP, want []string) { + t.Helper() + if !reflect.DeepEqual(transport.calls, want) { + t.Fatalf("broker request order differs: got=%v want=%v", transport.calls, want) + } + workflowGets := 0 + for _, call := range want { + if call == "GET "+workflowRunFixturePath(transport) { + workflowGets++ + } + } + if transport.workflowAuthorizationsChecked != workflowGets { + t.Fatalf("workflow authority count=%d, want one checked identity per GET (%d)", transport.workflowAuthorizationsChecked, workflowGets) + } +} + +type processTokenLifecycleRecord struct { + Phase string `json:"phase"` + Kind string `json:"kind"` + Slot string `json:"slot"` + Attempt json.RawMessage `json:"attempt"` + Journal json.RawMessage `json:"journal"` +} + +func assertProcessBrokerTokenLifecycle(t *testing.T, attemptDirectory, admissionDirectory string, wantRead, wantManagement bool) { + t.Helper() + readLifecycleRecords := func(path string, include map[string]bool) []processTokenLifecycleRecord { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("durable broker lifecycle unavailable: %v", err) + } + var records []processTokenLifecycleRecord + for _, line := range bytes.Split(data, []byte("\n")) { + if len(line) == 0 { + continue + } + var record processTokenLifecycleRecord + if err := json.Unmarshal(line, &record); err != nil { + t.Fatalf("durable lifecycle record malformed: %v", err) + } + name := record.Kind + if record.Phase != "" { + name = record.Phase + } + if include[name] { + record.Kind = name + records = append(records, record) + } + } + return records + } + brokerKinds := map[string]bool{ + "read_only_token_revocation_started": true, + "read_only_token_revoked": true, + "read_only_token_revocation_uncertain": true, + "read_only_token_revocation_skipped": true, + "management_token_revocation_started": true, + "management_token_revoked": true, + "management_token_revocation_uncertain": true, + "management_token_revocation_skipped": true, + } + wantBroker := make([]string, 0, 4) + if wantRead { + wantBroker = append(wantBroker, "read_only_token_revocation_started", "read_only_token_revoked") + } + if wantManagement { + wantBroker = append(wantBroker, "management_token_revocation_started", "management_token_revoked") + } + gotBroker := readLifecycleRecords(filepath.Join(attemptDirectory, "broker.jsonl"), brokerKinds) + var gotBrokerKinds []string + for _, record := range gotBroker { + gotBrokerKinds = append(gotBrokerKinds, record.Kind) + } + if !reflect.DeepEqual(gotBrokerKinds, wantBroker) { + t.Fatalf("durable broker revocation order differs: got=%v want=%v", gotBrokerKinds, wantBroker) + } + + admissionKinds := map[string]bool{ + "read_token_lifecycle_started": true, + "read_token_lifecycle_confirmed": true, + "read_token_revocation_started": true, + "read_token_revocation_confirmed": true, + "read_token_request_not_dispatched": true, + "management_token_lifecycle_started": true, + "management_token_lifecycle_confirmed": true, + "management_token_request_not_dispatched": true, + } + wantAdmission := make([]string, 0, 5) + if wantRead { + wantAdmission = append(wantAdmission, "read_token_lifecycle_started", "read_token_lifecycle_confirmed") + } + if wantManagement { + wantAdmission = append(wantAdmission, "management_token_lifecycle_started", "management_token_lifecycle_confirmed") + } else { + wantAdmission = append(wantAdmission, "management_token_request_not_dispatched") + } + gotAdmission := readLifecycleRecords(filepath.Join(admissionDirectory, "broker-admission.jsonl"), admissionKinds) + var gotAdmissionKinds []string + for _, record := range gotAdmission { + gotAdmissionKinds = append(gotAdmissionKinds, record.Kind) + } + if !reflect.DeepEqual(gotAdmissionKinds, wantAdmission) { + t.Fatalf("durable admission lifecycle differs: got=%v want=%v", gotAdmissionKinds, wantAdmission) + } + for index := 0; index+1 < len(gotAdmission); index += 2 { + started, confirmed := gotAdmission[index], gotAdmission[index+1] + if !strings.HasSuffix(started.Kind, "_started") || !strings.HasSuffix(confirmed.Kind, "_confirmed") || started.Slot == "" || started.Slot != confirmed.Slot || !bytes.Equal(started.Attempt, confirmed.Attempt) || !bytes.Equal(started.Journal, confirmed.Journal) { + t.Fatalf("durable token lifecycle identity differs: started=%+v confirmed=%+v", started, confirmed) + } + } + if wantRead && wantManagement && (gotAdmission[0].Slot != gotAdmission[2].Slot || !bytes.Equal(gotAdmission[0].Attempt, gotAdmission[2].Attempt) || !bytes.Equal(gotAdmission[0].Journal, gotAdmission[2].Journal)) { + t.Fatal("read and management token lifecycle records do not share one exact broker claim") + } + if wantRead && !wantManagement { + last := gotAdmission[len(gotAdmission)-1] + if last.Kind != "management_token_request_not_dispatched" || last.Slot != gotAdmission[0].Slot || !bytes.Equal(last.Attempt, gotAdmission[0].Attempt) || !bytes.Equal(last.Journal, gotAdmission[0].Journal) { + t.Fatal("unissued management token record is not bound to the read-token claim") + } + } +} + +// The adapter's focused unit test verifies the RS256 signature against its +// synthetic key. Process fixtures additionally bind provider requests to the +// intended App identity and the broker's bounded ten-minute JWT lifetime. +func processAppJWTMatches(authorization string, appID int64, now time.Time) bool { + if appID < 1 || !strings.HasPrefix(authorization, "Bearer ") { + return false + } + parts := strings.Split(strings.TrimPrefix(authorization, "Bearer "), ".") + if len(parts) != 3 { + return false + } + headerBytes, headerErr := base64.RawURLEncoding.DecodeString(parts[0]) + claimsBytes, claimsErr := base64.RawURLEncoding.DecodeString(parts[1]) + signature, signatureErr := base64.RawURLEncoding.DecodeString(parts[2]) + if headerErr != nil || claimsErr != nil || signatureErr != nil || len(signature) == 0 { + return false + } + var header struct { + Algorithm string `json:"alg"` + Type string `json:"typ"` + } + var claims struct { + Issuer string `json:"iss"` + IssuedAt int64 `json:"iat"` + ExpiresAt int64 `json:"exp"` + } + if json.Unmarshal(headerBytes, &header) != nil || json.Unmarshal(claimsBytes, &claims) != nil { + return false + } + nowUnix := now.Unix() + return header.Algorithm == "RS256" && header.Type == "JWT" && claims.Issuer == strconv.FormatInt(appID, 10) && + claims.IssuedAt <= nowUnix && claims.IssuedAt >= now.Add(-2*time.Minute).Unix() && + claims.ExpiresAt > nowUnix && claims.ExpiresAt-nowUnix <= 10*60 && claims.ExpiresAt-claims.IssuedAt == 10*60 } func (f *processBrokerHTTP) RoundTrip(r *http.Request) (*http.Response, error) { + if err := f.validateAuthorization(r); err != nil { + return nil, err + } f.calls = append(f.calls, r.Method+" "+r.URL.Path) repo := map[string]any{"id": 42, "full_name": "fixture-org/canary", "private": true, "fork": false, "owner": map[string]any{"id": 101, "login": "fixture-org"}} var payload any @@ -53,7 +347,7 @@ func (f *processBrokerHTTP) RoundTrip(r *http.Request) (*http.Response, error) { runID = f.approval.WorkflowRunID sha = f.approval.WorkflowSHA } - run := map[string]any{"id": runID, "head_sha": sha, "head_branch": "main", "event": "workflow_dispatch", "path": ".github/workflows/canary.yml", "run_attempt": 1, "repository": repo, "head_repository": repo} + run := map[string]any{"id": runID, "head_sha": sha, "head_branch": "main", "event": "workflow_dispatch", "status": "in_progress", "path": ".github/workflows/canary.yml", "run_attempt": 1, "repository": repo, "head_repository": repo} if f.approval != nil && f.approval.GroupPolicy != (handoff.GroupPolicy{}) { workflow := f.approval.GroupPolicy.Workflow at := strings.LastIndexByte(workflow, '@') @@ -103,12 +397,23 @@ func (f *processBrokerHTTP) RoundTrip(r *http.Request) (*http.Response, error) { } case "/installation/token": f.revocations++ - if r.Method != http.MethodDelete || r.Header.Get("Authorization") != "Bearer synthetic-process-read-installation-token" { - f.t.Error("read-token revocation used the wrong method or authority") + if r.Method != http.MethodDelete { + f.t.Error("installation-token revocation used the wrong method") } stored, e := os.ReadFile(filepath.Join(f.attempt, "broker.jsonl")) - if e != nil || !bytes.Contains(stored, []byte("read_only_token_revocation_started")) { - f.t.Error("read-token revocation preceded durable intent") + switch r.Header.Get("Authorization") { + case "Bearer synthetic-process-read-installation-token": + f.readOnlyRevocations++ + if e != nil || !bytes.Contains(stored, []byte("read_only_token_revocation_started")) { + f.t.Error("read-token revocation preceded durable intent") + } + case "Bearer synthetic-process-installation-token": + f.managementRevocations++ + if e != nil || !bytes.Contains(stored, []byte("management_token_revocation_started")) { + f.t.Error("management-token revocation preceded durable intent") + } + default: + f.t.Error("installation-token revocation used an unexpected authority") } status = http.StatusNoContent case "/installation/repositories": @@ -136,6 +441,33 @@ func (f *processBrokerHTTP) RoundTrip(r *http.Request) (*http.Response, error) { } return &http.Response{StatusCode: status, Header: make(http.Header), Body: io.NopCloser(bytes.NewReader(raw)), ContentLength: int64(len(raw))}, nil } + +func TestProcessBrokerWorkflowRunFixtureProvidesInProgressStatus(t *testing.T) { + attempt := t.TempDir() + if err := os.WriteFile(filepath.Join(attempt, "broker.jsonl"), []byte("attempt\n"), 0600); err != nil { + t.Fatal(err) + } + transport := &processBrokerHTTP{t: t, attempt: attempt} + request, err := http.NewRequest(http.MethodGet, "https://api.github.com"+workflowRunFixturePath(transport), nil) + if err != nil { + t.Fatal(err) + } + response, err := transport.RoundTrip(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + var workflow struct { + Status string `json:"status"` + } + if err := json.NewDecoder(response.Body).Decode(&workflow); err != nil { + t.Fatal(err) + } + if workflow.Status != "in_progress" { + t.Fatalf("workflow fixture status = %q, want in_progress", workflow.Status) + } +} + func TestControllerProcessHandoffActualBrokerSeparateProcess(t *testing.T) { for _, preflightFail := range []bool{false, true} { t.Run(fmt.Sprint(preflightFail), func(t *testing.T) { @@ -181,7 +513,7 @@ func TestControllerProcessHandoffActualBrokerSeparateProcess(t *testing.T) { t.Fatal(e) } grant := enrollment.WorkflowGrantForFixture{Request: request, OrganizationID: 101, RepositoryID: a.RepositoryID, ExpiresAt: a.ExpiresAt} - transport := &processBrokerHTTP{t: t, attempt: files.StateDirectory, failPreflight: preflightFail} + transport := &processBrokerHTTP{t: t, attempt: files.StateDirectory, approval: &a, expectedWorkflowAuthorizations: []string{"Bearer synthetic-observation-token"}, failPreflight: preflightFail} ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() result, effects, e := enrollment.RunProcessHandoffForFixture(ctx, files, transport, grant) @@ -196,9 +528,11 @@ func TestControllerProcessHandoffActualBrokerSeparateProcess(t *testing.T) { if !preflightFail { want = append(want, "GET /orgs/fixture-org/actions/runner-groups/3/repositories") } + want = append(want, "DELETE /installation/token") if !reflect.DeepEqual(transport.calls, want) { t.Fatalf("operation order changed: %v", transport.calls) } + assertProcessBrokerTokenLifecycle(t, files.StateDirectory, filepath.Join(parent, "admission"), false, true) stored, _ := os.ReadFile(filepath.Join(files.ControllerStateDirectory, "journal.jsonl")) if bytes.Contains(stored, []byte("controller-handoff-consumed")) == preflightFail { t.Fatal("unexpected child consumption") diff --git a/experiments/g01-scaleset/livecanary/controller_v3_process_fixture_test.go b/experiments/g01-scaleset/livecanary/controller_v3_process_fixture_test.go index 9ad212fb..014bead7 100644 --- a/experiments/g01-scaleset/livecanary/controller_v3_process_fixture_test.go +++ b/experiments/g01-scaleset/livecanary/controller_v3_process_fixture_test.go @@ -170,7 +170,14 @@ func newControllerV3ProcessFixture(t *testing.T) controllerV3ProcessFixture { if err := os.WriteFile(brokerApprovalPath, brokerRaw, 0600); err != nil { t.Fatal(err) } - base := &processBrokerHTTP{t: t, attempt: attempt, approval: &approval} + base := &processBrokerHTTP{ + t: t, attempt: attempt, approval: &approval, + expectedWorkflowAuthorizations: []string{ + "Bearer synthetic-observation-token", + "Bearer synthetic-process-verification-token", + "Bearer synthetic-process-verification-token", + }, + } return controllerV3ProcessFixture{ controller: controller, files: enrollment.BrokerFiles{ @@ -276,15 +283,17 @@ func TestControllerProcessHandoffV3ActualBrokerChild(t *testing.T) { result, effects, err := enrollment.RunProcessHandoffForFixtureWithOptions(context.Background(), fixture.files, fixture.transport, fixture.grant, options) controllerEffects := controllerV3ProcessEffects(t, fixture) events := controllerV3ProcessEvents(t, fixture) + assertProcessBrokerTrace(t, fixture.transport.base, expectedProcessBrokerV3Trace(*fixture.transport.base.approval, false)) + assertProcessBrokerTokenLifecycle(t, fixture.files.StateDirectory, filepath.Join(filepath.Dir(fixture.files.StateDirectory), "admission"), true, true) if tc.wantSuccess { receipt, receiptErr := handoff.DecodeStrictReceipt(observedReceipt) binding, bindingErr := handoff.DecodeStrictBinding(observedBinding) - if err != nil || result.Status != "controller_completed" || effects.WorkflowSignatures != 1 || effects.BindingSignatures != 1 || fixture.transport.base.mints != 2 || fixture.transport.base.revocations != 1 || receiptErr != nil || bindingErr != nil || receipt.Callee != fixture.controller.request.Callee || binding.Request != fixture.controller.request || controllerEffects["factory_calls"] != 1 || controllerEffects["create_calls"] != 1 || len(events) != 7 { + if err != nil || result.Status != "controller_completed" || effects.WorkflowSignatures != 1 || effects.BindingSignatures != 1 || fixture.transport.base.mints != 2 || fixture.transport.base.revocations != 2 || fixture.transport.base.readOnlyRevocations != 1 || fixture.transport.base.managementRevocations != 1 || receiptErr != nil || bindingErr != nil || receipt.Callee != fixture.controller.request.Callee || binding.Request != fixture.controller.request || controllerEffects["factory_calls"] != 1 || controllerEffects["create_calls"] != 1 || len(events) != 7 { t.Fatalf("positive child handoff: result=%+v err=%v effects=%+v mints=%d calls=%v controller_effects=%v journal_events=%d receipt_err=%v binding_err=%v", result, err, effects, fixture.transport.base.mints, fixture.transport.base.calls, controllerEffects, len(events), receiptErr, bindingErr) } return } - if err == nil || result.Status == "controller_completed" || effects.WorkflowSignatures != 1 || effects.BindingSignatures != 1 || fixture.transport.base.mints != 2 || fixture.transport.base.revocations != 1 || controllerEffects["factory_calls"] != 0 || controllerEffects["create_calls"] != 0 || len(events) != 0 { + if err == nil || result.Status == "controller_completed" || effects.WorkflowSignatures != 1 || effects.BindingSignatures != 1 || fixture.transport.base.mints != 2 || fixture.transport.base.revocations != 2 || fixture.transport.base.readOnlyRevocations != 1 || fixture.transport.base.managementRevocations != 1 || controllerEffects["factory_calls"] != 0 || controllerEffects["create_calls"] != 0 || len(events) != 0 { t.Fatalf("rejected child handoff crossed effect boundary: result=%+v err=%v effects=%+v mints=%d calls=%v controller_effects=%v journal_events=%d", result, err, effects, fixture.transport.base.mints, fixture.transport.base.calls, controllerEffects, len(events)) } }) diff --git a/experiments/g01-scaleset/livecanary/drain_command_fixture_test.go b/experiments/g01-scaleset/livecanary/drain_command_fixture_test.go index 79177e29..b45ae8d7 100644 --- a/experiments/g01-scaleset/livecanary/drain_command_fixture_test.go +++ b/experiments/g01-scaleset/livecanary/drain_command_fixture_test.go @@ -624,8 +624,10 @@ func TestExperimentalDrainCommand(t *testing.T) { readOnlyMints := f.transport.readOnlyMints managementMints := f.transport.managementMints revocations := f.transport.revocations + readOnlyRevocations := f.transport.readOnlyRevocations + managementRevocations := f.transport.managementRevocations f.brokerMu.Unlock() - if mints != 2 || readOnlyMints != 1 || managementMints != 1 || revocations != 1 { + if mints != 2 || readOnlyMints != 1 || managementMints != 1 || revocations != 2 || readOnlyRevocations != 1 || managementRevocations != 1 { t.Errorf("actual signed broker issuance lifecycle differs: total=%d read-only=%d management=%d revocations=%d", mints, readOnlyMints, managementMints, revocations) } events := f.assertDrain() diff --git a/experiments/g01-scaleset/livecanary/phase_command_fixture_test.go b/experiments/g01-scaleset/livecanary/phase_command_fixture_test.go index 03f14ff2..146641ed 100644 --- a/experiments/g01-scaleset/livecanary/phase_command_fixture_test.go +++ b/experiments/g01-scaleset/livecanary/phase_command_fixture_test.go @@ -99,8 +99,7 @@ func TestExperimentalActualCommandBrokerReceiver(t *testing.T) { for _, caseName := range cases { t.Run(caseName, func(t *testing.T) { mode := caseName - groupCase := strings.HasPrefix(caseName, "group-policy") - if groupCase { + if strings.HasPrefix(caseName, "group-policy") { mode = "before-ack" } variant := "old41" @@ -126,16 +125,15 @@ func TestExperimentalActualCommandBrokerReceiver(t *testing.T) { var physicalJIT, workerReads int committedName := "" if mode == "paired-terminal" { - terminal = newTerminalFixtureWithCreate(t, []string{"create", "start", "inspect", "cleanup"}, nil, func(j *FileJournal, a Approval) { completedHandoffCreate(t, j, a) }) + groupPolicy := handoff.GroupPolicy{Version: 2, OrganizationID: 101, Workflow: "fixture-org/canary/.github/workflows/runner.yml@" + strings.Repeat("d", 40)} + terminal = newTerminalFixtureWithControllerGroupPolicy(t, []string{"create", "start", "inspect", "cleanup"}, nil, func(j *FileJournal, a Approval) { completedHandoffCreate(t, j, a) }, &groupPolicy) base = terminal.c fixture := newExportedPairedFixture(t, terminal) files = fixture.files } else { a := approval() a.WorkflowRef = "refs/heads/main" - if groupCase { - a.GroupPolicy = handoff.GroupPolicy{Version: 2, OrganizationID: 101, Workflow: a.Organization + "/" + a.Repository + "/.github/workflows/runner.yml@" + strings.Repeat("d", 40)} - } + a.GroupPolicy = handoff.GroupPolicy{Version: 2, OrganizationID: 101, Workflow: a.Organization + "/" + a.Repository + "/.github/workflows/runner.yml@" + strings.Repeat("d", 40)} if caseName == "group-policy-org" { a.ManagementScope = "organization-only" } @@ -212,7 +210,7 @@ func TestExperimentalActualCommandBrokerReceiver(t *testing.T) { body = repo case "/orgs/fixture-org/actions/runner-groups/3": body = map[string]any{"id": 3, "visibility": "selected", "default": false, "allows_public_repositories": false, "inherited": false} - if groupCase { + if a.GroupPolicy.Version == 2 { expected := a.GroupPolicy.Workflow if caseName == "group-policy-controller-wrong" { expected = strings.Replace(expected, "dddd", "eeee", 1) @@ -313,7 +311,12 @@ func TestExperimentalActualCommandBrokerReceiver(t *testing.T) { t.Fatal("broker approval") } brokerFiles := enrollment.BrokerFiles{ApprovalPath: approvalPath, StateDirectory: filepath.Join(parent, "attempt-"+mode), ControllerBinary: binPath, ControllerApproval: files.ControllerApprovalPath, ControllerStateDirectory: files.ControllerStateDirectory, WorkerApproval: files.WorkerApprovalPath, WorkerStateDirectory: files.WorkerStateDirectory} - transport := &processBrokerHTTP{t: t, attempt: brokerFiles.StateDirectory, approval: &a, groupFault: caseName == "group-policy-broker-wrong"} + verificationAuthorization := "Bearer " + base.api.credentials.VerificationToken + transport := &processBrokerHTTP{ + t: t, attempt: brokerFiles.StateDirectory, approval: &a, + expectedWorkflowAuthorizations: []string{"Bearer synthetic-observation-credential", verificationAuthorization, verificationAuthorization}, + groupFault: caseName == "group-policy-broker-wrong", + } if e := enrollment.ConfigureBrokerExperimentForFixture(experimentalPolicy(t, a, raw, brokerMode, phase, "broker"), transport, admission, workerAdmission); e != nil { t.Fatal(e) } @@ -354,7 +357,7 @@ func TestExperimentalActualCommandBrokerReceiver(t *testing.T) { defer server.Close() policy := experimentalPolicy(t, a, raw, brokerMode, phase, "broker") if caseName == "unknown-policy" { - policy = strings.Replace(policy, `"version":1`, `"version":1,"unreviewed":true`, 1) + policy = strings.Replace(policy, `"version":3`, `"version":3,"unreviewed":true`, 1) } config, _ := json.Marshal(map[string]string{"policy": policy, "base_url": server.URL, "ca_pem": string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})), "admission": admission, "worker_admission": workerAdmission}) if os.WriteFile(filepath.Join(files.ControllerStateDirectory, "broker-experiment-fixture.json"), config, 0600) != nil { @@ -418,8 +421,8 @@ func TestExperimentalActualCommandBrokerReceiver(t *testing.T) { return } if mode == replacementPhase && (variant == "missing" || variant == "null") { - if e == nil || transport.mints != 1 || !reflect.DeepEqual(replacement.snapshot(), []string{"set", "worker", "open1", "poll1"}) || base.acks.Load() != 0 || base.acquires.Load() != 0 { - t.Fatal("missing/null identity crossed replacement boundary", replacement.snapshot()) + if e == nil || transport.mints != 2 || transport.readOnlyMints != 1 || transport.managementMints != 1 || transport.revocations != 2 || transport.readOnlyRevocations != 1 || transport.managementRevocations != 1 || !reflect.DeepEqual(replacement.snapshot(), []string{"set", "worker", "open1", "poll1"}) || base.acks.Load() != 0 || base.acquires.Load() != 0 { + t.Fatalf("missing/null identity crossed replacement boundary: err=%v trace=%v mints=%d read-only=%d management=%d revocations=%d read-only-revocations=%d management-revocations=%d ack=%d acquire=%d", e, replacement.snapshot(), transport.mints, transport.readOnlyMints, transport.managementMints, transport.revocations, transport.readOnlyRevocations, transport.managementRevocations, base.acks.Load(), base.acquires.Load()) } return } @@ -432,10 +435,25 @@ func TestExperimentalActualCommandBrokerReceiver(t *testing.T) { t.Fatalf("actual %s command produced no typed physical boundary evidence (physical JIT %d)", mode, physicalJIT) } } - if caseName == "group-policy-broker-wrong" || caseName == "group-policy-controller-wrong" { - if e == nil || transport.mints != 1 || base.acks.Load() != 0 || base.acquires.Load() != 0 { + if caseName == "group-policy-broker-wrong" { + if e == nil || transport.mints != 1 || transport.readOnlyMints != 1 || transport.managementMints != 0 || transport.revocations != 1 || transport.readOnlyRevocations != 1 || transport.managementRevocations != 0 || base.acks.Load() != 0 || base.acquires.Load() != 0 { t.Fatal("wrong group crossed actual command boundary", e, transport.mints) } + assertProcessBrokerTrace(t, transport, expectedProcessBrokerV3Trace(a, true)) + assertProcessBrokerTokenLifecycle(t, brokerFiles.StateDirectory, admission, true, false) + for _, event := range base.readEvents() { + if event.ControllerHandoff != nil && event.ControllerHandoff.Phase == phase { + t.Fatal("broker-side group rejection consumed the signed handoff") + } + } + return + } + if caseName == "group-policy-controller-wrong" { + if e == nil || transport.mints != 2 || transport.readOnlyMints != 1 || transport.managementMints != 1 || transport.revocations != 2 || transport.readOnlyRevocations != 1 || transport.managementRevocations != 1 || base.acks.Load() != 0 || base.acquires.Load() != 0 { + t.Fatal("wrong controller group crossed actual command boundary", e, transport.mints) + } + assertProcessBrokerTrace(t, transport, expectedProcessBrokerV3Trace(a, false)) + assertProcessBrokerTokenLifecycle(t, brokerFiles.StateDirectory, admission, true, true) consumed := false for _, event := range base.readEvents() { if event.ControllerHandoff != nil && event.ControllerHandoff.Phase == phase { @@ -447,7 +465,7 @@ func TestExperimentalActualCommandBrokerReceiver(t *testing.T) { } return } - if e != nil || transport.mints != 1 { + if e != nil || transport.mints != 2 || transport.readOnlyMints != 1 || transport.managementMints != 1 || transport.revocations != 2 || transport.readOnlyRevocations != 1 || transport.managementRevocations != 1 { for _, path := range []string{filepath.Join(brokerFiles.StateDirectory, "broker.jsonl"), filepath.Join(files.ControllerStateDirectory, "journal.jsonl")} { data, _ := os.ReadFile(path) for _, line := range bytes.Split(data, []byte("\n")) { @@ -458,8 +476,10 @@ func TestExperimentalActualCommandBrokerReceiver(t *testing.T) { } } t.Logf("safe counts: requests=%d forbidden=%d", base.requests.Load(), base.forbidden.Load()) - t.Fatalf("actual %s command failed: %v result=%+v mints=%d calls=%v", mode, e, result, transport.mints, transport.calls) + t.Fatalf("actual %s command failed: %v result=%+v mints=%d read-only=%d management=%d revocations=%d read-only-revocations=%d management-revocations=%d calls=%v", mode, e, result, transport.mints, transport.readOnlyMints, transport.managementMints, transport.revocations, transport.readOnlyRevocations, transport.managementRevocations, transport.calls) } + assertProcessBrokerTrace(t, transport, expectedProcessBrokerV3Trace(a, false)) + assertProcessBrokerTokenLifecycle(t, brokerFiles.StateDirectory, admission, true, true) data, e := os.ReadFile(filepath.Join(files.ControllerStateDirectory, "journal.jsonl")) if e != nil { t.Fatal(e) diff --git a/experiments/g01-scaleset/livecanary/session_replacement_fixture_test.go b/experiments/g01-scaleset/livecanary/session_replacement_fixture_test.go index 00c4aeca..4d91225a 100644 --- a/experiments/g01-scaleset/livecanary/session_replacement_fixture_test.go +++ b/experiments/g01-scaleset/livecanary/session_replacement_fixture_test.go @@ -166,7 +166,12 @@ func (f *replacementTrace) extra(w http.ResponseWriter, r *http.Request) bool { case "/repos/fixture-org/canary": value = repo case "/orgs/fixture-org/actions/runner-groups/3": - value = map[string]any{"id": 3, "visibility": "selected", "default": false, "allows_public_repositories": false, "inherited": false} + group := map[string]any{"id": 3, "name": "fixture-group", "visibility": "selected", "default": false, "allows_public_repositories": false, "inherited": false} + if a.GroupPolicy.Version == 2 { + group["restricted_to_workflows"] = true + group["selected_workflows"] = []string{a.GroupPolicy.Workflow} + } + value = group default: return false } diff --git a/experiments/g02-auth/broker_workflow_provenance_fixture.go b/experiments/g02-auth/broker_workflow_provenance_fixture.go index 8cd4fc51..0ac065f0 100644 --- a/experiments/g02-auth/broker_workflow_provenance_fixture.go +++ b/experiments/g02-auth/broker_workflow_provenance_fixture.go @@ -40,6 +40,7 @@ type WorkflowRunForFixture struct { HeadSHA string `json:"head_sha"` HeadBranch string `json:"head_branch"` Event string `json:"event"` + Status string `json:"status"` Path string `json:"path"` RunAttempt int `json:"run_attempt"` Repository WorkflowRepositoryForFixture `json:"repository"` diff --git a/scripts/check-offline-experiments.sh b/scripts/check-offline-experiments.sh index 7135e41f..faf2a0fe 100644 --- a/scripts/check-offline-experiments.sh +++ b/scripts/check-offline-experiments.sh @@ -47,8 +47,12 @@ paired_prep_regex='^TestPairedBrokerPrepareReviewedG01LiveBinary$' paired_prep_or_cadence_regex='^TestPairedBroker(PrepareReviewedG01LiveBinary|RealCadenceChildExceedsThirtySeconds)$' paired_broker_regex='^TestPaired' non_paired_heavy_regex='^Test(LegacyControllerEntryRejectsMalformedInputBeforeAdmission|LegacyPairedEntryRejectsMalformedInputBeforeAdmission|BrokerControllerClaimsAttemptBeforeProvenanceSideEffects)$' -final_fence_regex='^TestBrokerExperimentV3(FinalGETFencesCalleeBeforeMintAndHandoff|ControllerRootMismatchRefusesBeforeManagementInputAndMint|CredentialInputFailureConsumesAttempt)$' -g02_non_paired_skip_regex='^TestPaired|^TestBrokerExperimentV3(FinalGETFencesCalleeBeforeMintAndHandoff|ControllerRootMismatchRefusesBeforeManagementInputAndMint|CredentialInputFailureConsumesAttempt)$|^Test(LegacyControllerEntryRejectsMalformedInputBeforeAdmission|LegacyPairedEntryRejectsMalformedInputBeforeAdmission|BrokerControllerClaimsAttemptBeforeProvenanceSideEffects)$' +final_fence_regex='^TestBrokerExperimentV3(FinalGETFencesCalleeBeforeMintAndHandoff|ControllerRootMismatchRefusesBeforeManagementInputAndMint|CredentialInputFailureConsumesAttempt|WorkflowProvenanceTimeoutBlocksLaterInspect)$' +v3_management_regex='^TestBrokerExperimentV3(ManagementTokenRevokedBeforeClaimCompletion|RevokesKnownRejectedManagementTokens|LostManagementTokenResponseBlocksLaterInspection|ManagementRequestJournalFailureRecordsNoDispatch|UncertainManagementTokenRevocationBlocksLaterInspection|DoesNotRevokeCallerVerificationTokenCollision)$' +v3_read_only_regex='^TestBrokerExperimentV3(ReadOnlyPreflightRejectsBeforeWriteMint|StopsWhenReadOnlyTokenRevocationIsUncertain|RevokedRecordSyncFailureStopsBeforeWriteMint|DoubleRevocationJournalSyncFailureBlocksLaterInspect|LostReadTokenMintResponseBlocksLaterInspect|RevokesTokenFromRejectedReadIssuance|DoesNotRevokeVerificationTokenCollision|SelectionRejectsMissingStateBeforePipeOwnership)$' +v3_provenance_regex='^TestBrokerExperimentV3(LostWorkflowProvenanceResponseBlocksLaterInspect|WorkflowProvenancePreRequestFailureRecordsNoDispatchForLaterInspect|HistoricalWorkflowProvenanceClaimReplayBinding|LegacyIncompleteClaimWithoutManagementLifecycleBlocksLaterInspect|PreRequestFailureRecordsNoDispatchForLaterInspect|RepeatsCalleeFenceAfterWritePreflight)$' +paired_process_regex='^TestBrokerPaired' +g02_non_paired_skip_regex='^TestPaired|^TestBrokerPaired|^TestBrokerExperimentV3|^Test(LegacyControllerEntryRejectsMalformedInputBeforeAdmission|LegacyPairedEntryRejectsMalformedInputBeforeAdmission|BrokerControllerClaimsAttemptBeforeProvenanceSideEffects)$' # These are the established offline gate modules. Keep this list explicit so # a new or unreviewed experiment cannot enter public CI by directory naming. @@ -117,10 +121,10 @@ for module_dir in "${selected_modules[@]}"; do # Bounded fixture clone/build is a separate 45-second process so the # cadence partition keeps the production seven-times-five-second wait # without hiding preparation in an unbounded helper. Then isolate the - # exact cadence name, the bounded non-paired heavy family, the final v3 - # fence (whose fixture hashes the race test binary), the remaining - # TestPaired family, and an unfiltered complement so Example Output and - # fuzz seeds still run. + # exact cadence name, bounded non-paired heavy family, final fences, and + # the v3 management, read-only/pre-effect selection, and provenance families. Keep + # paired broker-child binary checks separate from paired fixtures too. + # The remaining complements still discover tests, examples, and fuzz seeds. # Package discovery stays on ./... in every command. g02_prep_dir=$(mktemp -d) # Remove only this owned mktemp path. EXIT covers success and set -e @@ -136,6 +140,10 @@ for module_dir in "${selected_modules[@]}"; do GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -run "${real_pair_cadence_regex}" ./... GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -run "${non_paired_heavy_regex}" ./... GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -run "${final_fence_regex}" ./... + GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -run "${v3_management_regex}" ./... + GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -run "${v3_read_only_regex}" ./... + GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -run "${v3_provenance_regex}" ./... + GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -run "${paired_process_regex}" ./... GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -run "${paired_broker_regex}" -skip "${paired_prep_or_cadence_regex}" ./... GOTOOLCHAIN="${exact_toolchain}" "${go_cmd}" test -race -count=1 -timeout=45s -skip "${g02_non_paired_skip_regex}" ./... unset G01_PAIR_BRIDGE_PREP_DIR diff --git a/scripts/tooling_test.go b/scripts/tooling_test.go index 3c19ca31..ab743763 100644 --- a/scripts/tooling_test.go +++ b/scripts/tooling_test.go @@ -3414,9 +3414,13 @@ func FuzzDefaultG01Fixture(f *testing.F) { "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestPairedBrokerPrepareReviewedG01LiveBinary$ ./...", "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", "go1.27.3\ttest -race -count=1 -timeout=45s -run ^Test(LegacyControllerEntryRejectsMalformedInputBeforeAdmission|LegacyPairedEntryRejectsMalformedInputBeforeAdmission|BrokerControllerClaimsAttemptBeforeProvenanceSideEffects)$ ./...", - "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestBrokerExperimentV3(FinalGETFencesCalleeBeforeMintAndHandoff|ControllerRootMismatchRefusesBeforeManagementInputAndMint|CredentialInputFailureConsumesAttempt)$ ./...", + "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestBrokerExperimentV3(FinalGETFencesCalleeBeforeMintAndHandoff|ControllerRootMismatchRefusesBeforeManagementInputAndMint|CredentialInputFailureConsumesAttempt|WorkflowProvenanceTimeoutBlocksLaterInspect)$ ./...", + "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestBrokerExperimentV3(ManagementTokenRevokedBeforeClaimCompletion|RevokesKnownRejectedManagementTokens|LostManagementTokenResponseBlocksLaterInspection|ManagementRequestJournalFailureRecordsNoDispatch|UncertainManagementTokenRevocationBlocksLaterInspection|DoesNotRevokeCallerVerificationTokenCollision)$ ./...", + "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestBrokerExperimentV3(ReadOnlyPreflightRejectsBeforeWriteMint|StopsWhenReadOnlyTokenRevocationIsUncertain|RevokedRecordSyncFailureStopsBeforeWriteMint|DoubleRevocationJournalSyncFailureBlocksLaterInspect|LostReadTokenMintResponseBlocksLaterInspect|RevokesTokenFromRejectedReadIssuance|DoesNotRevokeVerificationTokenCollision|SelectionRejectsMissingStateBeforePipeOwnership)$ ./...", + "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestBrokerExperimentV3(LostWorkflowProvenanceResponseBlocksLaterInspect|WorkflowProvenancePreRequestFailureRecordsNoDispatchForLaterInspect|HistoricalWorkflowProvenanceClaimReplayBinding|LegacyIncompleteClaimWithoutManagementLifecycleBlocksLaterInspect|PreRequestFailureRecordsNoDispatchForLaterInspect|RepeatsCalleeFenceAfterWritePreflight)$ ./...", + "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestBrokerPaired ./...", "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestPaired -skip ^TestPairedBroker(PrepareReviewedG01LiveBinary|RealCadenceChildExceedsThirtySeconds)$ ./...", - "go1.27.3\ttest -race -count=1 -timeout=45s -skip ^TestPaired|^TestBrokerExperimentV3(FinalGETFencesCalleeBeforeMintAndHandoff|ControllerRootMismatchRefusesBeforeManagementInputAndMint|CredentialInputFailureConsumesAttempt)$|^Test(LegacyControllerEntryRejectsMalformedInputBeforeAdmission|LegacyPairedEntryRejectsMalformedInputBeforeAdmission|BrokerControllerClaimsAttemptBeforeProvenanceSideEffects)$ ./...", + "go1.27.3\ttest -race -count=1 -timeout=45s -skip ^TestPaired|^TestBrokerPaired|^TestBrokerExperimentV3|^Test(LegacyControllerEntryRejectsMalformedInputBeforeAdmission|LegacyPairedEntryRejectsMalformedInputBeforeAdmission|BrokerControllerClaimsAttemptBeforeProvenanceSideEffects)$ ./...", } { count := 0 for _, line := range lines { @@ -3494,9 +3498,21 @@ func TestToolingDefaultG02PartitionsRun(t *testing.T) { const finalName = "TestBrokerExperimentV3FinalGETFencesCalleeBeforeMintAndHandoff" const finalRootMismatchName = "TestBrokerExperimentV3ControllerRootMismatchRefusesBeforeManagementInputAndMint" const finalInputFailureName = "TestBrokerExperimentV3CredentialInputFailureConsumesAttempt" + const workflowTimeoutName = "TestBrokerExperimentV3WorkflowProvenanceTimeoutBlocksLaterInspect" + const pairedBinaryPrefixName = "TestBrokerPairedTerminalRealDigestPrefixDoesNotTriggerFixtureSleep" + const v3ManagementName = "TestBrokerExperimentV3ManagementTokenRevokedBeforeClaimCompletion" + const v3ReadOnlyName = "TestBrokerExperimentV3ReadOnlyPreflightRejectsBeforeWriteMint" + const v3SelectionName = "TestBrokerExperimentV3SelectionRejectsMissingStateBeforePipeOwnership" + const v3ProvenanceName = "TestBrokerExperimentV3LostWorkflowProvenanceResponseBlocksLaterInspect" const finalSentinel = "default-g02-final-regression" const finalRootMismatchSentinel = "default-g02-final-root-mismatch-regression" const finalInputFailureSentinel = "default-g02-final-input-failure-regression" + const workflowTimeoutSentinel = "default-g02-workflow-timeout-regression" + const pairedBinaryPrefixSentinel = "default-g02-paired-binary-prefix-regression" + const v3ManagementSentinel = "default-g02-v3-management-regression" + const v3ReadOnlySentinel = "default-g02-v3-read-only-regression" + const v3SelectionSentinel = "default-g02-v3-selection-regression" + const v3ProvenanceSentinel = "default-g02-v3-provenance-regression" const pairedFamilyName = "TestPairedBrokerClaimFenceFixture" const pairedFamilySentinel = "default-g02-paired-family-regression" const remainderSentinel = "default-g02-remainder-regression" @@ -3678,13 +3694,61 @@ func FuzzG02Fixture(f *testing.F) { failureSource: defaultTestSource("fixture", finalInputFailureName, finalInputFailureSentinel, finalInputFailureSentinel), partition: 3, }, + { + name: "workflow provenance timeout", + path: g02Base + "/default_workflow_timeout_regression_test.go", + marker: workflowTimeoutSentinel, + positiveSource: defaultTestSource("fixture", workflowTimeoutName, workflowTimeoutSentinel, ""), + failureSource: defaultTestSource("fixture", workflowTimeoutName, workflowTimeoutSentinel, workflowTimeoutSentinel), + partition: 3, + }, + { + name: "v3 management lifecycle", + path: g02Base + "/default_v3_management_regression_test.go", + marker: v3ManagementSentinel, + positiveSource: defaultTestSource("fixture", v3ManagementName, v3ManagementSentinel, ""), + failureSource: defaultTestSource("fixture", v3ManagementName, v3ManagementSentinel, v3ManagementSentinel), + partition: 4, + }, + { + name: "v3 read-only lifecycle", + path: g02Base + "/default_v3_read_only_regression_test.go", + marker: v3ReadOnlySentinel, + positiveSource: defaultTestSource("fixture", v3ReadOnlyName, v3ReadOnlySentinel, ""), + failureSource: defaultTestSource("fixture", v3ReadOnlyName, v3ReadOnlySentinel, v3ReadOnlySentinel), + partition: 5, + }, + { + name: "v3 selection before pipe ownership", + path: g02Base + "/default_v3_selection_regression_test.go", + marker: v3SelectionSentinel, + positiveSource: defaultTestSource("fixture", v3SelectionName, v3SelectionSentinel, ""), + failureSource: defaultTestSource("fixture", v3SelectionName, v3SelectionSentinel, v3SelectionSentinel), + partition: 5, + }, + { + name: "v3 provenance recovery", + path: g02Base + "/default_v3_provenance_regression_test.go", + marker: v3ProvenanceSentinel, + positiveSource: defaultTestSource("fixture", v3ProvenanceName, v3ProvenanceSentinel, ""), + failureSource: defaultTestSource("fixture", v3ProvenanceName, v3ProvenanceSentinel, v3ProvenanceSentinel), + partition: 6, + }, + { + name: "paired binary prefix", + path: g02Base + "/default_paired_binary_prefix_regression_test.go", + marker: pairedBinaryPrefixSentinel, + positiveSource: defaultTestSource("fixture", pairedBinaryPrefixName, pairedBinaryPrefixSentinel, ""), + failureSource: defaultTestSource("fixture", pairedBinaryPrefixName, pairedBinaryPrefixSentinel, pairedBinaryPrefixSentinel), + partition: 7, + }, { name: "paired family", path: g02Base + "/default_paired_family_regression_test.go", marker: pairedFamilySentinel, positiveSource: defaultTestSource("fixture", pairedFamilyName, pairedFamilySentinel, ""), failureSource: defaultTestSource("fixture", pairedFamilyName, pairedFamilySentinel, pairedFamilySentinel), - partition: 4, + partition: 8, }, { name: "remainder", @@ -3692,7 +3756,7 @@ func FuzzG02Fixture(f *testing.F) { marker: remainderSentinel, positiveSource: defaultTestSource("remainderfixture", "TestDefaultG02RemainderFixture", remainderSentinel, ""), failureSource: defaultTestSource("remainderfixture", "TestDefaultG02RemainderFixture", remainderSentinel, remainderSentinel), - partition: 5, + partition: 9, }, { name: "other package", @@ -3700,7 +3764,7 @@ func FuzzG02Fixture(f *testing.F) { marker: otherPackageSentinel, positiveSource: defaultTestSource("otherfixture", "TestDefaultG02OtherPackageFixture", otherPackageSentinel, ""), failureSource: defaultTestSource("otherfixture", "TestDefaultG02OtherPackageFixture", otherPackageSentinel, otherPackageSentinel), - partition: 5, + partition: 9, }, { name: "same-name other package", @@ -3716,7 +3780,7 @@ func FuzzG02Fixture(f *testing.F) { marker: sameNamePairedFamilySentinel, positiveSource: defaultTestSource("pairedfixture", pairedFamilyName, sameNamePairedFamilySentinel, ""), failureSource: defaultTestSource("pairedfixture", pairedFamilyName, sameNamePairedFamilySentinel, sameNamePairedFamilySentinel), - partition: 4, + partition: 8, }, { name: "same-name prep other package", @@ -3732,7 +3796,7 @@ func FuzzG02Fixture(f *testing.F) { marker: exampleSentinel, positiveSource: defaultExampleSource(exampleSentinel, exampleSentinel), failureSource: defaultExampleSource(exampleSentinel, "unexpected-default-g02-example-output"), - partition: 5, + partition: 9, }, { name: "Fuzz seed", @@ -3740,7 +3804,7 @@ func FuzzG02Fixture(f *testing.F) { marker: fuzzSentinel, positiveSource: defaultFuzzSource(fuzzSentinel, ""), failureSource: defaultFuzzSource(fuzzSentinel, fuzzSentinel), - partition: 5, + partition: 9, }, } toolingFile(t, root, remainderPackageBase+"/fixture.go", "package remainderfixture\n", 0600) @@ -3763,9 +3827,13 @@ func FuzzG02Fixture(f *testing.F) { "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestPairedBrokerPrepareReviewedG01LiveBinary$ ./...", "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestPairedBrokerRealCadenceChildExceedsThirtySeconds$ ./...", "go1.27.3\ttest -race -count=1 -timeout=45s -run ^Test(LegacyControllerEntryRejectsMalformedInputBeforeAdmission|LegacyPairedEntryRejectsMalformedInputBeforeAdmission|BrokerControllerClaimsAttemptBeforeProvenanceSideEffects)$ ./...", - "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestBrokerExperimentV3(FinalGETFencesCalleeBeforeMintAndHandoff|ControllerRootMismatchRefusesBeforeManagementInputAndMint|CredentialInputFailureConsumesAttempt)$ ./...", + "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestBrokerExperimentV3(FinalGETFencesCalleeBeforeMintAndHandoff|ControllerRootMismatchRefusesBeforeManagementInputAndMint|CredentialInputFailureConsumesAttempt|WorkflowProvenanceTimeoutBlocksLaterInspect)$ ./...", + "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestBrokerExperimentV3(ManagementTokenRevokedBeforeClaimCompletion|RevokesKnownRejectedManagementTokens|LostManagementTokenResponseBlocksLaterInspection|ManagementRequestJournalFailureRecordsNoDispatch|UncertainManagementTokenRevocationBlocksLaterInspection|DoesNotRevokeCallerVerificationTokenCollision)$ ./...", + "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestBrokerExperimentV3(ReadOnlyPreflightRejectsBeforeWriteMint|StopsWhenReadOnlyTokenRevocationIsUncertain|RevokedRecordSyncFailureStopsBeforeWriteMint|DoubleRevocationJournalSyncFailureBlocksLaterInspect|LostReadTokenMintResponseBlocksLaterInspect|RevokesTokenFromRejectedReadIssuance|DoesNotRevokeVerificationTokenCollision|SelectionRejectsMissingStateBeforePipeOwnership)$ ./...", + "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestBrokerExperimentV3(LostWorkflowProvenanceResponseBlocksLaterInspect|WorkflowProvenancePreRequestFailureRecordsNoDispatchForLaterInspect|HistoricalWorkflowProvenanceClaimReplayBinding|LegacyIncompleteClaimWithoutManagementLifecycleBlocksLaterInspect|PreRequestFailureRecordsNoDispatchForLaterInspect|RepeatsCalleeFenceAfterWritePreflight)$ ./...", + "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestBrokerPaired ./...", "go1.27.3\ttest -race -count=1 -timeout=45s -run ^TestPaired -skip ^TestPairedBroker(PrepareReviewedG01LiveBinary|RealCadenceChildExceedsThirtySeconds)$ ./...", - "go1.27.3\ttest -race -count=1 -timeout=45s -skip ^TestPaired|^TestBrokerExperimentV3(FinalGETFencesCalleeBeforeMintAndHandoff|ControllerRootMismatchRefusesBeforeManagementInputAndMint|CredentialInputFailureConsumesAttempt)$|^Test(LegacyControllerEntryRejectsMalformedInputBeforeAdmission|LegacyPairedEntryRejectsMalformedInputBeforeAdmission|BrokerControllerClaimsAttemptBeforeProvenanceSideEffects)$ ./...", + "go1.27.3\ttest -race -count=1 -timeout=45s -skip ^TestPaired|^TestBrokerPaired|^TestBrokerExperimentV3|^Test(LegacyControllerEntryRejectsMalformedInputBeforeAdmission|LegacyPairedEntryRejectsMalformedInputBeforeAdmission|BrokerControllerClaimsAttemptBeforeProvenanceSideEffects)$ ./...", } // This positive control intentionally runs the full default module list. // The bounded G01 partitions add enough subprocesses to exceed the old limit.